From 8ba831a8e4965938f6fa27fc7e0f2083297899ac Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 12:03:30 -0500 Subject: [PATCH 001/201] WIP. Here's a letsencrypt-auto script that downloads a new copy of itself, checks a signature on it, and replaces itself with it. --- booty.sh | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 booty.sh diff --git a/booty.sh b/booty.sh new file mode 100755 index 000000000..b9d9e6583 --- /dev/null +++ b/booty.sh @@ -0,0 +1,156 @@ +#!/bin/sh +set -e # Work even if somebody does "sh thisscript.sh". + +# If not --_skip-to-install: + # Bootstrap + # TODO: Inline the bootstrap scripts by putting each one into its own function (so they don't leak scope). + +PYTHON=python +SUDO=sudo + +if [ "$1" != "--_skip-to-install" ]; then + # Now we drop into python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + # + # The following Python script prints a path to a new copy + # of letsencrypt-auto or returns non-zero. + # There is no $ interpolation due to quotes on heredoc delimiters. + set +e + DOWNLOAD_OUT=`$PYTHON - <<-"UNLIKELY_EOF" + +from json import loads +from os.path import join +from subprocess import check_call, CalledProcessError +from sys import exit +from tempfile import mkdtemp +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + + +PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe +4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B +2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww +s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T +QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE +33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP +rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 ++E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK +EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu +q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 +3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn +I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== +-----END PUBLIC KEY----- +""" # TODO: Replace with real one. + + +class HumanException(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise HumanException. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise HumanException("Couldn't download %s." % url, exc) + + +class TempDir(object): + def __init__(self): + self.path = mkdtemp() + + def write(self, contents, filename): + """Write something to a named file in me.""" + with open(join(self.path, filename), 'w') as file: + file.write(contents) + + +def latest_stable_tag(get): + """Return the git tag pointing to the latest stable release of LE. + + If anything goes wrong, raise HumanException. + + """ + try: + json = get('https://pypi.python.org/pypi/letsencrypt/json') + except (HTTPError, IOError) as exc: + raise HumanException("Couldn't query PyPI for the latest version of " + "Let's Encrypt.", exc) + metadata = loads(json) + # TODO: Make sure this really returns the latest stable version, not just the + # newest version. https://wiki.python.org/moin/PyPIJSON says it should. + return 'v' + metadata['info']['version'] + + +def verified_new_le_auto(get, tag, temp): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong, + raise HumanException. + + """ + root = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' % + tag) + temp.write(get(root + 'letsencrypt-auto'), 'letsencrypt-auto') + temp.write(get(root + 'letsencrypt-auto.sig'), 'letsencrypt-auto.sig') + temp.write(PUBLIC_KEY, 'public_key.pem') + le_auto_path = join(temp.path, 'letsencrypt-auto') + try: + check_call('openssl', 'dgst', '-sha256', '-verify', + join(temp.path, 'public_key.pem'), + '-signature', + join(temp.path, 'letsencrypt-auto.sig'), + le_auth_path) + except CalledProcessError as exc: + raise HumanException("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + else: # belt & suspenders + return le_auto_path + + +def main(): + get = HttpsGetter().get + temp = TempDir() + try: + stable_tag = latest_stable_tag(get) + print verified_new_le_auto(get, stable_tag, temp) + except HumanException as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +exit(main()) +"UNLIKELY_EOF"` + DOWNLOAD_STATUS=$? + set -e + if [ "$DOWNLOAD_STATUS" = 0 ]; then + NEW_LE_AUTO="$DOWNLOAD_OUT" + $SUDO cp "$NEW_LE_AUTO" $0 + else + # Report error: + echo $DOWNLOAD_OUT + fi +else # --_skip-to-install was passed. + echo skipping! +fi + +echo $TMP_DIR From ec415b26fdd52282ee31fe4afcd6c2402bb369cc Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 12:25:06 -0500 Subject: [PATCH 002/201] Add a sig to test against. --- letsencrypt-auto.sig | Bin 0 -> 512 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 letsencrypt-auto.sig diff --git a/letsencrypt-auto.sig b/letsencrypt-auto.sig new file mode 100644 index 0000000000000000000000000000000000000000..3423689f2be156793eeffbd11ca1f2ad5601d001 GIT binary patch literal 512 zcmV+b0{{ICu2Ad9O?7UKYmt;vm>ItDKQ^Nu$3Br$v<2K71I1vR2w=I18N`dm70DdY zYSH!;GFf5<3bHCCTFSA24g-SC+J7Z>m53<1bv3xLM^&U6AgU>n$mO-_F$`Z^PX{O! z1Spi>EIup=7CSIW8Qwa}bPz?5!e0YO233X`At(@W_Hv;J=+^h zKJ!0Z6lkU^u2m@%90AEIUk(k>u>;Sy!Ny5=yd^r$N@!|McmflUe+$7S#)y zyVwQY+1^@|c#5HY(Lc2wUN0zC0ZG?TkuL9$oNOViQ>H{U6#%-8Vxrvs!3}8J?jOda z2x16tx!Yhb(&q@_Q>gahpO<`-&mjcj`BRlxkP2&fv5TG=6EDDTQgEy_c!L_Qj&>62 zu*h<@x~91Q>#)M1X0S1~##gD8`A{wFqwv^~y5Hb2gZ}g%z(HMQ=!jvg#eRCH^jkhd zRVcC=a^}Y8yhJ3x6R$muhN!`leKtOCk!oNKKMa z!2vL!*W1dQZRt_Ok5=^=>94uW?_7juohsC3rwMvKW2pmF($A!AACOYVtGHqu+d!#c CH~rlJ literal 0 HcmV?d00001 From 1a8f40e01b8867fc060a31fb9f7f76de95836431 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 12:38:03 -0500 Subject: [PATCH 003/201] This works now, to the point where it calls the downloaded version of le_auto. --- booty.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/booty.sh b/booty.sh index b9d9e6583..f5ad8b687 100755 --- a/booty.sh +++ b/booty.sh @@ -20,6 +20,7 @@ if [ "$1" != "--_skip-to-install" ]; then DOWNLOAD_OUT=`$PYTHON - <<-"UNLIKELY_EOF" from json import loads +from os import devnull from os.path import join from subprocess import check_call, CalledProcessError from sys import exit @@ -113,11 +114,14 @@ def verified_new_le_auto(get, tag, temp): temp.write(PUBLIC_KEY, 'public_key.pem') le_auto_path = join(temp.path, 'letsencrypt-auto') try: - check_call('openssl', 'dgst', '-sha256', '-verify', - join(temp.path, 'public_key.pem'), - '-signature', - join(temp.path, 'letsencrypt-auto.sig'), - le_auth_path) + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp.path, 'public_key.pem'), + '-signature', + join(temp.path, 'letsencrypt-auto.sig'), + le_auto_path], + stdout=dev_null, + stderr=dev_null) except CalledProcessError as exc: raise HumanException("Couldn't verify signature of downloaded " "letsencrypt-auto.", exc) @@ -143,14 +147,18 @@ exit(main()) DOWNLOAD_STATUS=$? set -e if [ "$DOWNLOAD_STATUS" = 0 ]; then - NEW_LE_AUTO="$DOWNLOAD_OUT" - $SUDO cp "$NEW_LE_AUTO" $0 + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Upgrading letsencrypt-auto script at $0:" $SUDO cp "$DOWNLOAD_OUT" "$0" + $SUDO cp "$DOWNLOAD_OUT" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + "$0" --_skip-to-install "$@" else + echo $0 # Report error: echo $DOWNLOAD_OUT fi else # --_skip-to-install was passed. echo skipping! fi - -echo $TMP_DIR From a75c74303ed4ee55b0382227632fd244be6a70f0 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 17:04:48 -0500 Subject: [PATCH 004/201] Compute latest stable version of letsencrypt properly. PyPI does not appear to give it to us for free through its JSON interface. distutils gives us a sufficient (though not foolproof) comparator without having to go outside the stdlib. --- booty.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/booty.sh b/booty.sh index f5ad8b687..637ad3d39 100755 --- a/booty.sh +++ b/booty.sh @@ -19,9 +19,11 @@ if [ "$1" != "--_skip-to-install" ]; then set +e DOWNLOAD_OUT=`$PYTHON - <<-"UNLIKELY_EOF" +from distutils.version import LooseVersion from json import loads from os import devnull from os.path import join +import re from subprocess import check_call, CalledProcessError from sys import exit from tempfile import mkdtemp @@ -83,21 +85,24 @@ class TempDir(object): file.write(contents) -def latest_stable_tag(get): - """Return the git tag pointing to the latest stable release of LE. +def latest_stable_version(get, package): + """Apply a fairly safe heuristic to determine the latest stable release of + a PyPI package. If anything goes wrong, raise HumanException. """ try: - json = get('https://pypi.python.org/pypi/letsencrypt/json') + json = get('https://pypi.python.org/pypi/%s/json' % package) except (HTTPError, IOError) as exc: raise HumanException("Couldn't query PyPI for the latest version of " "Let's Encrypt.", exc) metadata = loads(json) - # TODO: Make sure this really returns the latest stable version, not just the - # newest version. https://wiki.python.org/moin/PyPIJSON says it should. - return 'v' + metadata['info']['version'] + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) def verified_new_le_auto(get, tag, temp): @@ -133,7 +138,7 @@ def main(): get = HttpsGetter().get temp = TempDir() try: - stable_tag = latest_stable_tag(get) + stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') print verified_new_le_auto(get, stable_tag, temp) except HumanException as exc: print exc.args[0], exc.args[1] @@ -150,15 +155,17 @@ exit(main()) # Install new copy of letsencrypt-auto. This preserves permissions and # ownership from the old copy. # TODO: Deal with quotes in pathnames. - echo "Upgrading letsencrypt-auto script at $0:" $SUDO cp "$DOWNLOAD_OUT" "$0" + echo "Upgrading letsencrypt-auto:" + echo " " $SUDO cp "$DOWNLOAD_OUT" "$0" $SUDO cp "$DOWNLOAD_OUT" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. "$0" --_skip-to-install "$@" else - echo $0 # Report error: echo $DOWNLOAD_OUT + exit 1 fi else # --_skip-to-install was passed. + # Install Python dependencies with peep. echo skipping! fi From 602e97755fe001fa5603aa0ae8627741748fc6a9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 17:25:17 -0500 Subject: [PATCH 005/201] Stop catching exception types that are no longer thrown by get(). --- booty.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/booty.sh b/booty.sh index 637ad3d39..2d950068d 100755 --- a/booty.sh +++ b/booty.sh @@ -87,17 +87,8 @@ class TempDir(object): def latest_stable_version(get, package): """Apply a fairly safe heuristic to determine the latest stable release of - a PyPI package. - - If anything goes wrong, raise HumanException. - - """ - try: - json = get('https://pypi.python.org/pypi/%s/json' % package) - except (HTTPError, IOError) as exc: - raise HumanException("Couldn't query PyPI for the latest version of " - "Let's Encrypt.", exc) - metadata = loads(json) + a PyPI package.""" + metadata = loads(get('https://pypi.python.org/pypi/%s/json' % package)) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. return str(max(LooseVersion(r) for r From 7fb9295394893e476d4f1745a774ce8cb523d48a Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 30 Nov 2015 17:26:13 -0500 Subject: [PATCH 006/201] Find a better semantic for HumanException. These are the exceptions that are likely to happen, so we give them extra, human-readable descriptions. Also, name them more in line with stdlib exceptions. --- booty.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/booty.sh b/booty.sh index 2d950068d..fb98172fd 100755 --- a/booty.sh +++ b/booty.sh @@ -47,7 +47,7 @@ I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== """ # TODO: Replace with real one. -class HumanException(Exception): +class ExpectedError(Exception): """A novice-readable exception that also carries the original exception for debugging""" @@ -66,13 +66,13 @@ class HttpsGetter(object): def get(self, url): """Return the document contents pointed to by an HTTPS URL. - If something goes wrong (404, timeout, etc.), raise HumanException. + If something goes wrong (404, timeout, etc.), raise ExpectedError. """ try: return self._opener.open(url).read() except (HTTPError, IOError) as exc: - raise HumanException("Couldn't download %s." % url, exc) + raise ExpectedError("Couldn't download %s." % url, exc) class TempDir(object): @@ -100,7 +100,7 @@ def verified_new_le_auto(get, tag, temp): """Return the path to a verified, up-to-date letsencrypt-auto script. If the download's signature does not verify or something else goes wrong, - raise HumanException. + raise ExpectedError. """ root = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' % @@ -119,8 +119,8 @@ def verified_new_le_auto(get, tag, temp): stdout=dev_null, stderr=dev_null) except CalledProcessError as exc: - raise HumanException("Couldn't verify signature of downloaded " - "letsencrypt-auto.", exc) + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) else: # belt & suspenders return le_auto_path @@ -131,7 +131,7 @@ def main(): try: stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') print verified_new_le_auto(get, stable_tag, temp) - except HumanException as exc: + except ExpectedError as exc: print exc.args[0], exc.args[1] return 1 else: From 2c36f595b3e9d25e01ae2f3d68a74b68570d7962 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 1 Dec 2015 11:32:27 -0500 Subject: [PATCH 007/201] Return a temp dir, not the file within. This lets us reuse the dir for other things and makes it easy to rm afterward. --- booty.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/booty.sh b/booty.sh index fb98172fd..b34b60460 100755 --- a/booty.sh +++ b/booty.sh @@ -9,15 +9,16 @@ PYTHON=python SUDO=sudo if [ "$1" != "--_skip-to-install" ]; then + echo "Upgrading letsencrypt-auto..." # Now we drop into python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. # - # The following Python script prints a path to a new copy - # of letsencrypt-auto or returns non-zero. - # There is no $ interpolation due to quotes on heredoc delimiters. + # The following Python script prints a path to a temp dir containing a new + # copy of letsencrypt-auto or returns non-zero. There is no $ interpolation + # due to quotes on heredoc delimiters. set +e - DOWNLOAD_OUT=`$PYTHON - <<-"UNLIKELY_EOF" + TEMP_DIR=`$PYTHON - <<"UNLIKELY_EOF" from distutils.version import LooseVersion from json import loads @@ -130,7 +131,7 @@ def main(): temp = TempDir() try: stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') - print verified_new_le_auto(get, stable_tag, temp) + print dirname(verified_new_le_auto(get, stable_tag, temp)) except ExpectedError as exc: print exc.args[0], exc.args[1] return 1 @@ -146,14 +147,14 @@ exit(main()) # Install new copy of letsencrypt-auto. This preserves permissions and # ownership from the old copy. # TODO: Deal with quotes in pathnames. - echo "Upgrading letsencrypt-auto:" - echo " " $SUDO cp "$DOWNLOAD_OUT" "$0" - $SUDO cp "$DOWNLOAD_OUT" "$0" + # TODO: Don't bother upgrading if we're already up to date. + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. - "$0" --_skip-to-install "$@" + "$0" --_skip-to-install "$TEMP_DIR" "$@" else # Report error: - echo $DOWNLOAD_OUT + echo $TEMP_DIR exit 1 fi else # --_skip-to-install was passed. From 86203c85dfe12ac14eae584f37e8da28b43daa23 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 1 Dec 2015 11:35:51 -0500 Subject: [PATCH 008/201] Add peep and sample requirements file. We cat it to a file rather than just calling it in place because otherwise the "-" arg would have to be stripped off by editing the script. --- booty.sh | 931 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 930 insertions(+), 1 deletion(-) diff --git a/booty.sh b/booty.sh index b34b60460..6def4ff93 100755 --- a/booty.sh +++ b/booty.sh @@ -159,5 +159,934 @@ exit(main()) fi else # --_skip-to-install was passed. # Install Python dependencies with peep. - echo skipping! + TEMP_DIR="$2" + shift 2 + echo "Installing Python package dependencies..." + cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt +# sha256: Jo-gDCfedW1xZj3WH3OkqNhydWm7G0dLLOYCBVOCaHI +# sha256: mXhebPcVzc3lne4FpnbpnwSDWnHnztIByjF0AcMiupY +certifi==2015.04.28 + +# sha256: mrHTE_mbIJ-PcaYp82gzAwyNfHIoLPd1aDS69WfcpmI +click==4.0 +UNLIKELY_EOF + + cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py +#!/usr/bin/env python +"""peep ("prudently examine every package") verifies that packages conform to a +trusted, locally stored hash and only then installs them:: + + peep install -r requirements.txt + +This makes your deployments verifiably repeatable without having to maintain a +local PyPI mirror or use a vendor lib. Just update the version numbers and +hashes in requirements.txt, and you're all set. + +""" +# This is here so embedded copies of peep.py are MIT-compliant: +# Copyright (c) 2013 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +try: + xrange = xrange +except NameError: + xrange = range +from base64 import urlsafe_b64encode +import cgi +from collections import defaultdict +from functools import wraps +from hashlib import sha256 +from itertools import chain +from linecache import getline +import mimetypes +from optparse import OptionParser +from os.path import join, basename, splitext, isdir +from pickle import dumps, loads +import re +import sys +from shutil import rmtree, copy +from sys import argv, exit +from tempfile import mkdtemp +import traceback +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 +# TODO: Probably use six to make urllib stuff work across 2/3. + +from pkg_resources import require, VersionConflict, DistributionNotFound + +# We don't admit our dependency on pip in setup.py, lest a naive user simply +# say `pip install peep.tar.gz` and thus pull down an untrusted copy of pip +# from PyPI. Instead, we make sure it's installed and new enough here and spit +# out an error message if not: + + +def activate(specifier): + """Make a compatible version of pip importable. Raise a RuntimeError if we + couldn't.""" + try: + for distro in require(specifier): + distro.activate() + except (VersionConflict, DistributionNotFound): + raise RuntimeError('The installed version of pip is too old; peep ' + 'requires ' + specifier) + +# Before 0.6.2, the log module wasn't there, so some +# of our monkeypatching fails. It probably wouldn't be +# much work to support even earlier, though. +activate('pip>=0.6.2') + +import pip +from pip.commands.install import InstallCommand +try: + from pip.download import url_to_path # 1.5.6 +except ImportError: + try: + from pip.util import url_to_path # 0.7.0 + except ImportError: + from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.index import PackageFinder, Link +try: + from pip.log import logger +except ImportError: + from pip import logger # 6.0 +from pip.req import parse_requirements +try: + from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner +except ImportError: + class NullProgressBar(object): + def __init__(self, *args, **kwargs): + pass + + def iter(self, ret, *args, **kwargs): + return ret + + DownloadProgressBar = DownloadProgressSpinner = NullProgressBar + + +__version__ = 2, 4, 1 + + +ITS_FINE_ITS_FINE = 0 +SOMETHING_WENT_WRONG = 1 +# "Traditional" for command-line errors according to optparse docs: +COMMAND_LINE_ERROR = 2 + +ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') + +MARKER = object() + + +class PipException(Exception): + """When I delegated to pip, it exited with an error.""" + + def __init__(self, error_code): + self.error_code = error_code + + +class UnsupportedRequirementError(Exception): + """An unsupported line was encountered in a requirements file.""" + + +class DownloadError(Exception): + def __init__(self, link, exc): + self.link = link + self.reason = str(exc) + + def __str__(self): + return 'Downloading %s failed: %s' % (self.link, self.reason) + + +def encoded_hash(sha): + """Return a short, 7-bit-safe representation of a hash. + + If you pass a sha256, this results in the hash algorithm that the Wheel + format (PEP 427) uses, except here it's intended to be run across the + downloaded archive before unpacking. + + """ + return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') + + +def run_pip(initial_args): + """Delegate to pip the given args (starting with the subcommand), and raise + ``PipException`` if something goes wrong.""" + status_code = pip.main(initial_args) + + # Clear out the registrations in the pip "logger" singleton. Otherwise, + # loggers keep getting appended to it with every run. Pip assumes only one + # command invocation will happen per interpreter lifetime. + logger.consumers = [] + + if status_code: + raise PipException(status_code) + + +def hash_of_file(path): + """Return the hash of a downloaded file.""" + with open(path, 'rb') as archive: + sha = sha256() + while True: + data = archive.read(2 ** 20) + if not data: + break + sha.update(data) + return encoded_hash(sha) + + +def is_git_sha(text): + """Return whether this is probably a git sha""" + # Handle both the full sha as well as the 7-character abbreviation + if len(text) in (40, 7): + try: + int(text, 16) + return True + except ValueError: + pass + return False + + +def filename_from_url(url): + parsed = urlparse(url) + path = parsed.path + return path.split('/')[-1] + + +def requirement_args(argv, want_paths=False, want_other=False): + """Return an iterable of filtered arguments. + + :arg argv: Arguments, starting after the subcommand + :arg want_paths: If True, the returned iterable includes the paths to any + requirements files following a ``-r`` or ``--requirement`` option. + :arg want_other: If True, the returned iterable includes the args that are + not a requirement-file path or a ``-r`` or ``--requirement`` flag. + + """ + was_r = False + for arg in argv: + # Allow for requirements files named "-r", don't freak out if there's a + # trailing "-r", etc. + if was_r: + if want_paths: + yield arg + was_r = False + elif arg in ['-r', '--requirement']: + was_r = True + else: + if want_other: + yield arg + + +HASH_COMMENT_RE = re.compile( + r""" + \s*\#\s+ # Lines that start with a '#' + (?Psha256):\s+ # Hash type is hardcoded to be sha256 for now. + (?P[^\s]+) # Hashes can be anything except '#' or spaces. + \s* # Suck up whitespace before the comment or + # just trailing whitespace if there is no + # comment. Also strip trailing newlines. + (?:\#(?P.*))? # Comments can be anything after a whitespace+# + # and are optional. + $""", re.X) + + +def peep_hash(argv): + """Return the peep hash of one or more files, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + parser = OptionParser( + usage='usage: %prog hash file [file ...]', + description='Print a peep hash line for one or more files: for ' + 'example, "# sha256: ' + 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY".') + _, paths = parser.parse_args(args=argv) + if paths: + for path in paths: + print('# sha256:', hash_of_file(path)) + return ITS_FINE_ITS_FINE + else: + parser.print_usage() + return COMMAND_LINE_ERROR + + +class EmptyOptions(object): + """Fake optparse options for compatibility with pip<1.2 + + pip<1.2 had a bug in parse_requirements() in which the ``options`` kwarg + was required. We work around that by passing it a mock object. + + """ + default_vcs = None + skip_requirements_regex = None + isolated_mode = False + + +def memoize(func): + """Memoize a method that should return the same result every time on a + given instance. + + """ + @wraps(func) + def memoizer(self): + if not hasattr(self, '_cache'): + self._cache = {} + if func.__name__ not in self._cache: + self._cache[func.__name__] = func(self) + return self._cache[func.__name__] + return memoizer + + +def package_finder(argv): + """Return a PackageFinder respecting command-line options. + + :arg argv: Everything after the subcommand + + """ + # We instantiate an InstallCommand and then use some of its private + # machinery--its arg parser--for our own purposes, like a virus. This + # approach is portable across many pip versions, where more fine-grained + # ones are not. Ignoring options that don't exist on the parser (for + # instance, --use-wheel) gives us a straightforward method of backward + # compatibility. + try: + command = InstallCommand() + except TypeError: + # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 + # given)" error. In that version, InstallCommand takes a top=level + # parser passed in from outside. + from pip.baseparser import create_main_parser + command = InstallCommand(create_main_parser()) + # The downside is that it essentially ruins the InstallCommand class for + # further use. Calling out to pip.main() within the same interpreter, for + # example, would result in arguments parsed this time turning up there. + # Thus, we deepcopy the arg parser so we don't trash its singletons. Of + # course, deepcopy doesn't work on these objects, because they contain + # uncopyable regex patterns, so we pickle and unpickle instead. Fun! + options, _ = loads(dumps(command.parser)).parse_args(argv) + + # Carry over PackageFinder kwargs that have [about] the same names as + # options attr names: + possible_options = [ + 'find_links', 'use_wheel', 'allow_external', 'allow_unverified', + 'allow_all_external', ('allow_all_prereleases', 'pre'), + 'process_dependency_links'] + kwargs = {} + for option in possible_options: + kw, attr = option if isinstance(option, tuple) else (option, option) + value = getattr(options, attr, MARKER) + if value is not MARKER: + kwargs[kw] = value + + # Figure out index_urls: + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + index_urls = [] + index_urls += getattr(options, 'mirrors', []) + + # If pip is new enough to have a PipSession, initialize one, since + # PackageFinder requires it: + if hasattr(command, '_build_session'): + kwargs['session'] = command._build_session(options) + + return PackageFinder(index_urls=index_urls, **kwargs) + + +class DownloadedReq(object): + """A wrapper around InstallRequirement which offers additional information + based on downloading and examining a corresponding package archive + + These are conceptually immutable, so we can get away with memoizing + expensive things. + + """ + def __init__(self, req, argv, finder): + """Download a requirement, compare its hashes, and return a subclass + of DownloadedReq depending on its state. + + :arg req: The InstallRequirement I am based on + :arg argv: The args, starting after the subcommand + + """ + self._req = req + self._argv = argv + self._finder = finder + + # We use a separate temp dir for each requirement so requirements + # (from different indices) that happen to have the same archive names + # don't overwrite each other, leading to a security hole in which the + # latter is a hash mismatch, the former has already passed the + # comparison, and the latter gets installed. + self._temp_path = mkdtemp(prefix='peep-') + # Think of DownloadedReq as a one-shot state machine. It's an abstract + # class that ratchets forward to being one of its own subclasses, + # depending on its package status. Then it doesn't move again. + self.__class__ = self._class() + + def dispose(self): + """Delete temp files and dirs I've made. Render myself useless. + + Do not call further methods on me after calling dispose(). + + """ + rmtree(self._temp_path) + + def _version(self): + """Deduce the version number of the downloaded package from its filename.""" + # TODO: Can we delete this method and just print the line from the + # reqs file verbatim instead? + def version_of_archive(filename, package_name): + # Since we know the project_name, we can strip that off the left, strip + # any archive extensions off the right, and take the rest as the + # version. + for ext in ARCHIVE_EXTENSIONS: + if filename.endswith(ext): + filename = filename[:-len(ext)] + break + # Handle github sha tarball downloads. + if is_git_sha(filename): + filename = package_name + '-' + filename + if not filename.lower().replace('_', '-').startswith(package_name.lower()): + # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -? + give_up(filename, package_name) + return filename[len(package_name) + 1:] # Strip off '-' before version. + + def version_of_wheel(filename, package_name): + # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file- + # name-convention) we know the format bits are '-' separated. + whl_package_name, version, _rest = filename.split('-', 2) + # Do the alteration to package_name from PEP 427: + our_package_name = re.sub(r'[^\w\d.]+', '_', package_name, re.UNICODE) + if whl_package_name != our_package_name: + give_up(filename, whl_package_name) + return version + + def give_up(filename, package_name): + raise RuntimeError("The archive '%s' didn't start with the package name " + "'%s', so I couldn't figure out the version number. " + "My bad; improve me." % + (filename, package_name)) + + get_version = (version_of_wheel + if self._downloaded_filename().endswith('.whl') + else version_of_archive) + return get_version(self._downloaded_filename(), self._project_name()) + + def _is_always_unsatisfied(self): + """Returns whether this requirement is always unsatisfied + + This would happen in cases where we can't determine the version + from the filename. + + """ + # If this is a github sha tarball, then it is always unsatisfied + # because the url has a commit sha in it and not the version + # number. + url = self._url() + if url: + filename = filename_from_url(url) + if filename.endswith(ARCHIVE_EXTENSIONS): + filename, ext = splitext(filename) + if is_git_sha(filename): + return True + return False + + def _path_and_line(self): + """Return the path and line number of the file from which our + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + self._req.comes_from).groups()) + return path, int(line) + + @memoize # Avoid hitting the file[cache] over and over. + def _expected_hashes(self): + """Return a list of known-good hashes for this package.""" + + def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line + ``line_number``. + + """ + for line_number in xrange(line_number - 1, 0, -1): + line = getline(path, line_number) + match = HASH_COMMENT_RE.match(line) + if match: + yield match.groupdict()['hash'] + elif not line.lstrip().startswith('#'): + # If we hit a non-comment line, abort + break + + hashes = list(hashes_above(*self._path_and_line())) + hashes.reverse() # because we read them backwards + return hashes + + def _download(self, link): + """Download a file, and return its name within my temp dir. + + This does no verification of HTTPS certs, but our checking hashes + makes that largely unimportant. It would be nice to be able to use the + requests lib, which can verify certs, but it is guaranteed to be + available only in pip >= 1.5. + + This also drops support for proxies and basic auth, though those could + be added back in. + + """ + # Based on pip 1.4.1's URLOpener but with cert verification removed + def opener(is_https): + if is_https: + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + else: + opener = build_opener() + return opener + + # Descended from unpack_http_url() in pip 1.4.1 + def best_filename(link, response): + """Return the most informative possible filename for a download, + ideally with a proper extension. + + """ + content_type = response.info().get('content-type', '') + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess: + content_disposition = response.info().get('content-disposition') + if content_disposition: + type, params = cgi.parse_header(content_disposition) + # We use ``or`` here because we don't want to use an "empty" value + # from the filename param: + filename = params.get('filename') or filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != response.geturl(): + ext = splitext(response.geturl())[1] + if ext: + filename += ext + return filename + + # Descended from _download_url() in pip 1.4.1 + def pipe_to_file(response, path, size=0): + """Pull the data off an HTTP response, shove it in a new file, and + show progress. + + :arg response: A file-like object to read from + :arg path: The path of the new file + :arg size: The expected size, in bytes, of the download. 0 for + unknown or to suppress progress indication (as for cached + downloads) + + """ + def response_chunks(chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + print('Downloading %s%s...' % ( + self._req.req, + (' (%sK)' % (size / 1000)) if size > 1000 else '')) + progress_indicator = (DownloadProgressBar(max=size).iter if size + else DownloadProgressSpinner().iter) + with open(path, 'wb') as file: + for chunk in progress_indicator(response_chunks(4096), 4096): + file.write(chunk) + + url = link.url.split('#', 1)[0] + try: + response = opener(urlparse(url).scheme != 'http').open(url) + except (HTTPError, IOError) as exc: + raise DownloadError(link, exc) + filename = best_filename(link, response) + try: + size = int(response.headers['content-length']) + except (ValueError, KeyError, TypeError): + size = 0 + pipe_to_file(response, join(self._temp_path, filename), size=size) + return filename + + # Based on req_set.prepare_files() in pip bb2a8428d4aebc8d313d05d590f386fa3f0bbd0f + @memoize # Avoid re-downloading. + def _downloaded_filename(self): + """Download the package's archive if necessary, and return its + filename. + + --no-deps is implied, as we have reimplemented the bits that would + ordinarily do dependency resolution. + + """ + # Peep doesn't support requirements that don't come down as a single + # file, because it can't hash them. Thus, it doesn't support editable + # requirements, because pip itself doesn't support editable + # requirements except for "local projects or a VCS url". Nor does it + # support VCS requirements yet, because we haven't yet come up with a + # portable, deterministic way to hash them. In summary, all we support + # is == requirements and tarballs/zips/etc. + + # TODO: Stop on reqs that are editable or aren't ==. + + # If the requirement isn't already specified as a URL, get a URL + # from an index: + link = self._link() or self._finder.find_requirement(self._req, upgrade=False) + + if link: + lower_scheme = link.scheme.lower() # pip lower()s it for some reason. + if lower_scheme == 'http' or lower_scheme == 'https': + file_path = self._download(link) + return basename(file_path) + elif lower_scheme == 'file': + # The following is inspired by pip's unpack_file_url(): + link_path = url_to_path(link.url_without_fragment) + if isdir(link_path): + raise UnsupportedRequirementError( + "%s: %s is a directory. So that it can compute " + "a hash, peep supports only filesystem paths which " + "point to files" % + (self._req, link.url_without_fragment)) + else: + copy(link_path, self._temp_path) + return basename(link_path) + else: + raise UnsupportedRequirementError( + "%s: The download link, %s, would not result in a file " + "that can be hashed. Peep supports only == requirements, " + "file:// URLs pointing to files (not folders), and " + "http:// and https:// URLs pointing to tarballs, zips, " + "etc." % (self._req, link.url)) + else: + raise UnsupportedRequirementError( + "%s: couldn't determine where to download this requirement from." + % (self._req,)) + + def install(self): + """Install the package I represent, without dependencies. + + Obey typical pip-install options passed in on the command line. + + """ + other_args = list(requirement_args(self._argv, want_other=True)) + archive_path = join(self._temp_path, self._downloaded_filename()) + # -U so it installs whether pip deems the requirement "satisfied" or + # not. This is necessary for GitHub-sourced zips, which change without + # their version numbers changing. + run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path]) + + @memoize + def _actual_hash(self): + """Download the package's archive if necessary, and return its hash.""" + return hash_of_file(join(self._temp_path, self._downloaded_filename())) + + def _project_name(self): + """Return the inner Requirement's "unsafe name". + + Raise ValueError if there is no name. + + """ + name = getattr(self._req.req, 'project_name', '') + if name: + return name + raise ValueError('Requirement has no project_name.') + + def _name(self): + return self._req.name + + def _link(self): + try: + return self._req.link + except AttributeError: + # The link attribute isn't available prior to pip 6.1.0, so fall + # back to the now deprecated 'url' attribute. + return Link(self._req.url) if self._req.url else None + + def _url(self): + link = self._link() + return link.url if link else None + + @memoize # Avoid re-running expensive check_if_exists(). + def _is_satisfied(self): + self._req.check_if_exists() + return (self._req.satisfied_by and + not self._is_always_unsatisfied()) + + def _class(self): + """Return the class I should be, spanning a continuum of goodness.""" + try: + self._project_name() + except ValueError: + return MalformedReq + if self._is_satisfied(): + return SatisfiedReq + if not self._expected_hashes(): + return MissingReq + if self._actual_hash() not in self._expected_hashes(): + return MismatchedReq + return InstallableReq + + @classmethod + def foot(cls): + """Return the text to be printed once, after all of the errors from + classes of my type are printed. + + """ + return '' + + +class MalformedReq(DownloadedReq): + """A requirement whose package name could not be determined""" + + @classmethod + def head(cls): + return 'The following requirements could not be processed:\n' + + def error(self): + return '* Unable to determine package name from URL %s; add #egg=' % self._url() + + +class MissingReq(DownloadedReq): + """A requirement for which no hashes were specified in the requirements file""" + + @classmethod + def head(cls): + return ('The following packages had no hashes specified in the requirements file, which\n' + 'leaves them open to tampering. Vet these packages to your satisfaction, then\n' + 'add these "sha256" lines like so:\n\n') + + def error(self): + if self._url(): + # _url() always contains an #egg= part, or this would be a + # MalformedRequest. + line = self._url() + else: + line = '%s==%s' % (self._name(), self._version()) + return '# sha256: %s\n%s\n' % (self._actual_hash(), line) + + +class MismatchedReq(DownloadedReq): + """A requirement for which the downloaded file didn't match any of my hashes.""" + @classmethod + def head(cls): + return ("THE FOLLOWING PACKAGES DIDN'T MATCH THE HASHES SPECIFIED IN THE REQUIREMENTS\n" + "FILE. If you have updated the package versions, update the hashes. If not,\n" + "freak out, because someone has tampered with the packages.\n\n") + + def error(self): + preamble = ' %s: expected' % self._project_name() + if len(self._expected_hashes()) > 1: + preamble += ' one of' + padding = '\n' + ' ' * (len(preamble) + 1) + return '%s %s\n%s got %s' % (preamble, + padding.join(self._expected_hashes()), + ' ' * (len(preamble) - 4), + self._actual_hash()) + + @classmethod + def foot(cls): + return '\n' + + +class SatisfiedReq(DownloadedReq): + """A requirement which turned out to be already installed""" + + @classmethod + def head(cls): + return ("These packages were already installed, so we didn't need to download or build\n" + "them again. If you installed them with peep in the first place, you should be\n" + "safe. If not, uninstall them, then re-attempt your install with peep.\n") + + def error(self): + return ' %s' % (self._req,) + + +class InstallableReq(DownloadedReq): + """A requirement whose hash matched and can be safely installed""" + + +# DownloadedReq subclasses that indicate an error that should keep us from +# going forward with installation, in the order in which their errors should +# be reported: +ERROR_CLASSES = [MismatchedReq, MissingReq, MalformedReq] + + +def bucket(things, key): + """Return a map of key -> list of things.""" + ret = defaultdict(list) + for thing in things: + ret[key(thing)].append(thing) + return ret + + +def first_every_last(iterable, first, every, last): + """Execute something before the first item of iter, something else for each + item, and a third thing after the last. + + If there are no items in the iterable, don't execute anything. + + """ + did_first = False + for item in iterable: + if not did_first: + did_first = True + first(item) + every(item) + if did_first: + last(item) + + +def downloaded_reqs_from_path(path, argv): + """Return a list of DownloadedReqs representing the requirements parsed + out of a given requirements file. + + :arg path: The path to the requirements file + :arg argv: The commandline args, starting after the subcommand + + """ + finder = package_finder(argv) + + def downloaded_reqs(parsed_reqs): + """Just avoid repeating this list comp.""" + return [DownloadedReq(req, argv, finder) for req in parsed_reqs] + + try: + return downloaded_reqs(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return downloaded_reqs(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + +def peep_install(argv): + """Perform the ``peep install`` subcommand, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + output = [] + out = output.append + reqs = [] + try: + req_paths = list(requirement_args(argv, want_paths=True)) + if not req_paths: + out("You have to ``this`` specify one or more requirements files with the -r option, because\n" + "otherwise there's nowhere for peep to look up the hashes.\n") + return COMMAND_LINE_ERROR + + # We're a "peep install" command, and we have some requirement paths. + reqs = list(chain.from_iterable( + downloaded_reqs_from_path(path, argv) + for path in req_paths)) + buckets = bucket(reqs, lambda r: r.__class__) + + # Skip a line after pip's "Cleaning up..." so the important stuff + # stands out: + if any(buckets[b] for b in ERROR_CLASSES): + out('\n') + + printers = (lambda r: out(r.head()), + lambda r: out(r.error() + '\n'), + lambda r: out(r.foot())) + for c in ERROR_CLASSES: + first_every_last(buckets[c], *printers) + + if any(buckets[b] for b in ERROR_CLASSES): + out('-------------------------------\n' + 'Not proceeding to installation.\n') + return SOMETHING_WENT_WRONG + else: + for req in buckets[InstallableReq]: + req.install() + + first_every_last(buckets[SatisfiedReq], *printers) + + return ITS_FINE_ITS_FINE + except (UnsupportedRequirementError, DownloadError) as exc: + out(str(exc)) + return SOMETHING_WENT_WRONG + finally: + for req in reqs: + req.dispose() + print(''.join(output)) + + +def main(): + """Be the top-level entrypoint. Return a shell status code.""" + commands = {'hash': peep_hash, + 'install': peep_install} + try: + if len(argv) >= 2 and argv[1] in commands: + return commands[argv[1]](argv[2:]) + else: + # Fall through to top-level pip main() for everything else: + return pip.main() + except PipException as exc: + return exc.error_code + + +def exception_handler(exc_type, exc_value, exc_tb): + print('Oh no! Peep had a problem while trying to do stuff. Please write up a bug report') + print('with the specifics so we can fix it:') + print() + print('https://github.com/erikrose/peep/issues/new') + print() + print('Here are some particulars you can copy and paste into the bug report:') + print() + print('---') + print('peep:', repr(__version__)) + print('python:', repr(sys.version)) + print('pip:', repr(getattr(pip, '__version__', 'no __version__ attr'))) + print('Command line: ', repr(sys.argv)) + print( + ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))) + print('---') + + +if __name__ == '__main__': + try: + exit(main()) + except Exception: + exception_handler(*sys.exc_info()) + exit(SOMETHING_WENT_WRONG) +UNLIKELY_EOF + + set +e + PEEP_OUT=`$PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_STATUS=$? + set -e + if [ "$PEEP_STATUS" = 0 ]; then + echo "Running letsencrypt..." + else + # Report error: + echo $PEEP_OUT + exit 1 + fi fi From fe77da2f7f06a27783caef6279056666e345dd1c Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 1 Dec 2015 11:37:21 -0500 Subject: [PATCH 009/201] Unquote heredoc terminators. Quoting them causes them to not be recognized sometimes. (Perhaps it's when they're not within backticks?) --- booty.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/booty.sh b/booty.sh index 6def4ff93..453be7be8 100755 --- a/booty.sh +++ b/booty.sh @@ -18,7 +18,7 @@ if [ "$1" != "--_skip-to-install" ]; then # copy of letsencrypt-auto or returns non-zero. There is no $ interpolation # due to quotes on heredoc delimiters. set +e - TEMP_DIR=`$PYTHON - <<"UNLIKELY_EOF" + TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" from distutils.version import LooseVersion from json import loads @@ -140,7 +140,7 @@ def main(): exit(main()) -"UNLIKELY_EOF"` +UNLIKELY_EOF` DOWNLOAD_STATUS=$? set -e if [ "$DOWNLOAD_STATUS" = 0 ]; then From e3ace6f84cc1afd31d1cb5dc6b9bc217cf7e8a1d Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 00:08:24 -0500 Subject: [PATCH 010/201] Split large independent scripts off from the main body of the proof-of-concept script. Integrate the bits of the old le-auto script that are still useful. This makes the script more readable and easier to work on. We'll stitch it together with a build process. Also, stop passing the sudo command as an arg to the experimental bootstrappers. They will be inlined into the main script and can just reference $SUDO. As a result, stop recommending devs run the scripts manually, instead running le-auto --os-packages-only. This has the nice side effect of making dev documentation simpler. Name the folder "letsencrypt_auto" rather than "letsencrypt-auto" because git yield endless pain when replacing a file with a dir. Perhaps we can change it with impunity in a latter commit. --- bootstrap/_arch_common.sh | 4 +- bootstrap/_gentoo_common.sh | 6 +- bootstrap/freebsd.sh | 2 +- docs/contributing.rst | 56 +--- .../letsencrypt-auto.template | 279 ++++++++++-------- letsencrypt_auto/pieces/download_upgrade.py | 120 ++++++++ .../pieces/letsencrypt-auto-requirements.txt | 6 + booty.sh => letsencrypt_auto/pieces/peep.py | 188 ------------ 8 files changed, 289 insertions(+), 372 deletions(-) rename letsencrypt-auto => letsencrypt_auto/letsencrypt-auto.template (57%) create mode 100644 letsencrypt_auto/pieces/download_upgrade.py create mode 100644 letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt rename booty.sh => letsencrypt_auto/pieces/peep.py (83%) diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh index f66067ffb..47361c6c4 100755 --- a/bootstrap/_arch_common.sh +++ b/bootstrap/_arch_common.sh @@ -20,8 +20,8 @@ deps=" pkg-config " -missing=$(pacman -T $deps) +missing=$("$SUDO" pacman -T $deps) if [ "$missing" ]; then - pacman -S --needed $missing + "$SUDO" pacman -S --needed $missing fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh index a718db7ff..164312c86 100755 --- a/bootstrap/_gentoo_common.sh +++ b/bootstrap/_gentoo_common.sh @@ -12,12 +12,12 @@ PACKAGES="dev-vcs/git case "$PACKAGE_MANAGER" in (paludis) - cave resolve --keep-targets if-possible $PACKAGES -x + "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x ;; (pkgcore) - pmerge --noreplace $PACKAGES + "$SUDO" pmerge --noreplace $PACKAGES ;; (portage|*) - emerge --noreplace $PACKAGES + "$SUDO" emerge --noreplace $PACKAGES ;; esac diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh index 180ee21b4..d16b1a5bb 100755 --- a/bootstrap/freebsd.sh +++ b/bootstrap/freebsd.sh @@ -1,6 +1,6 @@ #!/bin/sh -xe -pkg install -Ay \ +"$SUDO" pkg install -Ay \ git \ python \ py27-virtualenv \ diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..0fd7593ff 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -359,75 +359,37 @@ Now run tests inside the Docker image: Notes on OS dependencies ======================== -OS level dependencies are managed by scripts in ``bootstrap``. Some notes -are provided here mainly for the :ref:`developers ` reference. +OS-level dependencies can be installed like so: -In general: +.. code-block:: shell + + letsencrypt-auto/letsencrypt-auto --os-packages-only + +In general... * ``sudo`` is required as a suggested way of running privileged process * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies +What follow are OS-specific notes for the :ref:`developers ` reference. + .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Ubuntu ------- - -.. code-block:: shell - - sudo ./bootstrap/ubuntu.sh - Debian ------ -.. code-block:: shell - - sudo ./bootstrap/debian.sh - For squeeze you will need to: - Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. -.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 - - -Mac OSX -------- - -.. code-block:: shell - - ./bootstrap/mac.sh - - -Fedora ------- - -.. code-block:: shell - - sudo ./bootstrap/fedora.sh - - -Centos 7 --------- - -.. code-block:: shell - - sudo ./bootstrap/centos.sh - - FreeBSD ------- -.. code-block:: shell - - sudo ./bootstrap/freebsd.sh - -Bootstrap script for FreeBSD uses ``pkg`` for package installation, -i.e. it does not use ports. +Package installation for FreeBSD uses ``pkg``, not ports. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see below), you will need a compatible shell, e.g. ``pkg install bash && diff --git a/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto.template similarity index 57% rename from letsencrypt-auto rename to letsencrypt_auto/letsencrypt-auto.template index a3009fe52..b67f87e54 100755 --- a/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -1,12 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # -# A script to run the latest release version of the Let's Encrypt in a -# virtual environment -# -# Installs and updates the letencrypt virtualenv, and runs letsencrypt -# using that virtual environment. This allows the client to function decently -# without requiring specific versions of its dependencies from the operating -# system. +# Download and run the latest release version of the Let's Encrypt client. + +set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed @@ -15,6 +11,77 @@ VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ] ; then + if [ "$2" != "" ] ; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + if command -v python2.7 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2.7} + elif command -v python27 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python27} + elif command -v python2 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2} + elif command -v python > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python} + else + echo "Cannot find any Pythons... please install one!" + fi + + PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ $PYVER -eq 26 ] ; then + ExperimentalBootstrap "Python 2.6" + elif [ $PYVER -lt 26 ] ; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work." + exit 1 + fi +} + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ] ; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchLinux + elif [ -f /etc/manjaro-release ] ; then + ExperimentalBootstrap "Manjaro Linux" BootstrapManjaro + elif [ -f /etc/gentoo-release ] ; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a peep install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + fi +} + # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support # for experimental platforms) @@ -63,131 +130,81 @@ else SUDO= fi -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$DEBUG" = 1 ] ; then - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi +if [ "$1" = "--os-packages-only" ]; then + Bootstrap +elif [ "$1" != "--_skip-to-install" ]; then + echo "Upgrading letsencrypt-auto..." + + if [ ! -f $VENV_BIN/letsencrypt ]; then + OLD_VERSION="0.0.0" # ($VENV_BIN/letsencrypt --version) + else + OLD_VERSION="0.0.0" + fi + + # TODO: Don't bother upgrading if we're already up to date. + if [ "$OLD_VERSION" != "1.2.3" ]; then + Bootstrap + echo "Creating virtual environment..." + rm -rf "$VENV_PATH" + DeterminePythonVersion + if [ "$VERBOSE" = 1 ] ; then + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + else + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null fi + NEXT: Is all this stuff in the right if branch? + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the + # option of future Windows compatibility. + # + # This Python script prints a path to a temp dir + # containing a new copy of letsencrypt-auto or returns non-zero. + # There is no $ interpolation due to quotes on heredoc delimiters. + set +e + TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" +{download_upgrade} +UNLIKELY_EOF` + DOWNLOAD_STATUS=$? + set -e + if [ "$DOWNLOAD_STATUS" = 0 ]; then + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Installing new version of letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + "$0" --_skip-to-install "$TEMP_DIR" "$@" + else + # Report error: + echo $TEMP_DIR + exit 1 + fi + fi # should upgrade +else # --_skip-to-install was passed. + # Install Python dependencies with peep, then run letsencrypt. + echo "Installing Python package dependencies..." + TEMP_DIR="$2" + shift 2 + cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt +{requirements} +UNLIKELY_EOF + + cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py +{peep} +UNLIKELY_EOF + + set +e + PEEP_OUT=`$PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_STATUS=$? + set -e + if [ "$PEEP_STATUS" = 0 ]; then + echo "Running letsencrypt..." + $SUDO $VENV_BIN/letsencrypt "$@" else - echo "WARNING: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" + # Report error: + echo $PEEP_OUT exit 1 fi -} - -DeterminePythonVersion() { - if command -v python2.7 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2.7} - elif command -v python27 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python27} - elif command -v python2 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2} - elif command -v python > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python} - else - echo "Cannot find any Pythons... please install one!" - fi - - PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ] ; then - ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ] ; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work." - exit 1 - fi -} - - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - BOOTSTRAP=`dirname $0`/bootstrap - if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 - fi - - if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh - elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh - elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh - elif [ -f /etc/arch-release ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" - elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" - elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" - elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh - elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh - else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - fi - - DeterminePythonVersion - echo "Creating virtual environment..." - if [ "$VERBOSE" = 1 ] ; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH - else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null - fi -else - DeterminePythonVersion fi - - -printf "Updating letsencrypt and virtual environment dependencies..." -if [ "$VERBOSE" = 1 ] ; then - echo - $VENV_BIN/pip install -U setuptools - $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -r py26reqs.txt -U letsencrypt letsencrypt-apache - # nginx is buggy / disabled for now, but upgrade it if the user has - # installed it manually - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - $VENV_BIN/pip install -U letsencrypt letsencrypt-nginx - fi -else - $VENV_BIN/pip install -U setuptools > /dev/null - printf . - $VENV_BIN/pip install -U pip > /dev/null - printf . - # nginx is buggy / disabled for now... - $VENV_BIN/pip install -r py26reqs.txt > /dev/null - printf . - $VENV_BIN/pip install -U letsencrypt > /dev/null - printf . - $VENV_BIN/pip install -U letsencrypt-apache > /dev/null - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - printf . - $VENV_BIN/pip install -U letsencrypt-nginx > /dev/null - fi - echo -fi - -# Explain what's about to happen, for the benefit of those getting sudo -# password prompts... -echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt_auto/pieces/download_upgrade.py b/letsencrypt_auto/pieces/download_upgrade.py new file mode 100644 index 000000000..27e1b28d8 --- /dev/null +++ b/letsencrypt_auto/pieces/download_upgrade.py @@ -0,0 +1,120 @@ +from distutils.version import LooseVersion +from json import loads +from os import devnull +from os.path import join +import re +from subprocess import check_call, CalledProcessError +from sys import exit +from tempfile import mkdtemp +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + + +PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe +4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B +2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww +s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T +QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE +33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP +rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 ++E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK +EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu +q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 +3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn +I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== +-----END PUBLIC KEY----- +""" # TODO: Replace with real one. + + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +class TempDir(object): + def __init__(self): + self.path = mkdtemp() + + def write(self, contents, filename): + """Write something to a named file in me.""" + with open(join(self.path, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get, package): + """Apply a fairly safe heuristic to determine the latest stable release of + a PyPI package.""" + metadata = loads(get('https://pypi.python.org/pypi/%s/json' % package)) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong, + raise ExpectedError. + + """ + le_auto_dir = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/' + '%s/letsencrypt-auto/' % tag) + temp.write(get(le_auto_dir + 'letsencrypt-auto'), 'letsencrypt-auto') + temp.write(get(le_auto_dir + 'letsencrypt-auto.sig'), 'letsencrypt-auto.sig') + temp.write(PUBLIC_KEY, 'public_key.pem') + le_auto_path = join(temp.path, 'letsencrypt-auto') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp.path, 'public_key.pem'), + '-signature', + join(temp.path, 'letsencrypt-auto.sig'), + le_auto_path], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + else: # belt & suspenders + return le_auto_path + + +def main(): + get = HttpsGetter().get + temp = TempDir() + try: + stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') + print dirname(verified_new_le_auto(get, stable_tag, temp)) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +exit(main()) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt new file mode 100644 index 000000000..963177490 --- /dev/null +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -0,0 +1,6 @@ +# sha256: Jo-gDCfedW1xZj3WH3OkqNhydWm7G0dLLOYCBVOCaHI +# sha256: mXhebPcVzc3lne4FpnbpnwSDWnHnztIByjF0AcMiupY +certifi==2015.04.28 + +# sha256: mrHTE_mbIJ-PcaYp82gzAwyNfHIoLPd1aDS69WfcpmI +click==4.0 diff --git a/booty.sh b/letsencrypt_auto/pieces/peep.py similarity index 83% rename from booty.sh rename to letsencrypt_auto/pieces/peep.py index 453be7be8..23e917ac6 100755 --- a/booty.sh +++ b/letsencrypt_auto/pieces/peep.py @@ -1,177 +1,3 @@ -#!/bin/sh -set -e # Work even if somebody does "sh thisscript.sh". - -# If not --_skip-to-install: - # Bootstrap - # TODO: Inline the bootstrap scripts by putting each one into its own function (so they don't leak scope). - -PYTHON=python -SUDO=sudo - -if [ "$1" != "--_skip-to-install" ]; then - echo "Upgrading letsencrypt-auto..." - # Now we drop into python so we don't have to install even more - # dependencies (curl, etc.), for better flow control, and for the option of - # future Windows compatibility. - # - # The following Python script prints a path to a temp dir containing a new - # copy of letsencrypt-auto or returns non-zero. There is no $ interpolation - # due to quotes on heredoc delimiters. - set +e - TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" - -from distutils.version import LooseVersion -from json import loads -from os import devnull -from os.path import join -import re -from subprocess import check_call, CalledProcessError -from sys import exit -from tempfile import mkdtemp -from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError - - -PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe -4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B -2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww -s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T -QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE -33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP -rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 -+E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK -EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu -q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 -3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn -I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== ------END PUBLIC KEY----- -""" # TODO: Replace with real one. - - -class ExpectedError(Exception): - """A novice-readable exception that also carries the original exception for - debugging""" - - -class HttpsGetter(object): - def __init__(self): - """Build an HTTPS opener.""" - # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in self._opener.handlers: - if isinstance(handler, HTTPHandler): - self._opener.handlers.remove(handler) - - def get(self, url): - """Return the document contents pointed to by an HTTPS URL. - - If something goes wrong (404, timeout, etc.), raise ExpectedError. - - """ - try: - return self._opener.open(url).read() - except (HTTPError, IOError) as exc: - raise ExpectedError("Couldn't download %s." % url, exc) - - -class TempDir(object): - def __init__(self): - self.path = mkdtemp() - - def write(self, contents, filename): - """Write something to a named file in me.""" - with open(join(self.path, filename), 'w') as file: - file.write(contents) - - -def latest_stable_version(get, package): - """Apply a fairly safe heuristic to determine the latest stable release of - a PyPI package.""" - metadata = loads(get('https://pypi.python.org/pypi/%s/json' % package)) - # metadata['info']['version'] actually returns the latest of any kind of - # release release, contrary to https://wiki.python.org/moin/PyPIJSON. - return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() - if re.match('^[0-9.]+$', r))) - - -def verified_new_le_auto(get, tag, temp): - """Return the path to a verified, up-to-date letsencrypt-auto script. - - If the download's signature does not verify or something else goes wrong, - raise ExpectedError. - - """ - root = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' % - tag) - temp.write(get(root + 'letsencrypt-auto'), 'letsencrypt-auto') - temp.write(get(root + 'letsencrypt-auto.sig'), 'letsencrypt-auto.sig') - temp.write(PUBLIC_KEY, 'public_key.pem') - le_auto_path = join(temp.path, 'letsencrypt-auto') - try: - with open(devnull, 'w') as dev_null: - check_call(['openssl', 'dgst', '-sha256', '-verify', - join(temp.path, 'public_key.pem'), - '-signature', - join(temp.path, 'letsencrypt-auto.sig'), - le_auto_path], - stdout=dev_null, - stderr=dev_null) - except CalledProcessError as exc: - raise ExpectedError("Couldn't verify signature of downloaded " - "letsencrypt-auto.", exc) - else: # belt & suspenders - return le_auto_path - - -def main(): - get = HttpsGetter().get - temp = TempDir() - try: - stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') - print dirname(verified_new_le_auto(get, stable_tag, temp)) - except ExpectedError as exc: - print exc.args[0], exc.args[1] - return 1 - else: - return 0 - - -exit(main()) -UNLIKELY_EOF` - DOWNLOAD_STATUS=$? - set -e - if [ "$DOWNLOAD_STATUS" = 0 ]; then - # Install new copy of letsencrypt-auto. This preserves permissions and - # ownership from the old copy. - # TODO: Deal with quotes in pathnames. - # TODO: Don't bother upgrading if we're already up to date. - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - # TODO: Clean up temp dir safely, even if it has quotes in its path. - "$0" --_skip-to-install "$TEMP_DIR" "$@" - else - # Report error: - echo $TEMP_DIR - exit 1 - fi -else # --_skip-to-install was passed. - # Install Python dependencies with peep. - TEMP_DIR="$2" - shift 2 - echo "Installing Python package dependencies..." - cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt -# sha256: Jo-gDCfedW1xZj3WH3OkqNhydWm7G0dLLOYCBVOCaHI -# sha256: mXhebPcVzc3lne4FpnbpnwSDWnHnztIByjF0AcMiupY -certifi==2015.04.28 - -# sha256: mrHTE_mbIJ-PcaYp82gzAwyNfHIoLPd1aDS69WfcpmI -click==4.0 -UNLIKELY_EOF - - cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py #!/usr/bin/env python """peep ("prudently examine every package") verifies that packages conform to a trusted, locally stored hash and only then installs them:: @@ -1076,17 +902,3 @@ if __name__ == '__main__': except Exception: exception_handler(*sys.exc_info()) exit(SOMETHING_WENT_WRONG) -UNLIKELY_EOF - - set +e - PEEP_OUT=`$PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` - PEEP_STATUS=$? - set -e - if [ "$PEEP_STATUS" = 0 ]; then - echo "Running letsencrypt..." - else - # Report error: - echo $PEEP_OUT - exit 1 - fi -fi From ec9a498622b0f1ae0319d890a5034a8f14b0a44c Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 01:47:38 -0500 Subject: [PATCH 011/201] Move OS-package bootstrappers to a private folder. They're now used only by the le-auto build process. The new public interface for OS-level bootstrapping is le-auto --os-packages-only, which dispatches by OS automatically. That obsoletes install-deps.sh as well, saving some repetition. Also, switch to mustache-style templating to avoid colliding with shell variable references. To optimize for the docker cache, we could later add a shim script that sources just deb_common.sh and calls its bootstrap function. --- Dockerfile | 4 +- Dockerfile-dev | 4 +- Vagrantfile | 2 +- bootstrap/_arch_common.sh | 27 ---------- bootstrap/_deb_common.sh | 50 ------------------- bootstrap/_gentoo_common.sh | 23 --------- bootstrap/_rpm_common.sh | 49 ------------------ bootstrap/_suse_common.sh | 14 ------ bootstrap/archlinux.sh | 1 - bootstrap/centos.sh | 1 - bootstrap/debian.sh | 1 - bootstrap/fedora.sh | 1 - bootstrap/freebsd.sh | 8 --- bootstrap/gentoo.sh | 1 - bootstrap/install-deps.sh | 46 ----------------- bootstrap/mac.sh | 18 ------- bootstrap/manjaro.sh | 1 - bootstrap/suse.sh | 1 - bootstrap/ubuntu.sh | 1 - docs/contributing.rst | 2 +- letsencrypt_auto/letsencrypt-auto.template | 18 +++++-- .../pieces/bootstrappers/arch_common.sh | 27 ++++++++++ .../pieces/bootstrappers/deb_common.sh | 50 +++++++++++++++++++ .../pieces/bootstrappers/freebsd.sh | 8 +++ .../pieces/bootstrappers/gentoo_common.sh | 23 +++++++++ letsencrypt_auto/pieces/bootstrappers/mac.sh | 19 +++++++ .../pieces/bootstrappers/rpm_common.sh | 49 ++++++++++++++++++ .../pieces/bootstrappers/suse_common.sh | 14 ++++++ 28 files changed, 209 insertions(+), 254 deletions(-) delete mode 100755 bootstrap/_arch_common.sh delete mode 100755 bootstrap/_deb_common.sh delete mode 100755 bootstrap/_gentoo_common.sh delete mode 100755 bootstrap/_rpm_common.sh delete mode 100755 bootstrap/_suse_common.sh delete mode 120000 bootstrap/archlinux.sh delete mode 120000 bootstrap/centos.sh delete mode 120000 bootstrap/debian.sh delete mode 120000 bootstrap/fedora.sh delete mode 100755 bootstrap/freebsd.sh delete mode 120000 bootstrap/gentoo.sh delete mode 100755 bootstrap/install-deps.sh delete mode 100755 bootstrap/mac.sh delete mode 120000 bootstrap/manjaro.sh delete mode 120000 bootstrap/suse.sh delete mode 120000 bootstrap/ubuntu.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/arch_common.sh create mode 100644 letsencrypt_auto/pieces/bootstrappers/deb_common.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/freebsd.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/mac.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/rpm_common.sh create mode 100755 letsencrypt_auto/pieces/bootstrappers/suse_common.sh diff --git a/Dockerfile b/Dockerfile index 02aa0f0d7..0a953ddc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 838b60e8b..23b6a0a88 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Vagrantfile b/Vagrantfile index a2759440c..4a603c2ce 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,7 +7,7 @@ VAGRANTFILE_API_VERSION = "2" # Setup instructions from docs/contributing.rst $ubuntu_setup_script = < /dev/null ; then - virtualenv="virtualenv" -fi - -if apt-cache show python-virtualenv > /dev/null ; then - virtualenv="$virtualenv python-virtualenv" -fi - -apt-get install -y --no-install-recommends \ - git \ - python \ - python-dev \ - $virtualenv \ - gcc \ - dialog \ - libaugeas0 \ - libssl-dev \ - libffi-dev \ - ca-certificates \ - -if ! command -v virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting - exit 1 -fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh deleted file mode 100755 index 164312c86..000000000 --- a/bootstrap/_gentoo_common.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -PACKAGES="dev-vcs/git - dev-lang/python:2.7 - dev-python/virtualenv - dev-util/dialog - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - -case "$PACKAGE_MANAGER" in - (paludis) - "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - "$SUDO" pmerge --noreplace $PACKAGES - ;; - (portage|*) - "$SUDO" emerge --noreplace $PACKAGES - ;; -esac diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh deleted file mode 100755 index b975da444..000000000 --- a/bootstrap/_rpm_common.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# Tested with: -# - Fedora 22, 23 (x64) -# - Centos 7 (x64: onD igitalOcean droplet) - -if type dnf 2>/dev/null -then - tool=dnf -elif type yum 2>/dev/null -then - tool=yum - -else - echo "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 -fi - -# Some distros and older versions of current distros use a "python27" -# instead of "python" naming convention. Try both conventions. -if ! $tool install -y \ - python \ - python-devel \ - python-virtualenv -then - if ! $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi -fi - -# "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) -if ! $tool install -y \ - git-core \ - gcc \ - dialog \ - augeas-libs \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates -then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 -fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh deleted file mode 100755 index 46f9d693b..000000000 --- a/bootstrap/_suse_common.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# SLE12 don't have python-virtualenv - -zypper -nq in -l git-core \ - python \ - python-devel \ - python-virtualenv \ - gcc \ - dialog \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates \ diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/archlinux.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/centos.sh b/bootstrap/centos.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/centos.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/debian.sh b/bootstrap/debian.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/debian.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/fedora.sh b/bootstrap/fedora.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/fedora.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh deleted file mode 100755 index d16b1a5bb..000000000 --- a/bootstrap/freebsd.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -xe - -"$SUDO" pkg install -Ay \ - git \ - python \ - py27-virtualenv \ - augeas \ - libffi \ diff --git a/bootstrap/gentoo.sh b/bootstrap/gentoo.sh deleted file mode 120000 index 125d6a592..000000000 --- a/bootstrap/gentoo.sh +++ /dev/null @@ -1 +0,0 @@ -_gentoo_common.sh \ No newline at end of file diff --git a/bootstrap/install-deps.sh b/bootstrap/install-deps.sh deleted file mode 100755 index e907e7035..000000000 --- a/bootstrap/install-deps.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -e -# -# Install OS dependencies. In the glorious future, letsencrypt-auto will -# source this... - -if test "`id -u`" -ne "0" ; then - SUDO=sudo -else - SUDO= -fi - -BOOTSTRAP=`dirname $0` -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/arch-release ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif [ -f /etc/gentoo-release ] ; then - echo "Bootstrapping dependencies for Gentoo-based OSes..." - $SUDO $BOOTSTRAP/_gentoo_common.sh -elif uname | grep -iq FreeBSD ; then - echo "Bootstrapping dependencies for FreeBSD..." - $SUDO $BOOTSTRAP/freebsd.sh -elif `grep -qs openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE.." - $SUDO $BOOTSTRAP/suse.sh -elif uname | grep -iq Darwin ; then - echo "Bootstrapping dependencies for Mac OS X..." - echo "WARNING: Mac support is very experimental at present..." - $BOOTSTRAP/mac.sh -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - exit 1 -fi diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh deleted file mode 100755 index 4d1fb8208..000000000 --- a/bootstrap/mac.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e -if ! hash brew 2>/dev/null; then - echo "Homebrew Not Installed\nDownloading..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -fi - -brew install augeas -brew install dialog - -if ! hash pip 2>/dev/null; then - echo "pip Not Installed\nInstalling python from Homebrew..." - brew install python -fi - -if ! hash virtualenv 2>/dev/null; then - echo "virtualenv Not Installed\nInstalling with pip" - pip install virtualenv -fi diff --git a/bootstrap/manjaro.sh b/bootstrap/manjaro.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/manjaro.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/suse.sh b/bootstrap/suse.sh deleted file mode 120000 index fc4c1dee4..000000000 --- a/bootstrap/suse.sh +++ /dev/null @@ -1 +0,0 @@ -_suse_common.sh \ No newline at end of file diff --git a/bootstrap/ubuntu.sh b/bootstrap/ubuntu.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/ubuntu.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/docs/contributing.rst b/docs/contributing.rst index 0fd7593ff..04303a7be 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -22,7 +22,7 @@ once: git clone https://github.com/letsencrypt/letsencrypt cd letsencrypt - ./bootstrap/install-deps.sh + ./letsencrypt-auto/letsencrypt-auto --os-packages-only ./bootstrap/dev/venv.sh Then in each shell where you're working on the client, do: diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 99f0f25d8..f3fa542e2 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -49,6 +49,14 @@ DeterminePythonVersion() { fi } +{{bootstrap_deb_common}} +{{bootstrap_rpm_common}} +{{bootstrap_suse_common}} +{{bootstrap_arch_common}} +{{bootstrap_gentoo_common}} +{{bootstrap_free_bsd}} +{{bootstrap_mac}} + # Install required OS packages: Bootstrap() { if [ -f /etc/debian_version ] ; then @@ -63,7 +71,7 @@ Bootstrap() { elif [ -f /etc/arch-release ] ; then if [ "$DEBUG" = 1 ] ; then echo "Bootstrapping dependencies for Archlinux..." - BootstrapArchLinux + BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" echo "# pacman -S letsencrypt letsencrypt-apache" @@ -73,7 +81,7 @@ Bootstrap() { exit 1 fi elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" BootstrapManjaro + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon elif [ -f /etc/gentoo-release ] ; then ExperimentalBootstrap "Gentoo" BootstrapGentooCommon elif uname | grep -iq FreeBSD ; then @@ -172,7 +180,7 @@ elif [ "$1" != "--_skip-to-install" ]; then # There is no $ interpolation due to quotes on heredoc delimiters. set +e TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" -{download_upgrade} +{{download_upgrade}} UNLIKELY_EOF` DOWNLOAD_STATUS=$? set -e @@ -197,11 +205,11 @@ else # --_skip-to-install was passed. TEMP_DIR="$2" shift 2 cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt -{requirements} +{{requirements}} UNLIKELY_EOF cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py -{peep} +{{peep}} UNLIKELY_EOF set +e diff --git a/letsencrypt_auto/pieces/bootstrappers/arch_common.sh b/letsencrypt_auto/pieces/bootstrappers/arch_common.sh new file mode 100755 index 000000000..ef881448e --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/arch_common.sh @@ -0,0 +1,27 @@ +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./bootstrap/dev/_common_venv.sh + + deps=" + git + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + missing=$("$SUDO" pacman -T $deps) + + if [ "$missing" ]; then + "$SUDO" pacman -S --needed $missing + fi +} \ No newline at end of file diff --git a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh new file mode 100644 index 000000000..b699f3fea --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh @@ -0,0 +1,50 @@ +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null ; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null ; then + virtualenv="$virtualenv python-virtualenv" + fi + + $SUDO apt-get install -y --no-install-recommends \ + git \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + libaugeas0 \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} diff --git a/letsencrypt_auto/pieces/bootstrappers/freebsd.sh b/letsencrypt_auto/pieces/bootstrappers/freebsd.sh new file mode 100755 index 000000000..8c65c97c0 --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/freebsd.sh @@ -0,0 +1,8 @@ +BootstrapFreeBsd() { + "$SUDO" pkg install -Ay \ + git \ + python \ + py27-virtualenv \ + augeas \ + libffi \ +} \ No newline at end of file diff --git a/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh b/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh new file mode 100755 index 000000000..557ae2d5c --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh @@ -0,0 +1,23 @@ +BootstrapGentooCommon() { + PACKAGES="dev-vcs/git + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + "$SUDO" pmerge --noreplace $PACKAGES + ;; + (portage|*) + "$SUDO" emerge --noreplace $PACKAGES + ;; + esac +} \ No newline at end of file diff --git a/letsencrypt_auto/pieces/bootstrappers/mac.sh b/letsencrypt_auto/pieces/bootstrappers/mac.sh new file mode 100755 index 000000000..16bcfe1cf --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/mac.sh @@ -0,0 +1,19 @@ +BootstrapMac() { + if ! hash brew 2>/dev/null; then + echo "Homebrew Not Installed\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + brew install augeas + brew install dialog + + if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv + fi +} \ No newline at end of file diff --git a/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh b/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh new file mode 100755 index 000000000..459bd1408 --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh @@ -0,0 +1,49 @@ +BootstrapRpmCommon() { + # Tested with: + # - Fedora 22, 23 (x64) + # - Centos 7 (x64: onD igitalOcean droplet) + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if ! $SUDO $tool install -y \ + python \ + python-devel \ + python-virtualenv + then + if ! $SUDO $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + fi + + # "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) + if ! $SUDO $tool install -y \ + git-core \ + gcc \ + dialog \ + augeas-libs \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} \ No newline at end of file diff --git a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh new file mode 100755 index 000000000..dff40dcf1 --- /dev/null +++ b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh @@ -0,0 +1,14 @@ +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + $SUDO zypper -nq in -l git-core \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates \ +} \ No newline at end of file From cdd855c745e44ba9498e3e98c764bcf198364266 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 02:01:22 -0500 Subject: [PATCH 012/201] Add build script for letsencrypt-auto. Change template language to reference files, saving me some boilerplate over the dict-based .format() thing I originally had in mind. Put newlines at the ends of bootstrap scripts. It makes the built le-auto script prettier. --- letsencrypt_auto/build.py | 30 +++++++++++++++++++ letsencrypt_auto/letsencrypt-auto.template | 20 ++++++------- .../pieces/bootstrappers/arch_common.sh | 2 +- .../bootstrappers/{freebsd.sh => free_bsd.sh} | 2 +- .../pieces/bootstrappers/gentoo_common.sh | 2 +- letsencrypt_auto/pieces/bootstrappers/mac.sh | 2 +- .../pieces/bootstrappers/rpm_common.sh | 2 +- .../pieces/bootstrappers/suse_common.sh | 2 +- 8 files changed, 46 insertions(+), 16 deletions(-) create mode 100755 letsencrypt_auto/build.py rename letsencrypt_auto/pieces/bootstrappers/{freebsd.sh => free_bsd.sh} (98%) diff --git a/letsencrypt_auto/build.py b/letsencrypt_auto/build.py new file mode 100755 index 000000000..b7dd40890 --- /dev/null +++ b/letsencrypt_auto/build.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +"""Stitch together the letsencrypt-auto script. + +Implement a simple templating language in which {{ some/file }} turns into the +contents of the file at ./pieces/some/file. + +""" +from os.path import dirname, join +import re +from sys import argv + + +def main(): + dir = dirname(argv[0]) + + def replacer(match): + rel_path = match.group(1) + with open(join(dir, 'pieces', rel_path)) as replacement: + return replacement.read() + + with open(join(dir, 'letsencrypt-auto.template')) as template: + result = re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', + replacer, + template.read()) + with open(join(dir, 'letsencrypt-auto'), 'w') as out: + out.write(result) + + +if __name__ == '__main__': + main() diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index f3fa542e2..b20187506 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -49,13 +49,13 @@ DeterminePythonVersion() { fi } -{{bootstrap_deb_common}} -{{bootstrap_rpm_common}} -{{bootstrap_suse_common}} -{{bootstrap_arch_common}} -{{bootstrap_gentoo_common}} -{{bootstrap_free_bsd}} -{{bootstrap_mac}} +{{ bootstrappers/deb_common.sh }} +{{ bootstrappers/rpm_common.sh }} +{{ bootstrappers/suse_common.sh }} +{{ bootstrappers/arch_common.sh }} +{{ bootstrappers/gentoo_common.sh }} +{{ bootstrappers/free_bsd.sh }} +{{ bootstrappers/mac.sh }} # Install required OS packages: Bootstrap() { @@ -180,7 +180,7 @@ elif [ "$1" != "--_skip-to-install" ]; then # There is no $ interpolation due to quotes on heredoc delimiters. set +e TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" -{{download_upgrade}} +{{ download_upgrade.py }} UNLIKELY_EOF` DOWNLOAD_STATUS=$? set -e @@ -205,11 +205,11 @@ else # --_skip-to-install was passed. TEMP_DIR="$2" shift 2 cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt -{{requirements}} +{{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py -{{peep}} +{{ peep.py }} UNLIKELY_EOF set +e diff --git a/letsencrypt_auto/pieces/bootstrappers/arch_common.sh b/letsencrypt_auto/pieces/bootstrappers/arch_common.sh index ef881448e..a6114787e 100755 --- a/letsencrypt_auto/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/arch_common.sh @@ -24,4 +24,4 @@ BootstrapArchCommon() { if [ "$missing" ]; then "$SUDO" pacman -S --needed $missing fi -} \ No newline at end of file +} diff --git a/letsencrypt_auto/pieces/bootstrappers/freebsd.sh b/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh similarity index 98% rename from letsencrypt_auto/pieces/bootstrappers/freebsd.sh rename to letsencrypt_auto/pieces/bootstrappers/free_bsd.sh index 8c65c97c0..371ca03f3 100755 --- a/letsencrypt_auto/pieces/bootstrappers/freebsd.sh +++ b/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh @@ -5,4 +5,4 @@ BootstrapFreeBsd() { py27-virtualenv \ augeas \ libffi \ -} \ No newline at end of file +} diff --git a/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh b/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh index 557ae2d5c..1d4753633 100755 --- a/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh @@ -20,4 +20,4 @@ BootstrapGentooCommon() { "$SUDO" emerge --noreplace $PACKAGES ;; esac -} \ No newline at end of file +} diff --git a/letsencrypt_auto/pieces/bootstrappers/mac.sh b/letsencrypt_auto/pieces/bootstrappers/mac.sh index 16bcfe1cf..9318d18c8 100755 --- a/letsencrypt_auto/pieces/bootstrappers/mac.sh +++ b/letsencrypt_auto/pieces/bootstrappers/mac.sh @@ -16,4 +16,4 @@ BootstrapMac() { echo "virtualenv Not Installed\nInstalling with pip" pip install virtualenv fi -} \ No newline at end of file +} diff --git a/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh b/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh index 459bd1408..ebc64413c 100755 --- a/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh @@ -46,4 +46,4 @@ BootstrapRpmCommon() { echo "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi -} \ No newline at end of file +} diff --git a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh index dff40dcf1..34511bf22 100755 --- a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh @@ -11,4 +11,4 @@ BootstrapSuseCommon() { libopenssl-devel \ libffi-devel \ ca-certificates \ -} \ No newline at end of file +} From 66436c525591750ec8e58e6c4031042bea70ed74 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 02:23:38 -0500 Subject: [PATCH 013/201] le-auto now doesn't trigger sh syntax errors when run. --- letsencrypt_auto/pieces/bootstrappers/free_bsd.sh | 2 +- letsencrypt_auto/pieces/bootstrappers/suse_common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh b/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh index 371ca03f3..641a6483f 100755 --- a/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh +++ b/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh @@ -4,5 +4,5 @@ BootstrapFreeBsd() { python \ py27-virtualenv \ augeas \ - libffi \ + libffi } diff --git a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh index 34511bf22..8e3583fd9 100755 --- a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/suse_common.sh @@ -10,5 +10,5 @@ BootstrapSuseCommon() { augeas-lenses \ libopenssl-devel \ libffi-devel \ - ca-certificates \ + ca-certificates } From f9d1de6179b76f8aab7f478b0c0b71e80e1cb8c6 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 11:40:30 -0500 Subject: [PATCH 014/201] Remove test signature, which I shouldn't have committed. --- letsencrypt-auto.sig | Bin 512 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 letsencrypt-auto.sig diff --git a/letsencrypt-auto.sig b/letsencrypt-auto.sig deleted file mode 100644 index 3423689f2be156793eeffbd11ca1f2ad5601d001..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 512 zcmV+b0{{ICu2Ad9O?7UKYmt;vm>ItDKQ^Nu$3Br$v<2K71I1vR2w=I18N`dm70DdY zYSH!;GFf5<3bHCCTFSA24g-SC+J7Z>m53<1bv3xLM^&U6AgU>n$mO-_F$`Z^PX{O! z1Spi>EIup=7CSIW8Qwa}bPz?5!e0YO233X`At(@W_Hv;J=+^h zKJ!0Z6lkU^u2m@%90AEIUk(k>u>;Sy!Ny5=yd^r$N@!|McmflUe+$7S#)y zyVwQY+1^@|c#5HY(Lc2wUN0zC0ZG?TkuL9$oNOViQ>H{U6#%-8Vxrvs!3}8J?jOda z2x16tx!Yhb(&q@_Q>gahpO<`-&mjcj`BRlxkP2&fv5TG=6EDDTQgEy_c!L_Qj&>62 zu*h<@x~91Q>#)M1X0S1~##gD8`A{wFqwv^~y5Hb2gZ}g%z(HMQ=!jvg#eRCH^jkhd zRVcC=a^}Y8yhJ3x6R$muhN!`leKtOCk!oNKKMa z!2vL!*W1dQZRt_Ok5=^=>94uW?_7juohsC3rwMvKW2pmF($A!AACOYVtGHqu+d!#c CH~rlJ From 9d6cbea5cea40e536a1d649bff3b5e18c50c5d5b Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 11:41:00 -0500 Subject: [PATCH 015/201] Fix some errors. Use the correct Python interpreter. Fix a syntax error. Fix a missing import. --- letsencrypt_auto/letsencrypt-auto.template | 5 ++--- letsencrypt_auto/pieces/download_upgrade.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index b20187506..0c153c2f2 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -169,7 +169,6 @@ elif [ "$1" != "--_skip-to-install" ]; then else virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null fi - NEXT: Is all this stuff in the right if branch? # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the @@ -179,7 +178,7 @@ elif [ "$1" != "--_skip-to-install" ]; then # containing a new copy of letsencrypt-auto or returns non-zero. # There is no $ interpolation due to quotes on heredoc delimiters. set +e - TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF" + TEMP_DIR=`$LE_PYTHON - << "UNLIKELY_EOF" {{ download_upgrade.py }} UNLIKELY_EOF` DOWNLOAD_STATUS=$? @@ -213,7 +212,7 @@ UNLIKELY_EOF UNLIKELY_EOF set +e - PEEP_OUT=`$PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_OUT=`$LE_PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` PEEP_STATUS=$? set -e if [ "$PEEP_STATUS" = 0 ]; then diff --git a/letsencrypt_auto/pieces/download_upgrade.py b/letsencrypt_auto/pieces/download_upgrade.py index 27e1b28d8..a117e9e0a 100644 --- a/letsencrypt_auto/pieces/download_upgrade.py +++ b/letsencrypt_auto/pieces/download_upgrade.py @@ -1,7 +1,7 @@ from distutils.version import LooseVersion from json import loads from os import devnull -from os.path import join +from os.path import dirname, join import re from subprocess import check_call, CalledProcessError from sys import exit From 3f0bcb5c9a568665fb8fc0b9d1c675a10664d422 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 14:42:58 -0500 Subject: [PATCH 016/201] Add real requirements, suitable as of ab9051ff09ef69a6cdf272deaa6e7df8b2f4d8a5 on master. --- .../pieces/letsencrypt-auto-requirements.txt | 190 +++++++++++++++++- 1 file changed, 185 insertions(+), 5 deletions(-) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 963177490..74e6f343f 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -1,6 +1,186 @@ -# sha256: Jo-gDCfedW1xZj3WH3OkqNhydWm7G0dLLOYCBVOCaHI -# sha256: mXhebPcVzc3lne4FpnbpnwSDWnHnztIByjF0AcMiupY -certifi==2015.04.28 +# This is the flattened list of requirements for the packages letsencrypt-auto +# installs. -# sha256: mrHTE_mbIJ-PcaYp82gzAwyNfHIoLPd1aDS69WfcpmI -click==4.0 +# sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 +# sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo +# sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 +# sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY +# sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc +# sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U +# sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis +# sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU +# sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M +# sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA +# sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs +# sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA +cffi==1.3.1 + +# sha256: 0ayQAF3qd2CBys5QjLnHMi4EONHA82AN8auXEZEBJME +https://github.com/kuba/ConfigArgParse/archive/a58b35d75a10e8b8fbee7f3c69163b63bb506325.zip#egg=ConfigArgParse + +# sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI +configobj==5.0.6 + +# sha256: rvaUNVR6WdmmY0V7hb2CtQXOOC24gvhAeWskGV6QjUM +# sha256: F-Cyopbq5TqKIyBxLA2fXjJkUjnlCuLAxwiToL88ZnI +# sha256: agSFbNkcDV2-0d-J8qsKBo4kGMbChiqXjTOaPpaXEsM +# sha256: Gvlc-QCXZIRSkVyi1i3echM_hL5HRbjGF7iE64Ozmho +# sha256: AL5UcOcPhyZBjf18qhTaS0oNH-eAk0UEzyqkfEkLbrQ +# sha256: jg2Kw83Grq8C5ZK_lDNJQ3OHzpNlEve64O4rPv6guYk +# sha256: 2WqBQR437XCoZdk2lyq5guhPVklKDUmC8sGLCA7E16c +# sha256: PWl5CSvtvm3ZEMr1jnbiMTEWAKDcCoKYprdaUIHC7-w +# sha256: e5vcpKkOmI7IYbvzsXB5LstBnzVUq1PWy5v0Tn6ojfc +# sha256: 1vc40Cac149QSCQ9qPFNN9YIR5gZ6tb_tZ5rZMdSZyI +# sha256: 7Tuo9Y-M71rHKXB4W61mQ4OyQ7yhraOhAB8mceCEv2A +# sha256: LC2rxmUOLqj82E-FqbbTcCZMWiO2tL1aAHZv7ASzMaQ +# sha256: gMNj9i7awTcvu1HUqNRR8-BmuvEGRdjBmdCnFlAxPtA +# sha256: B88JF-T8_-mH_DZxErK9gYy6l1YmOfsTv_moOgOKClQ +# sha256: VYSnU5WgJE790QPYe5jnq68Q8C4h_V21yOf52N-g2bE +# sha256: tz2PKTF-1lK0s2Dvw-IPLM5X5EYTk_STfMIx0IOtLyw +# sha256: J4eqAyfnD8bQbQsDvgp97hkUMLE9j1hHoCQMbmjMpeE +# sha256: WiBGvL2G_GOQ2vlSJ1rLSITuPQ2lTH-Baq2la745U8U +# sha256: lkC04TkXiWgAUtJZFcrDsPdNC8l8tj_26atSc9IwFmA +# sha256: vjY4IvCRJqUvHyI81AesY9bcPjXH4oPeipLqJiXN_64 +# sha256: KRKSOvdFX7LSQ5oDckJQfBLK7OfdZlnWL6gqYe2yuuA +cryptography==1.1.1 + +# sha256: nUqSIOTrq9f_YNhT5pw92J3rrV3euaxedor4EezncI4 +# sha256: TTNFnDMlThutz9tHo0CoAc2ARm5G--NdJdMIvxSNrXA +enum34==1.1.1 + +# sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 +# sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM +funcsigs==0.4 + +# sha256: my_FC9PEujBrllG2lBHvIgJtTYM1uTr8IhTO8SRs5wc +# sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs +idna==2.0 + +# sha256: WJWfvsOuQ49axxC7YcQrYQ2Ju05bzDJHswd-I0hzTa0 +# sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY +ipaddress==1.0.15 + +# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs +# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY +mock==1.3.0 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs +# sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs +parsedatetime==1.5 + +# sha256: Rsjbda51oFa9HMB_ohc0_i5gPRGgeDPswe63TDXHLgw +# sha256: 4hJ2JqkebIhduJZol22zECDwry2nKJJLVkgPx8zwlkk +pbr==1.8.1 + +# sha256: WE8LKfzF1SO0M8uJGLL8dNZ-MO4LRKlbrwMVKPQkYZ8 +# sha256: KMoLbp2Zqo3Chuh0ekRxNitpgSolKR3im2qNcKFUWg0 +# sha256: FnrV__UqZyxN3BwaCyUUbWgT67CKmqsKOsRfiltmnDs +# sha256: 5t6mFzqYhye7Ij00lzSa1c3vXAsoLv8tg-X5BlxT-F8 +# sha256: KvXgpKrWYEmVXQc0qk49yMqhep6vi0waJ6Xx7m5A9vw +# sha256: 2YhNwNwuVeJEjklXeNyYmcHIvzeusvQ0wb6nSvk8JoM +# sha256: 4nwv5t_Mhzi-PSxaAi94XrcpcQV-Gp4eNPunO86KcaY +# sha256: Za_W_syPOu0J7kvmNYO8jrRy8GzqpP4kxNHVoaPA4T8 +# sha256: uhxVj7_N-UUVwjlLEVXB3FbivCqcF9MDSYJ8ntimfkY +# sha256: upXqACLctk028MEzXAYF-uNb3z4P6o2S9dD2RWo15Vs +# sha256: QhtlkdFrUJqqjYwVgh1mu5TLSo3EOFytXFG4XUoJbYU +# sha256: MmswXL22-U2vv-LCaxHaiLCrB7igf4GIq511_wxuhBo +# sha256: mu3lsrb-RrN0jqjlIURDiQ0WNAJ77z0zt9rRZVaDAng +# sha256: c77R24lNGqnDx-YR0wLN6reuig3A7q92cnh42xrFzYc +# sha256: k1td1tVYr1EvQlAafAj0HXr_E5rxuzlZ2qOruFkjTWw +# sha256: TKARHPFX3MDy9poyPFtUeHGNaNRfyUNdhL4OwPGGIVs +# sha256: tvE8lTmKP88CJsTc-kSFYLpYZSWc2W7CgQZYZR6TIYk +# sha256: 7mvjDRY1u96kxDJdUH3IoNu95-HBmL1i3bn0MZi54hQ +# sha256: 36eGhYwmjX-74bYXXgAewCc418-uCnzne_m2Ua9nZyk +# sha256: qnf53nKvnBbMKIzUokz1iCQ4j1fXqB5ADEYWRXYphw4 +# sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus +psutil==3.3.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + +# sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M +pycparser==2.14 + +# sha256: iORea7Jd_tJyoe8ucoRh1EtjTCzWiemJtuVqNJxaOuU +# sha256: 8KJgcNbbCIHei8x4RpNLfDyTDY-cedRYg-5ImEvA1nI +pyOpenSSL==0.15.1 + +# sha256: 7qMYNcVuIJavQ2OldFp4SHimHQQ-JH06bWoKMql0H1Y +# sha256: jfvGxFi42rocDzYgqMeACLMjomiye3NZ6SpK5BMl9TU +pyRFC3339==1.0 + +# sha256: Z9WdZs26jWJOA4m4eyqDoXbyHxaodVO1D1cDsj8pusI +python-augeas==0.5.0 + +# sha256: BOk_JJlcQ92Q8zjV2GXKcs4_taU1jU2qSWVXHbNfw-w +# sha256: Pm9ZP-rZj4pSa8PjBpM1MyNuM3KfVS9SiW6lBPVTE_o +python2-pythondialog==3.3.0 + +# sha256: Or5qbT_C-75MYBRCEfRdou2-MYKm9lEa9ru6BZix-ZI +# sha256: k575weEiTZgEBWial__PeCjFbRUXsx1zRkNWwfK3dp4 +# sha256: 6tSu-nAHJJ4F5RsBCVcZ1ajdlXYAifVzCqxWmLGTKRg +# sha256: PMoN8IvQ7ZhDI5BJTOPe0AP15mGqRgvnpzS__jWYNgU +# sha256: Pt5HDT0XujwHY436DRBFK8G25a0yYSemW6d-aq6xG-w +# sha256: aMR5ZPcYbuwwaxNilidyK5B5zURH7Z5eyuzU6shMpzQ +# sha256: 3V05kZUKrkCmyB3hV4lC5z1imAjO_FHRLNFXmA5s_Bg +# sha256: p3xSBiwH63x7MFRdvHPjKZW34Rfup1Axe1y1x6RhjxQ +# sha256: ga-a7EvJYKmgEnxIjxh3La5GNGiSM_BvZUQ-exHr61E +# sha256: 4Hmx2txcBiRswbtv4bI6ULHRFz8u3VEE79QLtzoo9AY +# sha256: -9JnRncsJMuTyLl8va1cueRshrvbG52KdD7gDi-x_F0 +# sha256: mSZu8wo35Dky3uwrfKc-g8jbw7n_cD7HPsprHa5r7-o +# sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM +pytz==2015.7 + +# sha256: ifGx8l3Ne2j1FOjTQaWy60ZvlgrnVoIuqrSAo8GoHCg +# sha256: hP6NW_Tc3MSQAkRsR6FG0XrBD6zwDZCGZZBkrEO2wls +requests==2.8.1 + +# sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE +# sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo +six==1.10.0 + +# sha256: tqk-kJsx4-FGWVhP9zKYZCdZBd3BuGU4Yb3-HllyUPQ +# sha256: 60-YmUtAqOLtziiegRyaOIgK5T65_28DHQ4kOmmw_L8 +Werkzeug==0.11.2 + +# sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo +zope.component==4.2.2 + +# sha256: 3HpZov2Rcw03kxMaXSYbKek-xOKpfxvEh86N7-4v54Y +zope.event==4.1.0 + +# sha256: 8HtjH3pgHNjL0zMtVPQxQscIioMpn4WTVvCNHU1CWbM +# sha256: 3lzKCDuUOdgAL7drvmtJmMWlpyH6sluEKYln8ALfTJQ +# sha256: Z4hBb36n9bipe-lIJTd6ol6L3HNGPge6r5hYsp5zcHc +# sha256: bzIw9yVFGCAeWjcIy7LemMhIME8G497Yv7OeWCXLouE +# sha256: X6V1pSQPBCAMMIhCfQ1Le3N_bpAYgYpR2ND5J6aiUXo +# sha256: UiGUrWpUVzXt11yKg_SNZdGvBk5DKn0yDWT1a6_BLpk +# sha256: 6Mey1AlD9xyZFIyX9myqf1E0FH9XQj-NtbSCUJnOmgk +# sha256: J5Ak8CCGAcPKqQfFOHbjetiGJffq8cs4QtvjYLIocBc +# sha256: LiIanux8zFiImieOoT3P7V75OdgLB4Gamos8scaBSE8 +# sha256: aRGJZUEOyG1E3GuQF-4929WC4MCr7vYrOhnb9sitEys +# sha256: 0E34aG7IZNDK3ozxmff4OuzUFhCaIINNVo-DEN7RLeo +# sha256: 51qUfhXul-fnHgLqMC_rL8YtOiu0Zov5377UOlBqx-c +# sha256: TkXSL7iDIipaufKCoRb-xe4ujRpWjM_2otdbvQ62vPw +# sha256: vOkzm7PHpV4IA7Y9IcWDno5Hm8hcSt9CrkFbcvlPrLI +# sha256: koE4NlJFoOiGmlmZ-8wqRUdaCm7VKklNYNvcVAM1_t0 +# sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro +# sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I +zope.interface==4.1.3 From a1b26262a2ad47c4a5345f46d438c56df7f3f540 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 14:53:33 -0500 Subject: [PATCH 017/201] Print the final letsencrypt invocation before doing it. People like to know what they're sudo-ing. --- letsencrypt_auto/letsencrypt-auto.template | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 0c153c2f2..a0e073f4a 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -217,6 +217,7 @@ UNLIKELY_EOF set -e if [ "$PEEP_STATUS" = 0 ]; then echo "Running letsencrypt..." + echo " " $SUDO $VENV_BIN/letsencrypt "$@" $SUDO $VENV_BIN/letsencrypt "$@" else # Report error: From 346ec588b449c9c95f75b514ad54c2bfe7f59feb Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 15:00:04 -0500 Subject: [PATCH 018/201] Add visual separators between language changes. On his first time auditing, pde thought this would help. --- letsencrypt_auto/letsencrypt-auto.template | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index a0e073f4a..ecb86c020 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -178,9 +178,11 @@ elif [ "$1" != "--_skip-to-install" ]; then # containing a new copy of letsencrypt-auto or returns non-zero. # There is no $ interpolation due to quotes on heredoc delimiters. set +e + # ------------------------------------------------------------------------- TEMP_DIR=`$LE_PYTHON - << "UNLIKELY_EOF" {{ download_upgrade.py }} UNLIKELY_EOF` + # ------------------------------------------------------------------------- DOWNLOAD_STATUS=$? set -e if [ "$DOWNLOAD_STATUS" = 0 ]; then @@ -203,14 +205,15 @@ else # --_skip-to-install was passed. echo "Installing Python package dependencies..." TEMP_DIR="$2" shift 2 + # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF - + # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py {{ peep.py }} UNLIKELY_EOF - + # --------------------------------------------------------------------------- set +e PEEP_OUT=`$LE_PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` PEEP_STATUS=$? From 4a69584a845ac21a59c72a4598244b58fc0f6921 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 16:08:28 -0500 Subject: [PATCH 019/201] Standardize semicolon use. --- letsencrypt_auto/letsencrypt-auto.template | 24 +++++++++---------- .../pieces/letsencrypt-auto-requirements.txt | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index ecb86c020..595ec9eb9 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -13,8 +13,8 @@ VENV_BIN=${VENV_PATH}/bin ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ] ; then - if [ "$2" != "" ] ; then + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then echo "Bootstrapping dependencies via $1..." $2 fi @@ -40,9 +40,9 @@ DeterminePythonVersion() { fi PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ] ; then + if [ $PYVER -eq 26 ]; then ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ] ; then + elif [ $PYVER -lt 26 ]; then echo "You have an ancient version of Python entombed in your operating system..." echo "This isn't going to work; you'll need at least version 2.6." exit 1 @@ -59,17 +59,17 @@ DeterminePythonVersion() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ] ; then + if [ -f /etc/debian_version ]; then echo "Bootstrapping dependencies for Debian-based OSes..." BootstrapDebCommon - elif [ -f /etc/redhat-release ] ; then + elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon elif `grep -q openSUSE /etc/os-release` ; then echo "Bootstrapping dependencies for openSUSE-based OSes..." BootstrapSuseCommon - elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then echo "Bootstrapping dependencies for Archlinux..." BootstrapArchCommon else @@ -80,9 +80,9 @@ Bootstrap() { echo "--debug flag." exit 1 fi - elif [ -f /etc/manjaro-release ] ; then + elif [ -f /etc/manjaro-release ]; then ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon - elif [ -f /etc/gentoo-release ] ; then + elif [ -f /etc/gentoo-release ]; then ExperimentalBootstrap "Gentoo" BootstrapGentooCommon elif uname | grep -iq FreeBSD ; then ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd @@ -106,7 +106,7 @@ for arg in "$@" ; do # This first clause is redundant with the third, but hedging on portability if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then VERBOSE=1 - elif [ "$arg" = "--debug" ] ; then + elif [ "$arg" = "--debug" ]; then DEBUG=1 fi done @@ -164,7 +164,7 @@ elif [ "$1" != "--_skip-to-install" ]; then echo "Creating virtual environment..." rm -rf "$VENV_PATH" DeterminePythonVersion - if [ "$VERBOSE" = 1 ] ; then + if [ "$VERBOSE" = 1 ]; then virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH else virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 74e6f343f..a13ef2c94 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -1,5 +1,6 @@ # This is the flattened list of requirements for the packages letsencrypt-auto -# installs. +# installs. To generate this, do `pip install -r py26reqs.txt -e acme -e . -e +# letsencrypt-apache`, `pip freeze`, and then gather the hashes. # sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo From fc52608b40177c5dfb55c1679cf341112c488db7 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 16:37:42 -0500 Subject: [PATCH 020/201] Rewrap some comments. --- letsencrypt_auto/letsencrypt-auto.template | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 595ec9eb9..6f11efce3 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -171,12 +171,12 @@ elif [ "$1" != "--_skip-to-install" ]; then fi # Now we drop into Python so we don't have to install even more - # dependencies (curl, etc.), for better flow control, and for the - # option of future Windows compatibility. + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. # - # This Python script prints a path to a temp dir - # containing a new copy of letsencrypt-auto or returns non-zero. - # There is no $ interpolation due to quotes on heredoc delimiters. + # This Python script prints a path to a temp dir containing a new copy of + # letsencrypt-auto or returns non-zero. There is no $ interpolation due to + # quotes on heredoc delimiters. set +e # ------------------------------------------------------------------------- TEMP_DIR=`$LE_PYTHON - << "UNLIKELY_EOF" From 5bae8e0ac19b8af0acf970ba59e38106d08e0b2f Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 17:48:59 -0500 Subject: [PATCH 021/201] Install not only LE's dependencies but LE itself. --- .../pieces/letsencrypt-auto-requirements.txt | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index a13ef2c94..8d4d275b5 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -1,6 +1,6 @@ -# This is the flattened list of requirements for the packages letsencrypt-auto -# installs. To generate this, do `pip install -r py26reqs.txt -e acme -e . -e -# letsencrypt-apache`, `pip freeze`, and then gather the hashes. +# This is the flattened list of packages letsencrypt-auto installs. To generate +# this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, +# `pip freeze`, and then gather the hashes. # sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo @@ -185,3 +185,15 @@ zope.event==4.1.0 # sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 + +# sha256: QQXOBA1ikgir11szeDu2mzswOs0XB_P8j0Q8rtLb6ws +# sha256: rGNwiYAwPNxSEIw0lj6fZPHvlCJHzJLtzEBv7e5a4FM +acme==0.0.0.dev20151201 + +# sha256: ySOpZpw5OfxYrTcEDWavQPAB0L10NDaAzFcjuAvPn0Q +# sha256: nEFgN9dyy36JKgl7UX26E0xoH36VmpW-JVg6Vt-ZVzE +letsencrypt==0.0.0.dev20151201 + +# sha256: 3mej6BrXy_CIQ4UO0QlQXTa1oM6NtdG8vPFxLHuRhdY +# sha256: hwwDNpfTZCsJcogSAo0kpW8wmO4l_7S101OiPOKnmZw +letsencrypt-apache==0.0.0.dev20151201 From be6c34de32eec35b861decc08fedf94a0d3a566e Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 17:56:39 -0500 Subject: [PATCH 022/201] Make --no-self-upgrade public. This replaces --_skip-to-install and is suitable for people who have audited letsencrypt-auto and wish to run it as is, without upgrading to the latest version. Also... * rm temp dirs when done. No longer reuse a single temp dir across phases so the user doesn't have to pass a temp dir with --no-self-upgrade as phase 1 itself used to. * Swap stanzas in the big "if" so we aren't testing negatives all the time. * Fix a bug in which we ran peep with $LE_PYTHON rather than the python in the venv. * Bootstrap only if it looks like we never got to the point of making a venv before. * Move venv creation into Phase 2. Besides the practical benefit of ensuring there's a venv if a user passes --no-self-upgrade, this has the philosophical advantage of making Phase 1 more minimal, giving us more latitude to change behavior in updates. --- letsencrypt_auto/letsencrypt-auto.template | 93 +++++++++++++--------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 6f11efce3..3c64356c0 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -1,6 +1,9 @@ #!/bin/sh # # Download and run the latest release version of the Let's Encrypt client. +# +# WARNING: "letsencrypt-auto" IS A GENERATED FILE. EDIT +# letsencrypt-auto.template INSTEAD. set -e # Work even if somebody does "sh thisscript.sh". @@ -147,10 +150,55 @@ else SUDO= fi -if [ "$1" = "--os-packages-only" ]; then +if [ ! -f $VENV_BIN/letsencrypt ]; then + # If it looks like we've never bootstrapped before, bootstrap: Bootstrap -elif [ "$1" != "--_skip-to-install" ]; then - echo "Upgrading letsencrypt-auto..." +fi +if [ "$1" = "--os-packages-only" ]; then + echo "OS packages installed." +elif [ "$1" = "--no-self-upgrade" ]; then + # Phase 2: Create venv, install LE, and run. + + shift 1 # the --no-self-upgrade arg + echo "Creating virtual environment..." + # TODO: Embed LE version here, and compare it against letsencrypt --version. + # If it matches, there's no need to recreate the venv. + rm -rf "$VENV_PATH" + DeterminePythonVersion + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + else + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null + fi + + # Install Python dependencies with peep, then run letsencrypt. + echo "Installing Python package dependencies..." + TEMP_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'le'` # Linux || OS X + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt +{{ letsencrypt-auto-requirements.txt }} +UNLIKELY_EOF + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py +{{ peep.py }} +UNLIKELY_EOF + # --------------------------------------------------------------------------- + set +e + PEEP_OUT=`$VENV_BIN/python $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_STATUS=$? + set -e + rm -rf $TEMP_DIR + if [ "$PEEP_STATUS" = 0 ]; then + echo "Running letsencrypt..." + echo " " $SUDO $VENV_BIN/letsencrypt "$@" + $SUDO $VENV_BIN/letsencrypt "$@" + else + # Report error: + echo $PEEP_OUT + exit 1 + fi +else + # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. if [ ! -f $VENV_BIN/letsencrypt ]; then OLD_VERSION="0.0.0" # ($VENV_BIN/letsencrypt --version) @@ -160,15 +208,8 @@ elif [ "$1" != "--_skip-to-install" ]; then # TODO: Don't bother upgrading if we're already up to date. if [ "$OLD_VERSION" != "1.2.3" ]; then - Bootstrap - echo "Creating virtual environment..." - rm -rf "$VENV_PATH" + echo "Upgrading letsencrypt-auto..." DeterminePythonVersion - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH - else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null - fi # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -193,38 +234,12 @@ UNLIKELY_EOF` echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. - "$0" --_skip-to-install "$TEMP_DIR" "$@" + rm -rf $TEMP_DIR + "$0" --no-self-upgrade "$@" else # Report error: echo $TEMP_DIR exit 1 fi fi # should upgrade -else # --_skip-to-install was passed. - # Install Python dependencies with peep, then run letsencrypt. - echo "Installing Python package dependencies..." - TEMP_DIR="$2" - shift 2 - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt -{{ letsencrypt-auto-requirements.txt }} -UNLIKELY_EOF - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py -{{ peep.py }} -UNLIKELY_EOF - # --------------------------------------------------------------------------- - set +e - PEEP_OUT=`$LE_PYTHON $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` - PEEP_STATUS=$? - set -e - if [ "$PEEP_STATUS" = 0 ]; then - echo "Running letsencrypt..." - echo " " $SUDO $VENV_BIN/letsencrypt "$@" - $SUDO $VENV_BIN/letsencrypt "$@" - else - # Report error: - echo $PEEP_OUT - exit 1 - fi fi From 02255fa024cb101e72e3f5ff7119f20c60abd27b Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 22:38:48 -0500 Subject: [PATCH 023/201] Upgrade peep to 2.5, for compatibility with pip 7.x. --- letsencrypt_auto/pieces/peep.py | 155 +++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 50 deletions(-) diff --git a/letsencrypt_auto/pieces/peep.py b/letsencrypt_auto/pieces/peep.py index 23e917ac6..6b9393a5e 100755 --- a/letsencrypt_auto/pieces/peep.py +++ b/letsencrypt_auto/pieces/peep.py @@ -26,13 +26,13 @@ try: xrange = xrange except NameError: xrange = range -from base64 import urlsafe_b64encode +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify import cgi from collections import defaultdict from functools import wraps from hashlib import sha256 -from itertools import chain -from linecache import getline +from itertools import chain, islice import mimetypes from optparse import OptionParser from os.path import join, basename, splitext, isdir @@ -104,8 +104,18 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar +__version__ = 2, 5, 0 -__version__ = 2, 4, 1 +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True ITS_FINE_ITS_FINE = 0 @@ -149,6 +159,45 @@ def encoded_hash(sha): return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + def run_pip(initial_args): """Delegate to pip the given args (starting with the subcommand), and raise ``PipException`` if something goes wrong.""" @@ -217,6 +266,8 @@ def requirement_args(argv, want_paths=False, want_other=False): if want_other: yield arg +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') HASH_COMMENT_RE = re.compile( r""" @@ -311,7 +362,7 @@ def package_finder(argv): # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ - 'find_links', 'use_wheel', 'allow_external', 'allow_unverified', + 'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified', 'allow_all_external', ('allow_all_prereleases', 'pre'), 'process_dependency_links'] kwargs = {} @@ -434,36 +485,10 @@ class DownloadedReq(object): return True return False - def _path_and_line(self): - """Return the path and line number of the file from which our - InstallRequirement came. - - """ - path, line = (re.match(r'-r (.*) \(line (\d+)\)$', - self._req.comes_from).groups()) - return path, int(line) - @memoize # Avoid hitting the file[cache] over and over. def _expected_hashes(self): """Return a list of known-good hashes for this package.""" - - def hashes_above(path, line_number): - """Yield hashes from contiguous comment lines before line - ``line_number``. - - """ - for line_number in xrange(line_number - 1, 0, -1): - line = getline(path, line_number) - match = HASH_COMMENT_RE.match(line) - if match: - yield match.groupdict()['hash'] - elif not line.lstrip().startswith('#'): - # If we hit a non-comment line, abort - break - - hashes = list(hashes_above(*self._path_and_line())) - hashes.reverse() # because we read them backwards - return hashes + return hashes_above(*path_and_line(self._req)) def _download(self, link): """Download a file, and return its name within my temp dir. @@ -783,6 +808,22 @@ def first_every_last(iterable, first, every, last): last(item) +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + def downloaded_reqs_from_path(path, argv): """Return a list of DownloadedReqs representing the requirements parsed out of a given requirements file. @@ -792,22 +833,8 @@ def downloaded_reqs_from_path(path, argv): """ finder = package_finder(argv) - - def downloaded_reqs(parsed_reqs): - """Just avoid repeating this list comp.""" - return [DownloadedReq(req, argv, finder) for req in parsed_reqs] - - try: - return downloaded_reqs(parse_requirements( - path, options=EmptyOptions(), finder=finder)) - except TypeError: - # session is a required kwarg as of pip 6.0 and will raise - # a TypeError if missing. It needs to be a PipSession instance, - # but in older versions we can't import it from pip.download - # (nor do we need it at all) so we only import it in this except block - from pip.download import PipSession - return downloaded_reqs(parse_requirements( - path, options=EmptyOptions(), session=PipSession(), finder=finder)) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] def peep_install(argv): @@ -823,7 +850,7 @@ def peep_install(argv): try: req_paths = list(requirement_args(argv, want_paths=True)) if not req_paths: - out("You have to ``this`` specify one or more requirements files with the -r option, because\n" + out("You have to specify one or more requirements files with the -r option, because\n" "otherwise there's nowhere for peep to look up the hashes.\n") return COMMAND_LINE_ERROR @@ -864,10 +891,38 @@ def peep_install(argv): print(''.join(output)) +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + def main(): """Be the top-level entrypoint. Return a shell status code.""" commands = {'hash': peep_hash, - 'install': peep_install} + 'install': peep_install, + 'port': peep_port} try: if len(argv) >= 2 and argv[1] in commands: return commands[argv[1]](argv[2:]) From 46779da3b5eea7b4a768820eab24bd417b746e50 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 16:32:59 -0500 Subject: [PATCH 024/201] In Phase 2, recreate a venv and reinstall Python packages only if necessary. * Teach the build script how to do special vars. Factor up file reading. * Use a static string for the PyPI JSON location, as it will soon be overrideable via an env var for testing. --- letsencrypt_auto/build.py | 38 +++++++++--- letsencrypt_auto/letsencrypt-auto.template | 68 +++++++++++---------- letsencrypt_auto/pieces/download_upgrade.py | 9 ++- 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/letsencrypt_auto/build.py b/letsencrypt_auto/build.py index b7dd40890..d89c81470 100755 --- a/letsencrypt_auto/build.py +++ b/letsencrypt_auto/build.py @@ -2,7 +2,8 @@ """Stitch together the letsencrypt-auto script. Implement a simple templating language in which {{ some/file }} turns into the -contents of the file at ./pieces/some/file. +contents of the file at ./pieces/some/file except for certain tokens which have +other, special definitions. """ from os.path import dirname, join @@ -10,18 +11,37 @@ import re from sys import argv +def le_version(build_script_dir): + """Return the version number stamped in letsencrypt/__init__.py.""" + return re.search('''^__version__ = ['"](.+)['"].*''', + file_contents(join(dirname(build_script_dir), + 'letsencrypt', + '__init__.py')), + re.M).group(1) + + +def file_contents(path): + with open(path) as file: + return file.read() + + def main(): dir = dirname(argv[0]) - def replacer(match): - rel_path = match.group(1) - with open(join(dir, 'pieces', rel_path)) as replacement: - return replacement.read() + special_replacements = { + 'LE_AUTO_VERSION': le_version(dir) + } - with open(join(dir, 'letsencrypt-auto.template')) as template: - result = re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', - replacer, - template.read()) + def replacer(match): + token = match.group(1) + if token in special_replacements: + return special_replacements[token] + else: + return file_contents(join(dir, 'pieces', token)) + + result = re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', + replacer, + file_contents(join(dir, 'letsencrypt-auto.template'))) with open(join(dir, 'letsencrypt-auto'), 'w') as out: out.write(result) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 3c64356c0..70dafde2c 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -160,43 +160,49 @@ elif [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. shift 1 # the --no-self-upgrade arg - echo "Creating virtual environment..." - # TODO: Embed LE version here, and compare it against letsencrypt --version. - # If it matches, there's no need to recreate the venv. - rm -rf "$VENV_PATH" - DeterminePythonVersion - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + if [ -f $VENV_BIN/letsencrypt ]; then + INSTALLED_VERSION=$($VENV_BIN/letsencrypt --version 2>&1 | cut -d " " -f 2) else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null + INSTALLED_VERSION="0.0.0" fi + if [ "{{ LE_AUTO_VERSION }}" = $INSTALLED_VERSION ]; then + echo "Reusing old virtual environment." + else + echo "Creating virtual environment..." + rm -rf "$VENV_PATH" + DeterminePythonVersion + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + else + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null + fi - # Install Python dependencies with peep, then run letsencrypt. - echo "Installing Python package dependencies..." - TEMP_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'le'` # Linux || OS X - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt + # Install Python dependencies with peep, then run letsencrypt. + echo "Installing Python package dependencies..." + TEMP_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'le'` # Linux || OS X + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py {{ peep.py }} UNLIKELY_EOF - # --------------------------------------------------------------------------- - set +e - PEEP_OUT=`$VENV_BIN/python $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` - PEEP_STATUS=$? - set -e - rm -rf $TEMP_DIR - if [ "$PEEP_STATUS" = 0 ]; then - echo "Running letsencrypt..." - echo " " $SUDO $VENV_BIN/letsencrypt "$@" - $SUDO $VENV_BIN/letsencrypt "$@" - else - # Report error: - echo $PEEP_OUT - exit 1 + # --------------------------------------------------------------------------- + set +e + PEEP_OUT=`$VENV_BIN/python $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_STATUS=$? + set -e + rm -rf $TEMP_DIR + if [ "$PEEP_STATUS" != 0 ]; then + # Report error: + echo $PEEP_OUT + exit 1 + fi fi + echo "Running letsencrypt..." + echo " " $SUDO $VENV_BIN/letsencrypt "$@" + $SUDO $VENV_BIN/letsencrypt "$@" else # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. @@ -215,9 +221,7 @@ else # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. # - # This Python script prints a path to a temp dir containing a new copy of - # letsencrypt-auto or returns non-zero. There is no $ interpolation due to - # quotes on heredoc delimiters. + # There is no $ interpolation due to quotes on heredoc delimiters. set +e # ------------------------------------------------------------------------- TEMP_DIR=`$LE_PYTHON - << "UNLIKELY_EOF" diff --git a/letsencrypt_auto/pieces/download_upgrade.py b/letsencrypt_auto/pieces/download_upgrade.py index a117e9e0a..ec0a11549 100644 --- a/letsencrypt_auto/pieces/download_upgrade.py +++ b/letsencrypt_auto/pieces/download_upgrade.py @@ -1,3 +1,8 @@ +"""Print a path to a temp dir containing a new copy of letsencrypt-auto. + +On failure, return non-zero. + +""" from distutils.version import LooseVersion from json import loads from os import devnull @@ -67,7 +72,7 @@ class TempDir(object): def latest_stable_version(get, package): """Apply a fairly safe heuristic to determine the latest stable release of a PyPI package.""" - metadata = loads(get('https://pypi.python.org/pypi/%s/json' % package)) + metadata = loads(get('https://pypi.python.org/pypi/letsencrypt/json')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. return str(max(LooseVersion(r) for r @@ -108,7 +113,7 @@ def main(): get = HttpsGetter().get temp = TempDir() try: - stable_tag = 'v' + latest_stable_version(get, 'letsencrypt') + stable_tag = 'v' + latest_stable_version(get) print dirname(verified_new_le_auto(get, stable_tag, temp)) except ExpectedError as exc: print exc.args[0], exc.args[1] From 5cc69d92e7fc67c52a967af9f9af65df1016d177 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 18:51:18 -0500 Subject: [PATCH 025/201] In Phase 1, download a new letsencrypt-auto script only if necessary. * Temp dir creation is now always done in shell. * Split download_upgrade.py into 2 phases itself so we can have it report back the latest LE version and make a decision based on it before doing and major downloading. Rename it for clarity. --- letsencrypt_auto/letsencrypt-auto.template | 79 +++++++++---------- .../pieces/{download_upgrade.py => fetch.py} | 61 +++++++------- 2 files changed, 70 insertions(+), 70 deletions(-) rename letsencrypt_auto/pieces/{download_upgrade.py => fetch.py} (69%) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 70dafde2c..293cc8f84 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -13,6 +13,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin +LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name @@ -102,6 +103,10 @@ Bootstrap() { fi } +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support # for experimental platforms) @@ -165,7 +170,7 @@ elif [ "$1" = "--no-self-upgrade" ]; then else INSTALLED_VERSION="0.0.0" fi - if [ "{{ LE_AUTO_VERSION }}" = $INSTALLED_VERSION ]; then + if [ "$LE_AUTO_VERSION" = "$INSTALLED_VERSION" ]; then echo "Reusing old virtual environment." else echo "Creating virtual environment..." @@ -177,9 +182,9 @@ elif [ "$1" = "--no-self-upgrade" ]; then virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null fi - # Install Python dependencies with peep, then run letsencrypt. - echo "Installing Python package dependencies..." - TEMP_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'le'` # Linux || OS X + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + # There is no $ interpolation due to quotes on heredoc delimiters. # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt {{ letsencrypt-auto-requirements.txt }} @@ -195,7 +200,7 @@ UNLIKELY_EOF set -e rm -rf $TEMP_DIR if [ "$PEEP_STATUS" != 0 ]; then - # Report error: + # Report error. (Otherwise, be quiet.) echo $PEEP_OUT exit 1 fi @@ -205,45 +210,37 @@ UNLIKELY_EOF $SUDO $VENV_BIN/letsencrypt "$@" else # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. - - if [ ! -f $VENV_BIN/letsencrypt ]; then - OLD_VERSION="0.0.0" # ($VENV_BIN/letsencrypt --version) - else - OLD_VERSION="0.0.0" - fi - - # TODO: Don't bother upgrading if we're already up to date. - if [ "$OLD_VERSION" != "1.2.3" ]; then - echo "Upgrading letsencrypt-auto..." - DeterminePythonVersion + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # letsencrypt-auto (which is always the same as that of the letsencrypt + # package). Phase 2 checks the version of the locally installed letsencrypt. + + echo "Checking for new version..." + TEMP_DIR=$(TempDir) + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > $TEMP_DIR/fetch.py +{{ fetch.py }} +UNLIKELY_EOF + # ------------------------------------------------------------------------- + DeterminePythonVersion + REMOTE_VERSION=`$LE_PYTHON $TEMP_DIR/fetch.py --latest-version` + if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. - # - # There is no $ interpolation due to quotes on heredoc delimiters. - set +e - # ------------------------------------------------------------------------- - TEMP_DIR=`$LE_PYTHON - << "UNLIKELY_EOF" -{{ download_upgrade.py }} -UNLIKELY_EOF` - # ------------------------------------------------------------------------- - DOWNLOAD_STATUS=$? - set -e - if [ "$DOWNLOAD_STATUS" = 0 ]; then - # Install new copy of letsencrypt-auto. This preserves permissions and - # ownership from the old copy. - # TODO: Deal with quotes in pathnames. - echo "Installing new version of letsencrypt-auto..." - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" - # TODO: Clean up temp dir safely, even if it has quotes in its path. - rm -rf $TEMP_DIR - "$0" --no-self-upgrade "$@" - else - # Report error: - echo $TEMP_DIR - exit 1 - fi + $LE_PYTHON "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Installing new version of letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + rm -rf $TEMP_DIR fi # should upgrade + "$0" --no-self-upgrade "$@" fi diff --git a/letsencrypt_auto/pieces/download_upgrade.py b/letsencrypt_auto/pieces/fetch.py similarity index 69% rename from letsencrypt_auto/pieces/download_upgrade.py rename to letsencrypt_auto/pieces/fetch.py index ec0a11549..9e7a431b8 100644 --- a/letsencrypt_auto/pieces/download_upgrade.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -1,4 +1,11 @@ -"""Print a path to a temp dir containing a new copy of letsencrypt-auto. +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 On failure, return non-zero. @@ -9,8 +16,7 @@ from os import devnull from os.path import dirname, join import re from subprocess import check_call, CalledProcessError -from sys import exit -from tempfile import mkdtemp +from sys import argv, exit from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError @@ -59,62 +65,59 @@ class HttpsGetter(object): raise ExpectedError("Couldn't download %s." % url, exc) -class TempDir(object): - def __init__(self): - self.path = mkdtemp() - - def write(self, contents, filename): - """Write something to a named file in me.""" - with open(join(self.path, filename), 'w') as file: - file.write(contents) +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) -def latest_stable_version(get, package): - """Apply a fairly safe heuristic to determine the latest stable release of - a PyPI package.""" +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" metadata = loads(get('https://pypi.python.org/pypi/letsencrypt/json')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. return str(max(LooseVersion(r) for r in metadata['releases'].iterkeys() if re.match('^[0-9.]+$', r))) -def verified_new_le_auto(get, tag, temp): +def verified_new_le_auto(get, tag, temp_dir): """Return the path to a verified, up-to-date letsencrypt-auto script. - If the download's signature does not verify or something else goes wrong, - raise ExpectedError. + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. """ le_auto_dir = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/' '%s/letsencrypt-auto/' % tag) - temp.write(get(le_auto_dir + 'letsencrypt-auto'), 'letsencrypt-auto') - temp.write(get(le_auto_dir + 'letsencrypt-auto.sig'), 'letsencrypt-auto.sig') - temp.write(PUBLIC_KEY, 'public_key.pem') - le_auto_path = join(temp.path, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', - join(temp.path, 'public_key.pem'), + join(temp_dir, 'public_key.pem'), '-signature', - join(temp.path, 'letsencrypt-auto.sig'), - le_auto_path], + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], stdout=dev_null, stderr=dev_null) except CalledProcessError as exc: raise ExpectedError("Couldn't verify signature of downloaded " "letsencrypt-auto.", exc) - else: # belt & suspenders - return le_auto_path def main(): get = HttpsGetter().get - temp = TempDir() + flag = argv[1] try: - stable_tag = 'v' + latest_stable_version(get) - print dirname(verified_new_le_auto(get, stable_tag, temp)) + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: print exc.args[0], exc.args[1] return 1 From 55a52d1b963a6126d5268384e59885772819e9e9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 19:11:39 -0500 Subject: [PATCH 026/201] "none" is clearer than "0.0.0" as a sentinel value. --- letsencrypt_auto/letsencrypt-auto.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 293cc8f84..7025cbd9d 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -168,7 +168,7 @@ elif [ "$1" = "--no-self-upgrade" ]; then if [ -f $VENV_BIN/letsencrypt ]; then INSTALLED_VERSION=$($VENV_BIN/letsencrypt --version 2>&1 | cut -d " " -f 2) else - INSTALLED_VERSION="0.0.0" + INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" = "$INSTALLED_VERSION" ]; then echo "Reusing old virtual environment." From 4bcd594234f18f17a3b6fe345cd468b9b49cc01c Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 19:18:48 -0500 Subject: [PATCH 027/201] Put off rm-ing the venv for as long as possible, since it triggers a re-bootstrap. If DeterminePythonVersion has an error, we shouldn't re-bootstrap. --- letsencrypt_auto/letsencrypt-auto.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 7025cbd9d..2f96c8c03 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -174,8 +174,8 @@ elif [ "$1" = "--no-self-upgrade" ]; then echo "Reusing old virtual environment." else echo "Creating virtual environment..." - rm -rf "$VENV_PATH" DeterminePythonVersion + rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH else From 4a44c46c603a299f20f91ef0ae77762df9d25823 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 19:19:20 -0500 Subject: [PATCH 028/201] Add a header for peep errors... ...since the shell's collected output is such a line-break-lacking mess. --- letsencrypt_auto/letsencrypt-auto.template | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 2f96c8c03..0966285f8 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -201,6 +201,7 @@ UNLIKELY_EOF rm -rf $TEMP_DIR if [ "$PEEP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) + echo "Had a problem while downloading and verifying Python packages:" echo $PEEP_OUT exit 1 fi From 6db54e21f6e5ff23666f8c96d31ecfbaa27e0427 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 3 Dec 2015 19:20:49 -0500 Subject: [PATCH 029/201] Correct length of dividers. --- letsencrypt_auto/letsencrypt-auto.template | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 0966285f8..f2b5267c1 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -185,15 +185,15 @@ elif [ "$1" = "--no-self-upgrade" ]; then echo "Installing Python packages..." TEMP_DIR=$(TempDir) # There is no $ interpolation due to quotes on heredoc delimiters. - # --------------------------------------------------------------------------- + # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF - # --------------------------------------------------------------------------- + # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py {{ peep.py }} UNLIKELY_EOF - # --------------------------------------------------------------------------- + # ------------------------------------------------------------------------- set +e PEEP_OUT=`$VENV_BIN/python $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` PEEP_STATUS=$? @@ -219,11 +219,11 @@ else echo "Checking for new version..." TEMP_DIR=$(TempDir) - # ------------------------------------------------------------------------- + # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > $TEMP_DIR/fetch.py {{ fetch.py }} UNLIKELY_EOF - # ------------------------------------------------------------------------- + # --------------------------------------------------------------------------- DeterminePythonVersion REMOTE_VERSION=`$LE_PYTHON $TEMP_DIR/fetch.py --latest-version` if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then From 1da5e472b8fa6af39add67883de114147cc699f7 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 4 Dec 2015 14:22:07 -0500 Subject: [PATCH 030/201] Put quotes around variables that might contain spaces. Bourne does the dumb thing when substituting vars; this helps that. Bash does the smart thing but is unhurt by this. We don't do it to $SUDO because Bourne takes "" as a command rather than a no-op, and we don't want the SUDO= case to generate command-not-found errors. --- letsencrypt_auto/letsencrypt-auto.template | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index f2b5267c1..07adef413 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -43,7 +43,7 @@ DeterminePythonVersion() { echo "Cannot find any Pythons... please install one!" fi - PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ $PYVER -eq 26 ]; then ExperimentalBootstrap "Python 2.6" elif [ $PYVER -lt 26 ]; then @@ -155,7 +155,7 @@ else SUDO= fi -if [ ! -f $VENV_BIN/letsencrypt ]; then +if [ ! -f "$VENV_BIN/letsencrypt" ]; then # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi @@ -165,8 +165,8 @@ elif [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. shift 1 # the --no-self-upgrade arg - if [ -f $VENV_BIN/letsencrypt ]; then - INSTALLED_VERSION=$($VENV_BIN/letsencrypt --version 2>&1 | cut -d " " -f 2) + if [ -f "$VENV_BIN/letsencrypt" ]; then + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) else INSTALLED_VERSION="none" fi @@ -177,28 +177,28 @@ elif [ "$1" = "--no-self-upgrade" ]; then DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi echo "Installing Python packages..." TEMP_DIR=$(TempDir) # There is no $ interpolation due to quotes on heredoc delimiters. # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py + cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" {{ peep.py }} UNLIKELY_EOF # ------------------------------------------------------------------------- set +e - PEEP_OUT=`$VENV_BIN/python $TEMP_DIR/peep.py install -r $TEMP_DIR/letsencrypt-auto-requirements.txt` + PEEP_OUT=`"$VENV_BIN/python" "$TEMP_DIR/peep.py" install -r "$TEMP_DIR/letsencrypt-auto-requirements.txt"` PEEP_STATUS=$? set -e - rm -rf $TEMP_DIR + rm -rf "$TEMP_DIR" if [ "$PEEP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while downloading and verifying Python packages:" @@ -207,8 +207,8 @@ UNLIKELY_EOF fi fi echo "Running letsencrypt..." - echo " " $SUDO $VENV_BIN/letsencrypt "$@" - $SUDO $VENV_BIN/letsencrypt "$@" + echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" + $SUDO "$VENV_BIN/letsencrypt" "$@" else # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. # @@ -220,28 +220,28 @@ else echo "Checking for new version..." TEMP_DIR=$(TempDir) # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > $TEMP_DIR/fetch.py + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" {{ fetch.py }} UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion - REMOTE_VERSION=`$LE_PYTHON $TEMP_DIR/fetch.py --latest-version` + REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. - $LE_PYTHON "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" # Install new copy of letsencrypt-auto. This preserves permissions and # ownership from the old copy. # TODO: Deal with quotes in pathnames. - echo "Installing new version of letsencrypt-auto..." + echo "Replacing letsencrypt-auto..." echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" # TODO: Clean up temp dir safely, even if it has quotes in its path. - rm -rf $TEMP_DIR + rm -rf "$TEMP_DIR" fi # should upgrade "$0" --no-self-upgrade "$@" fi From 8b2c5cbec774c79741e3b489c094fd320d5579e4 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 4 Dec 2015 17:27:37 -0500 Subject: [PATCH 031/201] Update LE package pins to 0.1.0, the public beta. --- .../pieces/letsencrypt-auto-requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 8d4d275b5..820b44396 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -186,14 +186,14 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: QQXOBA1ikgir11szeDu2mzswOs0XB_P8j0Q8rtLb6ws -# sha256: rGNwiYAwPNxSEIw0lj6fZPHvlCJHzJLtzEBv7e5a4FM -acme==0.0.0.dev20151201 +# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 +# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo +acme==0.1.0 -# sha256: ySOpZpw5OfxYrTcEDWavQPAB0L10NDaAzFcjuAvPn0Q -# sha256: nEFgN9dyy36JKgl7UX26E0xoH36VmpW-JVg6Vt-ZVzE -letsencrypt==0.0.0.dev20151201 +# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 +# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 +letsencrypt==0.1.0 -# sha256: 3mej6BrXy_CIQ4UO0QlQXTa1oM6NtdG8vPFxLHuRhdY -# sha256: hwwDNpfTZCsJcogSAo0kpW8wmO4l_7S101OiPOKnmZw -letsencrypt-apache==0.0.0.dev20151201 +# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU +# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI +letsencrypt-apache==0.1.0 From 0c4a7bb3bc580111e9404cb5b528ca46b4d2ccb8 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 9 Dec 2015 13:31:53 -0500 Subject: [PATCH 032/201] Make le-auto pull the requisite things from env vars so we can run against test servers. This should let us create a harness that won't force us to mess with GitHub or PyPI just to test. (I haven't tried this commit yet, but you can if you want to get a head start on testing.) --- letsencrypt_auto/pieces/fetch.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index 9e7a431b8..9625c224a 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -12,7 +12,7 @@ On failure, return non-zero. """ from distutils.version import LooseVersion from json import loads -from os import devnull +from os import devnull, environ from os.path import dirname, join import re from subprocess import check_call, CalledProcessError @@ -20,7 +20,7 @@ from sys import argv, exit from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError -PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe 4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B 2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww @@ -34,7 +34,7 @@ q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== -----END PUBLIC KEY----- -""" # TODO: Replace with real one. +""") # TODO: Replace with real one. class ExpectedError(Exception): @@ -73,7 +73,9 @@ def write(contents, dir, filename): def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" - metadata = loads(get('https://pypi.python.org/pypi/letsencrypt/json')) + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/letsencrypt/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most @@ -90,8 +92,10 @@ def verified_new_le_auto(get, tag, temp_dir): with the verification process, raise ExpectedError. """ - le_auto_dir = ('https://raw.githubusercontent.com/letsencrypt/letsencrypt/' - '%s/letsencrypt-auto/' % tag) + le_auto_dir = environ.get( + 'LE_AUTO_DOWNLOAD_TEMPLATE', + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'letsencrypt-auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') From f5ef0b01a8262243f433ec120da11e522b98d46e Mon Sep 17 00:00:00 2001 From: Greg Osuri Date: Tue, 15 Dec 2015 19:09:23 -0800 Subject: [PATCH 033/201] Vagrantfile: enable NAT engine to use host's resolver VirualBox fails to resolve external hosts on some machines. Handle cases when the host is behind a private network by making the NAT engine use the host's resolver mechanisms to handle DNS requests. --- Vagrantfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index a2759440c..de259a0dc 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,6 +21,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Cannot allocate memory" when running # letsencrypt.client.tests.display.util_test.NcursesDisplayTest v.memory = 1024 + + # Handle cases when the host is behind a private network by making the + # NAT engine use the host's resolver mechanisms to handle DNS requests. + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] end end From edd20a8b8d97c6b374bb30089d1b3e9103c0876b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 11:11:03 -0800 Subject: [PATCH 034/201] The actual 0.1.1 release Due to bug #1966, the previous v0.1.1 tag was missing a version change to letsencrypt/__init__.py this commit and the v0.1.1-corrected tag are the code that was signed and uploaded to PyPI. --- letsencrypt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 1c7815f78..e011c3f9b 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.0.dev0' +__version__ = '0.1.1' From 9d50c3eac902c31764ac4897aa51b07517c51e1a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 13:22:25 -0800 Subject: [PATCH 035/201] Apparently there were more missing version strings :( --- letsencrypt-compatibility-test/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index eb7e23036..91e9099b4 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.1.1' install_requires = [ 'letsencrypt=={0}'.format(version), From edfa79fc5c43a8f30c60d694f73780394d8c8e1c Mon Sep 17 00:00:00 2001 From: evilaliv3 Date: Sun, 27 Dec 2015 12:50:30 +0100 Subject: [PATCH 036/201] Set python version in the travis matrix --- .travis.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8dde06ceb..9863e1d81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,16 @@ env: global: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH - matrix: - - TOXENV=py26 BOULDER_INTEGRATION=1 - - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=lint - - TOXENV=cover +matrix: + include: + - python: "2.6" + env: TOXENV=py26 BOULDER_INTEGRATION=1 + - python: "2.7" + env: TOXENV=py27 BOULDER_INTEGRATION=1 + - python: "2.7" + env: TOXENV=lint + - python: "2.7" + env: TOXENV=cover # Only build pushes to the master branch, PRs, and branches beginning with @@ -44,7 +49,6 @@ addons: sources: - augeas packages: # keep in sync with bootstrap/ubuntu.sh and Boulder - - python - python-dev - python-virtualenv - gcc From c3c4c6c632f25002b7ec8a8911844392b46505f8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 25 Nov 2015 21:24:13 -0800 Subject: [PATCH 037/201] Begin work on a noninteractive iDisplay --- letsencrypt/display/util.py | 255 ++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 01a8cbc92..303c078c3 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -403,6 +403,261 @@ class FileDisplay(object): return OK, selection +class NoninteractiveDisplay(object): + """File-based display.""" + + zope.interface.implements(interfaces.IDisplay) + + def __init__(self, outfile): + super(FileDisplay, self).__init__() + self.outfile = outfile + + def _interaction_fail(self, message, extra=""): + "Error out in case of an attempt to interact in noninteractive mode" + msg ="Missing command line flag or config entry for this setting:\n" + msg += message + if extra: + msg += "\n" + extra + raise MissingCommandlineFlag, msg + + def notification(self, message, height=10, pause=True): + # pylint: disable=unused-argument + """Displays a notification and waits for user acceptance. + + :param str message: Message to display to stdout + :param int height: No effect for NoninteractiveDisplay + :param bool pause: The NoninteractiveDisplay waits for no keyboard + + """ + side_frame = "-" * 79 + message = self._wrap_lines(message) + self.outfile.write( + "{line}{frame}{line}{msg}{line}{frame}{line}".format( + line=os.linesep, frame=side_frame, msg=message)) + + def menu(self, message, choices, ok_label="", cancel_label="", + help_label="", default=None): + # pylint: disable=unused-argument + """Display a menu. + + .. todo:: This doesn't enable the help label/button (I wasn't sold on + any interface I came up with for this). It would be a nice feature + + :param str message: title of menu + :param choices: Menu lines, len must be > 0 + :type choices: list of tuples (tag, item) or + list of descriptions (tags will be enumerated) + + :returns: tuple of the form (code, tag) where + code - int display exit code + tag - str corresponding to the item chosen + :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default + + """ + if default is None: + self._interaction_fail(message, "Choices: " + repr(choices)) + if default == None: + msg ="Missing command line flag or config entry for this choice:\n" + msg += message + msg += "\nChoices: " + repr(choices) + raise MissingCommandlineFlag, msg + + return OK, choices.index(default) + + def input(self, message, default=None): + # pylint: disable=no-self-use + """Accept input from the user. + + :param str message: message to display to the user + + :returns: tuple of (`code`, `input`) where + `code` - str display exit code + `input` - str of the user's input + :rtype: tuple + + """ + ans = raw_input( + textwrap.fill("%s (Enter 'c' to cancel): " % message, 80)) + + if ans == "c" or ans == "C": + return CANCEL, "-1" + else: + return OK, ans + + def yesno(self, message, yes_label="Yes", no_label="No"): + """Query the user with a yes/no question. + + Yes and No label must begin with different letters, and must contain at + least one letter each. + + :param str message: question for the user + :param str yes_label: Label of the "Yes" parameter + :param str no_label: Label of the "No" parameter + + :returns: True for "Yes", False for "No" + :rtype: bool + + """ + side_frame = ("-" * 79) + os.linesep + + message = self._wrap_lines(message) + + self.outfile.write("{0}{frame}{msg}{0}{frame}".format( + os.linesep, frame=side_frame, msg=message)) + + while True: + ans = raw_input("{yes}/{no}: ".format( + yes=_parens_around_char(yes_label), + no=_parens_around_char(no_label))) + + # Couldn't get pylint indentation right with elif + # elif doesn't matter in this situation + if (ans.startswith(yes_label[0].lower()) or + ans.startswith(yes_label[0].upper())): + return True + if (ans.startswith(no_label[0].lower()) or + ans.startswith(no_label[0].upper())): + return False + + def checklist(self, message, tags, default_status=True): + # pylint: disable=unused-argument + """Display a checklist. + + :param str message: Message to display to user + :param list tags: `str` tags to select, len(tags) > 0 + :param bool default_status: Not used for FileDisplay + + :returns: tuple of (`code`, `tags`) where + `code` - str display exit code + `tags` - list of selected tags + :rtype: tuple + + """ + while True: + self._print_menu(message, tags) + + code, ans = self.input("Select the appropriate numbers separated " + "by commas and/or spaces") + + if code == OK: + indices = separate_list_input(ans) + selected_tags = self._scrub_checklist_input(indices, tags) + if selected_tags: + return code, selected_tags + else: + self.outfile.write( + "** Error - Invalid selection **%s" % os.linesep) + else: + return code, [] + + def _scrub_checklist_input(self, indices, tags): + # pylint: disable=no-self-use + """Validate input and transform indices to appropriate tags. + + :param list indices: input + :param list tags: Original tags of the checklist + + :returns: valid tags the user selected + :rtype: :class:`list` of :class:`str` + + """ + # They should all be of type int + try: + indices = [int(index) for index in indices] + except ValueError: + return [] + + # Remove duplicates + indices = list(set(indices)) + + # Check all input is within range + for index in indices: + if index < 1 or index > len(tags): + return [] + # Transform indices to appropriate tags + return [tags[index - 1] for index in indices] + + def _print_menu(self, message, choices): + """Print a menu on the screen. + + :param str message: title of menu + :param choices: Menu lines + :type choices: list of tuples (tag, item) or + list of descriptions (tags will be enumerated) + + """ + # Can take either tuples or single items in choices list + if choices and isinstance(choices[0], tuple): + choices = ["%s - %s" % (c[0], c[1]) for c in choices] + + # Write out the message to the user + self.outfile.write( + "{new}{msg}{new}".format(new=os.linesep, msg=message)) + side_frame = ("-" * 79) + os.linesep + self.outfile.write(side_frame) + + # Write out the menu choices + for i, desc in enumerate(choices, 1): + self.outfile.write( + textwrap.fill("{num}: {desc}".format(num=i, desc=desc), 80)) + + # Keep this outside of the textwrap + self.outfile.write(os.linesep) + + self.outfile.write(side_frame) + + def _wrap_lines(self, msg): # pylint: disable=no-self-use + """Format lines nicely to 80 chars. + + :param str msg: Original message + + :returns: Formatted message respecting newlines in message + :rtype: str + + """ + lines = msg.splitlines() + fixed_l = [] + for line in lines: + fixed_l.append(textwrap.fill(line, 80)) + + return os.linesep.join(fixed_l) + + def _get_valid_int_ans(self, max_): + """Get a numerical selection. + + :param int max: The maximum entry (len of choices), must be positive + + :returns: tuple of the form (`code`, `selection`) where + `code` - str display exit code ('ok' or cancel') + `selection` - int user's selection + :rtype: tuple + + """ + selection = -1 + if max_ > 1: + input_msg = ("Select the appropriate number " + "[1-{max_}] then [enter] (press 'c' to " + "cancel): ".format(max_=max_)) + else: + input_msg = ("Press 1 [enter] to confirm the selection " + "(press 'c' to cancel): ") + while selection < 1: + ans = raw_input(input_msg) + if ans.startswith("c") or ans.startswith("C"): + return CANCEL, -1 + try: + selection = int(ans) + if selection < 1 or selection > max_: + selection = -1 + raise ValueError + + except ValueError: + self.outfile.write( + "{0}** Invalid input **{0}".format(os.linesep)) + + return OK, selection + def separate_list_input(input_): """Separate a comma or space separated list. From 4f33a4dbb5e761f0f21b14c835cee2ff760f487a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 27 Dec 2015 15:24:52 -0800 Subject: [PATCH 038/201] Noninteractive iDisplay basic implementation (no tests or hooks, yet) --- letsencrypt/display/util.py | 244 +++++-------------------- letsencrypt/errors.py | 5 + letsencrypt/tests/display/util_test.py | 2 +- 3 files changed, 55 insertions(+), 196 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 303c078c3..0c7926bd4 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -6,7 +6,7 @@ import dialog import zope.interface from letsencrypt import interfaces - +from letsencrypt import errors WIDTH = 72 HEIGHT = 20 @@ -21,6 +21,20 @@ CANCEL = "cancel" HELP = "help" """Display exit code when for when the user requests more help.""" +def _wrap_lines(msg): # pylint: disable=no-self-use + """Format lines nicely to 80 chars. + + :param str msg: Original message + + :returns: Formatted message respecting newlines in message + :rtype: str + + """ + lines = msg.splitlines() + fixed_l = [] + for line in lines: + fixed_l.append(textwrap.fill(line, 80)) + return os.linesep.join(fixed_l) class NcursesDisplay(object): """Ncurses-based display.""" @@ -178,7 +192,7 @@ class FileDisplay(object): """ side_frame = "-" * 79 - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( line=os.linesep, frame=side_frame, msg=message)) @@ -246,7 +260,7 @@ class FileDisplay(object): """ side_frame = ("-" * 79) + os.linesep - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write("{0}{frame}{msg}{0}{frame}".format( os.linesep, frame=side_frame, msg=message)) @@ -352,21 +366,6 @@ class FileDisplay(object): self.outfile.write(side_frame) - def _wrap_lines(self, msg): # pylint: disable=no-self-use - """Format lines nicely to 80 chars. - - :param str msg: Original message - - :returns: Formatted message respecting newlines in message - :rtype: str - - """ - lines = msg.splitlines() - fixed_l = [] - for line in lines: - fixed_l.append(textwrap.fill(line, 80)) - - return os.linesep.join(fixed_l) def _get_valid_int_ans(self, max_): """Get a numerical selection. @@ -409,20 +408,22 @@ class NoninteractiveDisplay(object): zope.interface.implements(interfaces.IDisplay) def __init__(self, outfile): - super(FileDisplay, self).__init__() + super(NoninteractiveDisplay, self).__init__() self.outfile = outfile - def _interaction_fail(self, message, extra=""): + def _interaction_fail(self, message, cli_flag, extra=""): "Error out in case of an attempt to interact in noninteractive mode" - msg ="Missing command line flag or config entry for this setting:\n" + msg = "Missing command line flag or config entry for this setting:\n" msg += message if extra: msg += "\n" + extra - raise MissingCommandlineFlag, msg + if cli_flag: + msg += "\n\n(You can set this with the {0} flag)".format(cli_flag) + raise errors.MissingCommandlineFlag, msg - def notification(self, message, height=10, pause=True): + def notification(self, message, height=10, pause=False): # pylint: disable=unused-argument - """Displays a notification and waits for user acceptance. + """Displays a notification without waiting for user acceptance. :param str message: Message to display to stdout :param int height: No effect for NoninteractiveDisplay @@ -430,23 +431,20 @@ class NoninteractiveDisplay(object): """ side_frame = "-" * 79 - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( line=os.linesep, frame=side_frame, msg=message)) - def menu(self, message, choices, ok_label="", cancel_label="", - help_label="", default=None): + def menu(self, message, choices, default=None, cli_flag=None, **kwargs): # pylint: disable=unused-argument - """Display a menu. - - .. todo:: This doesn't enable the help label/button (I wasn't sold on - any interface I came up with for this). It would be a nice feature + """Avoid displaying a menu. :param str message: title of menu :param choices: Menu lines, len must be > 0 :type choices: list of tuples (tag, item) or list of descriptions (tags will be enumerated) + :param dict kwargs: absorbs various irrelevant labelling arguments :returns: tuple of the form (code, tag) where code - int display exit code @@ -456,17 +454,11 @@ class NoninteractiveDisplay(object): """ if default is None: - self._interaction_fail(message, "Choices: " + repr(choices)) - if default == None: - msg ="Missing command line flag or config entry for this choice:\n" - msg += message - msg += "\nChoices: " + repr(choices) - raise MissingCommandlineFlag, msg + self._interaction_fail(message, cli_flag, "Choices: " + repr(choices)) return OK, choices.index(default) - def input(self, message, default=None): - # pylint: disable=no-self-use + def input(self, message, default=None, cli_flag=None): """Accept input from the user. :param str message: message to display to the user @@ -475,58 +467,39 @@ class NoninteractiveDisplay(object): `code` - str display exit code `input` - str of the user's input :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default """ - ans = raw_input( - textwrap.fill("%s (Enter 'c' to cancel): " % message, 80)) - - if ans == "c" or ans == "C": - return CANCEL, "-1" + if default is None: + self._interaction_fail(message, cli_flag) else: - return OK, ans + return OK, default - def yesno(self, message, yes_label="Yes", no_label="No"): - """Query the user with a yes/no question. - Yes and No label must begin with different letters, and must contain at - least one letter each. + def yesno(self, message, default=None, cli_flag=None, **kwargs): + # pylint: disable=unused-argument + """Decide Yes or No, without asking anybody :param str message: question for the user - :param str yes_label: Label of the "Yes" parameter - :param str no_label: Label of the "No" parameter + :param dict kwargs: absorbs yes_label, no_label + :raises errors.MissingCommandlineFlag: if there was no default :returns: True for "Yes", False for "No" :rtype: bool """ - side_frame = ("-" * 79) + os.linesep + if default is None: + self._interaction_fail(message, cli_flag) + else: + return OK, default - message = self._wrap_lines(message) - - self.outfile.write("{0}{frame}{msg}{0}{frame}".format( - os.linesep, frame=side_frame, msg=message)) - - while True: - ans = raw_input("{yes}/{no}: ".format( - yes=_parens_around_char(yes_label), - no=_parens_around_char(no_label))) - - # Couldn't get pylint indentation right with elif - # elif doesn't matter in this situation - if (ans.startswith(yes_label[0].lower()) or - ans.startswith(yes_label[0].upper())): - return True - if (ans.startswith(no_label[0].lower()) or - ans.startswith(no_label[0].upper())): - return False - - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default=None, cli_flag=None, **kwargs): # pylint: disable=unused-argument """Display a checklist. :param str message: Message to display to user :param list tags: `str` tags to select, len(tags) > 0 - :param bool default_status: Not used for FileDisplay + :param dict kwargs: absorbs default_status arg :returns: tuple of (`code`, `tags`) where `code` - str display exit code @@ -534,129 +507,10 @@ class NoninteractiveDisplay(object): :rtype: tuple """ - while True: - self._print_menu(message, tags) - - code, ans = self.input("Select the appropriate numbers separated " - "by commas and/or spaces") - - if code == OK: - indices = separate_list_input(ans) - selected_tags = self._scrub_checklist_input(indices, tags) - if selected_tags: - return code, selected_tags - else: - self.outfile.write( - "** Error - Invalid selection **%s" % os.linesep) - else: - return code, [] - - def _scrub_checklist_input(self, indices, tags): - # pylint: disable=no-self-use - """Validate input and transform indices to appropriate tags. - - :param list indices: input - :param list tags: Original tags of the checklist - - :returns: valid tags the user selected - :rtype: :class:`list` of :class:`str` - - """ - # They should all be of type int - try: - indices = [int(index) for index in indices] - except ValueError: - return [] - - # Remove duplicates - indices = list(set(indices)) - - # Check all input is within range - for index in indices: - if index < 1 or index > len(tags): - return [] - # Transform indices to appropriate tags - return [tags[index - 1] for index in indices] - - def _print_menu(self, message, choices): - """Print a menu on the screen. - - :param str message: title of menu - :param choices: Menu lines - :type choices: list of tuples (tag, item) or - list of descriptions (tags will be enumerated) - - """ - # Can take either tuples or single items in choices list - if choices and isinstance(choices[0], tuple): - choices = ["%s - %s" % (c[0], c[1]) for c in choices] - - # Write out the message to the user - self.outfile.write( - "{new}{msg}{new}".format(new=os.linesep, msg=message)) - side_frame = ("-" * 79) + os.linesep - self.outfile.write(side_frame) - - # Write out the menu choices - for i, desc in enumerate(choices, 1): - self.outfile.write( - textwrap.fill("{num}: {desc}".format(num=i, desc=desc), 80)) - - # Keep this outside of the textwrap - self.outfile.write(os.linesep) - - self.outfile.write(side_frame) - - def _wrap_lines(self, msg): # pylint: disable=no-self-use - """Format lines nicely to 80 chars. - - :param str msg: Original message - - :returns: Formatted message respecting newlines in message - :rtype: str - - """ - lines = msg.splitlines() - fixed_l = [] - for line in lines: - fixed_l.append(textwrap.fill(line, 80)) - - return os.linesep.join(fixed_l) - - def _get_valid_int_ans(self, max_): - """Get a numerical selection. - - :param int max: The maximum entry (len of choices), must be positive - - :returns: tuple of the form (`code`, `selection`) where - `code` - str display exit code ('ok' or cancel') - `selection` - int user's selection - :rtype: tuple - - """ - selection = -1 - if max_ > 1: - input_msg = ("Select the appropriate number " - "[1-{max_}] then [enter] (press 'c' to " - "cancel): ".format(max_=max_)) + if default is None: + self._interaction_fail(message, cli_flag, "? ".join(tags)) else: - input_msg = ("Press 1 [enter] to confirm the selection " - "(press 'c' to cancel): ") - while selection < 1: - ans = raw_input(input_msg) - if ans.startswith("c") or ans.startswith("C"): - return CANCEL, -1 - try: - selection = int(ans) - if selection < 1 or selection > max_: - selection = -1 - raise ValueError - - except ValueError: - self.outfile.write( - "{0}** Invalid input **{0}".format(os.linesep)) - - return OK, selection + return OK, default def separate_list_input(input_): diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index 1358d1048..99bb29d9d 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -102,3 +102,8 @@ class StandaloneBindError(Error): class ConfigurationError(Error): """Configuration sanity error.""" + +# NoninteractiveDisplay iDisplay plugin error: + +class MissingCommandlineFlag(Error): + """A command line argument was missing in noninteractive usage""" diff --git a/letsencrypt/tests/display/util_test.py b/letsencrypt/tests/display/util_test.py index 001a9e578..1d0c10d31 100644 --- a/letsencrypt/tests/display/util_test.py +++ b/letsencrypt/tests/display/util_test.py @@ -250,7 +250,7 @@ class FileOutputDisplayTest(unittest.TestCase): "This function is only meant to be for easy viewing{0}" "Test a really really really really really really really really " "really really really really long line...".format(os.linesep)) - text = self.displayer._wrap_lines(msg) + text = display_util._wrap_lines(msg) self.assertEqual(text.count(os.linesep), 3) From d471169b6b2fafade218251897a35f8fab72d7a9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 28 Dec 2015 23:47:32 -0800 Subject: [PATCH 039/201] Add default-handling to the iDisplay interface (and the other iDisplay implementations) --- letsencrypt/display/util.py | 28 ++++++++++++++++++---------- letsencrypt/interfaces.py | 20 +++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 0c7926bd4..0d2dbc9d1 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -63,8 +63,8 @@ class NcursesDisplay(object): """ self.dialog.msgbox(message, height, width=self.width) - def menu(self, message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(self, message, choices, ok_label="OK", cancel_label="Cancel", + help_label="", **_kwargs): """Display a menu. :param str message: title of menu @@ -75,6 +75,7 @@ class NcursesDisplay(object): :param str ok_label: label of the OK button :param str help_label: label of the help button + :param dict _kwargs: absorbs default / cli_args :returns: tuple of the form (`code`, `tag`) where `code` - `str` display_util exit code @@ -119,10 +120,11 @@ class NcursesDisplay(object): return code, int(tag) - 1 - def input(self, message): + def input(self, message, **_kwargs): """Display an input box to the user. :param str message: Message to display that asks for input. + :param dict _kwargs: absorbs default / cli_args :returns: tuple of the form (code, string) where `code` - int display exit code @@ -136,7 +138,7 @@ class NcursesDisplay(object): return self.dialog.inputbox(message, width=self.width, height=height) - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **_kwargs): """Display a Yes/No dialog box. Yes and No label must begin with different letters. @@ -144,6 +146,7 @@ class NcursesDisplay(object): :param str message: message to display to user :param str yes_label: label on the "yes" button :param str no_label: label on the "no" button + :param dict _kwargs: absorbs default / cli_args :returns: if yes_label was selected :rtype: bool @@ -153,13 +156,14 @@ class NcursesDisplay(object): message, self.height, self.width, yes_label=yes_label, no_label=no_label) - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **_kwargs): """Displays a checklist. :param message: Message to display before choices :param list tags: where each is of type :class:`str` len(tags) > 0 :param bool default_status: If True, items are in a selected state by default. + :param dict _kwargs: absorbs default / cli_args :returns: tuple of the form (code, list_tags) where @@ -199,8 +203,8 @@ class FileDisplay(object): if pause: raw_input("Press Enter to Continue") - def menu(self, message, choices, - ok_label="", cancel_label="", help_label=""): + def menu(self, message, choices, ok_label="", cancel_label="", + help_label="", **_kwargs): # pylint: disable=unused-argument """Display a menu. @@ -211,6 +215,7 @@ class FileDisplay(object): :param choices: Menu lines, len must be > 0 :type choices: list of tuples (tag, item) or list of descriptions (tags will be enumerated) + :param dict _kwargs: absorbs default / cli_args :returns: tuple of the form (code, tag) where code - int display exit code @@ -224,11 +229,12 @@ class FileDisplay(object): return code, selection - 1 - def input(self, message): + def input(self, message, **_kwargs): # pylint: disable=no-self-use """Accept input from the user. :param str message: message to display to the user + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `input`) where `code` - str display exit code @@ -244,7 +250,7 @@ class FileDisplay(object): else: return OK, ans - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **_kwargs): """Query the user with a yes/no question. Yes and No label must begin with different letters, and must contain at @@ -253,6 +259,7 @@ class FileDisplay(object): :param str message: question for the user :param str yes_label: Label of the "Yes" parameter :param str no_label: Label of the "No" parameter + :param dict _kwargs: absorbs default / cli_args :returns: True for "Yes", False for "No" :rtype: bool @@ -279,13 +286,14 @@ class FileDisplay(object): ans.startswith(no_label[0].upper())): return False - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **_kwargs): # pylint: disable=unused-argument """Display a checklist. :param str message: Message to display to user :param list tags: `str` tags to select, len(tags) > 0 :param bool default_status: Not used for FileDisplay + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `tags`) where `code` - str display exit code diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index c8a725fde..02d2802ed 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -365,8 +365,8 @@ class IDisplay(zope.interface.Interface): """ - def menu(message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(message, choices, ok_label="OK", # pylint: disable=too-many-arguments + cancel_label="Cancel", help_label="", default=None, cli_flag=None): """Displays a generic menu. :param str message: message to display @@ -377,6 +377,8 @@ class IDisplay(zope.interface.Interface): :param str ok_label: label for OK button :param str cancel_label: label for Cancel button :param str help_label: label for Help button + :param str default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--keep" :returns: tuple of (`code`, `index`) where `code` - str display exit code @@ -384,7 +386,7 @@ class IDisplay(zope.interface.Interface): """ - def input(message): + def input(message, default=None, cli_args=None): """Accept input from the user. :param str message: message to display to the user @@ -396,25 +398,29 @@ class IDisplay(zope.interface.Interface): """ - def yesno(message, yes_label="Yes", no_label="No"): + def yesno(message, yes_label="Yes", no_label="No", default=None, + cli_args=None): """Query the user with a yes/no question. Yes and No label must begin with different letters. :param str message: question for the user + :param str default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--redirect / --no-redirect" :returns: True for "Yes", False for "No" :rtype: bool """ - def checklist(message, tags, default_state): + def checklist(message, tags, default_state, default=None, cli_args=None): """Allow for multiple selections from a menu. :param str message: message to display to the user :param list tags: where each is of type :class:`str` len(tags) > 0 - :param bool default_status: If True, items are in a selected state by - default. + :param bool default_status: If True, items are in a selected state by default. + :param str default: default (non-interactive) state of the checklist + :param str cli_flag: to automate choice from the menu, eg "--domains" """ From 6daaa7a7630f22d59b554ff80524ebf629b0acd2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 00:03:57 -0800 Subject: [PATCH 040/201] Add defaults / cli_flags for yesno() input --- letsencrypt/cli.py | 5 +++-- letsencrypt/display/ops.py | 3 ++- letsencrypt/plugins/manual.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index aba9116f9..5536d122f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -156,7 +156,7 @@ def _determine_account(args, config): "server at {1}".format( regr.terms_of_service, config.server)) return zope.component.getUtility(interfaces.IDisplay).yesno( - msg, "Agree", "Cancel") + msg, "Agree", "Cancel", cli_flag="--agree-tos") try: acc, acme = client.register( @@ -315,7 +315,8 @@ def _handle_subset_cert_request(config, domains, cert): ", ".join(domains), br=os.linesep) if config.expand or config.renew_by_default or zope.component.getUtility( - interfaces.IDisplay).yesno(question, "Expand", "Cancel"): + interfaces.IDisplay).yesno(question, "Expand", "Cancel", + default=True): return "renew", cert else: reporter_util = zope.component.getUtility(interfaces.IReporter) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 5ceb7fcfc..08ce0a0b3 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -197,7 +197,8 @@ def choose_names(installer): "specify ServerNames in your config files in order to allow for " "accurate installation of your certificate.{0}" "If you do use the default vhost, you may specify the name " - "manually. Would you like to continue?{0}".format(os.linesep)) + "manually. Would you like to continue?{0}".format(os.linesep), + default=False, cli_flag="--domains") if manual: return _choose_names_manually() diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 793285e62..7f782a41b 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -165,7 +165,8 @@ s.serve_forever()" """ else: if not self.conf("public-ip-logging-ok"): if not zope.component.getUtility(interfaces.IDisplay).yesno( - self.IP_DISCLAIMER, "Yes", "No"): + self.IP_DISCLAIMER, "Yes", "No", + cli_flag="--manual-public-ip-logging-ok"): raise errors.PluginError("Must agree to IP logging to proceed") self._notify_and_wait(self.MESSAGE_TEMPLATE.format( From 430604b63f336296b9149799878f60161fd1b8c5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 00:12:08 -0800 Subject: [PATCH 041/201] Add flag to enable non-interactivity --- letsencrypt/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5536d122f..4a2ad5e85 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -947,6 +947,12 @@ def prepare_and_parse_args(plugins, args): helpful.add( None, "-t", "--text", dest="text_mode", action="store_true", help="Use the text output instead of the curses UI.") + helpful.add( + None, "-n", "--non-interactive", "--noninteractive", + dest="noninteractive_mode", action="store_true", + help="Run without ever asking for user input. This may require " + "additional command line flags; the client will try to explain " + "which ones are required if it finds one missing") helpful.add( None, "--register-unsafely-without-email", action="store_true", help="Specifying this flag enables registering an account with no " @@ -1374,7 +1380,9 @@ def main(cli_args=sys.argv[1:]): sys.excepthook = functools.partial(_handle_exception, args=args) # Displayer - if args.text_mode: + if args.noninteractive_mode: + displayer = display_util.NoninteractiveDisplay(sys.stdout) + elif args.text_mode: displayer = display_util.FileDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From 78f188968afc462002be0b7097f9e587bdbbc88c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 13:24:23 -0800 Subject: [PATCH 042/201] Update the plugin help to say something that always makes sense (Even if you do --help plugins) This can change again when #1460 is done --- letsencrypt/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4a2ad5e85..62357a191 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1166,9 +1166,8 @@ def _plugins_parsing(helpful, plugins): "plugins", description="Let's Encrypt client supports an " "extensible plugins architecture. See '%(prog)s plugins' for a " "list of all installed plugins and their names. You can force " - "a particular plugin by setting options provided below. Further " - "down this help message you will find plugin-specific options " - "(prefixed by --{plugin_name}).") + "a particular plugin by setting options provided below. Running " + "--help will list flags specific to that plugin.") helpful.add( "plugins", "-a", "--authenticator", help="Authenticator plugin name.") helpful.add( From 4e0b010d3deaf043978ec50bea49095e8f0baa03 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 14:06:40 -0800 Subject: [PATCH 043/201] Improve interface docstrings --- letsencrypt/interfaces.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index 02d2802ed..ced84bc54 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -377,13 +377,16 @@ class IDisplay(zope.interface.Interface): :param str ok_label: label for OK button :param str cancel_label: label for Cancel button :param str help_label: label for Help button - :param str default: default (non-interactive) choice from the menu + :param int default: default (non-interactive) choice from the menu :param str cli_flag: to automate choice from the menu, eg "--keep" :returns: tuple of (`code`, `index`) where `code` - str display exit code `index` - int index of the user's selection + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ def input(message, default=None, cli_args=None): @@ -411,6 +414,9 @@ class IDisplay(zope.interface.Interface): :returns: True for "Yes", False for "No" :rtype: bool + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ def checklist(message, tags, default_state, default=None, cli_args=None): @@ -422,6 +428,14 @@ class IDisplay(zope.interface.Interface): :param str default: default (non-interactive) state of the checklist :param str cli_flag: to automate choice from the menu, eg "--domains" + :returns: tuple of the form (code, list_tags) where + `code` - int display exit code + `list_tags` - list of str tags selected by the user + :rtype: tuple + + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ From 762697524833c00283feb089edf11da7f5a34f9d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 14:21:05 -0800 Subject: [PATCH 044/201] Make iDisplay.menu() calls non-interactive where possible - And provide helpful errors where they're not --- .../letsencrypt_apache/display_ops.py | 16 +++++++++++----- letsencrypt/cli.py | 2 +- letsencrypt/display/enhancements.py | 2 +- letsencrypt/display/ops.py | 12 ++++++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index 45c55f49a..f4ce14a24 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -78,11 +78,17 @@ def _vhost_menu(domain, vhosts): name_size=disp_name_size) ) - code, tag = zope.component.getUtility(interfaces.IDisplay).menu( - "We were unable to find a vhost with a ServerName or Address of {0}.{1}" - "Which virtual host would you like to choose?".format( - domain, os.linesep), - choices, help_label="More Info", ok_label="Select") + try: + code, tag = zope.component.getUtility(interfaces.IDisplay).menu( + "We were unable to find a vhost with a ServerName or Address of {0}.{1}" + "Which virtual host would you like to choose?".format(domain, os.linesep), + choices, help_label="More Info", ok_label="Select") + except errors.MissingCommandlineFlag, e: + msg = ("Failed to run Apache plugin non-interactively{1}{0}{1}" + "(The best solution is to add ServerName or ServerAlias entries to " + "the VirtualHost directives of your apache configuration files.)".format(e, + os.linesep)) + raise errors.MissingCommandlineFlag, msg return code, tag diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 62357a191..0e77d65e4 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -280,7 +280,7 @@ def _handle_identical_cert_request(config, cert): "Cancel this operation and do nothing"] display = zope.component.getUtility(interfaces.IDisplay) - response = display.menu(question, choices, "OK", "Cancel") + response = display.menu(question, choices, "OK", "Cancel", default=0) if response[0] == "cancel" or response[1] == 2: # TODO: Add notification related to command-line options for # skipping the menu for this case. diff --git a/letsencrypt/display/enhancements.py b/letsencrypt/display/enhancements.py index c56198161..39def1651 100644 --- a/letsencrypt/display/enhancements.py +++ b/letsencrypt/display/enhancements.py @@ -48,7 +48,7 @@ def redirect_by_default(): code, selection = util(interfaces.IDisplay).menu( "Please choose whether HTTPS access is required or optional.", - choices) + choices, default=0, cli_flag="--redirect / --no-redirect") if code != display_util.OK: return False diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 08ce0a0b3..fde9c62d0 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -31,8 +31,16 @@ def choose_plugin(prepared, question): for plugin_ep in prepared] while True: - code, index = util(interfaces.IDisplay).menu( - question, opts, help_label="More Info") + disp = util(interfaces.IDisplay) + try: + code, index = disp.menu(question, opts, help_label="More Info") + except errors.MissingCommandlineFlag: + # use a custom message for this case + raise errors.MissingCommandlineFlag, ("Missing command line flags. For non-interactive " + "execution, you will need to specify a plugin on the command line. Run with " + "'--help plugins' to see a list of options, and see " + " https://eff.org/letsencrypt-plugins for more detail on what the plugins " + "do and how to use them.") if code == display_util.OK: plugin_ep = prepared[index] From 5ed9ac5ae6558280525d30a42ddedfb25fa76165 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 20:02:33 -0800 Subject: [PATCH 045/201] Handle non-interactivity of iDisplay.input() --- letsencrypt/display/ops.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index fde9c62d0..147a8b245 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -151,7 +151,12 @@ def get_email(more=False, invalid=False): msg += ('\n\nIf you really want to skip this, you can run the client with ' '--register-unsafely-without-email but make sure you backup your ' 'account key from /etc/letsencrypt/accounts\n\n') - code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + try: + code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + except errors.MissingCommandlineFlag, e: + msg = ("You should register before running non-interactively, or provide --agree-tos" + " and --email flags") + raise errors.MissingCommandlineFlag, msg if code == display_util.OK: if le_util.safe_email(email): @@ -259,7 +264,8 @@ def _choose_names_manually(): """Manually input names for those without an installer.""" code, input_ = util(interfaces.IDisplay).input( - "Please enter in your domain name(s) (comma and/or space separated) ") + "Please enter in your domain name(s) (comma and/or space separated) ", + cli_flag="--domains") if code == display_util.OK: invalid_domains = dict() From 7daf773c7325699d7cc2c535b37bd5e5160903d7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 20:23:23 -0800 Subject: [PATCH 046/201] Handle noninteractiv calls to iDisplay.checklist() --- letsencrypt/display/ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 147a8b245..aef8f65e6 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -211,7 +211,7 @@ def choose_names(installer): "accurate installation of your certificate.{0}" "If you do use the default vhost, you may specify the name " "manually. Would you like to continue?{0}".format(os.linesep), - default=False, cli_flag="--domains") + default=True) if manual: return _choose_names_manually() @@ -256,7 +256,7 @@ def _filter_names(names): """ code, names = util(interfaces.IDisplay).checklist( "Which names would you like to activate HTTPS for?", - tags=names) + tags=names, cli_flag="--domains") return code, [str(s) for s in names] From 548ba6b6552747fc9edfb42d67b92003ecd70c56 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 29 Dec 2015 20:25:07 -0800 Subject: [PATCH 047/201] lint --- letsencrypt-apache/letsencrypt_apache/display_ops.py | 1 + letsencrypt/display/ops.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index f4ce14a24..73fec220e 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -4,6 +4,7 @@ import os import zope.component +from letsencrypt import errors from letsencrypt import interfaces import letsencrypt.display.util as display_util diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index aef8f65e6..90d3d97c3 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -153,7 +153,7 @@ def get_email(more=False, invalid=False): 'account key from /etc/letsencrypt/accounts\n\n') try: code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) - except errors.MissingCommandlineFlag, e: + except errors.MissingCommandlineFlag: msg = ("You should register before running non-interactively, or provide --agree-tos" " and --email flags") raise errors.MissingCommandlineFlag, msg From 50fa6c7f22093a14c93fef79cf7f0c875ea8e044 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 30 Dec 2015 15:10:02 -0800 Subject: [PATCH 048/201] Error helpfully on "letsencrypt --standalone" - Also refactor choose_configurator_plugins a bit --- letsencrypt/cli.py | 79 ++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0e77d65e4..32833e51a 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -432,21 +432,6 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "a test certificate (domains: {0}). We will not do that " "unless you use the --break-my-certs flag!".format(names)) -def set_configurator(previously, now): - """ - Setting configurators multiple ways is okay, as long as they all agree - :param str previously: previously identified request for the installer/authenticator - :param str requested: the request currently being processed - """ - if now is None: - # we're not actually setting anything - return previously - if previously: - if previously != now: - msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" - raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) - return now - def diagnose_configurator_problem(cfg_type, requested, plugins): """ @@ -480,22 +465,28 @@ def diagnose_configurator_problem(cfg_type, requested, plugins): raise errors.PluginSelectionError(msg) -def choose_configurator_plugins(args, config, plugins, verb): # pylint: disable=too-many-branches +def set_configurator(previously, now): """ - Figure out which configurator we're going to use - :raises error.PluginSelectionError if there was a problem + Setting configurators multiple ways is okay, as long as they all agree + :param str previously: previously identified request for the installer/authenticator + :param str requested: the request currently being processed """ + if now is None: + # we're not actually setting anything + return previously + if previously: + if previously != now: + msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" + raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) + return now - # Which plugins do we need? - need_inst = need_auth = (verb == "run") - if verb == "certonly": - need_auth = True - if verb == "install": - need_inst = True - if args.authenticator: - logger.warn("Specifying an authenticator doesn't make sense in install mode") +def cli_plugin_requests(args): + """ + Figure out which plugins the user requested with CLI and config options - # Which plugins did the user request? + :returns: (requested authenticator string or None, requested installer string or None) + :rtype: tuple + """ req_inst = req_auth = args.configurator req_inst = set_configurator(req_inst, args.installer) req_auth = set_configurator(req_auth, args.authenticator) @@ -512,6 +503,40 @@ def choose_configurator_plugins(args, config, plugins, verb): # pylint: disable if args.manual: req_auth = set_configurator(req_auth, "manual") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) + return req_auth, req_inst + + +noninstaller_plugins = ["webroot", "manual", "standalone"] + +def choose_configurator_plugins(args, config, plugins, verb): + """ + Figure out which configurator we're going to use + :raises errors.PluginSelectionError if there was a problem + """ + + req_auth, req_inst = cli_plugin_requests(args) + + # Which plugins do we need? + if verb == "run": + need_inst = need_auth = True + if req_auth in noninstaller_plugins and not req_inst: + msg = ('With the {0} plugin, you probably want to use the "certonly" command, eg:{1}' + '{1} {2} certonly --{0}{1}{1}' + '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins' + '{1} and "--help plugins" for more information.)'.format( + req_auth, os.linesep, cmd)) + + raise errors.MissingCommandlineFlag, msg + else: + need_inst = need_auth = False + if verb == "certonly": + need_auth = True + if verb == "install": + need_inst = True + if args.authenticator: + logger.warn("Specifying an authenticator doesn't make sense in install mode") + + # Try to meet the user's request and/or ask them to pick plugins authenticator = installer = None From 833e61a411fb983992b54c998aa12addd1fe82c3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 30 Dec 2015 15:11:27 -0800 Subject: [PATCH 049/201] Update usage to be letsencrypt-auto or letsencrypt, as appropriate --- letsencrypt/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 32833e51a..196c64f98 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -44,6 +44,11 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) +# For help strings, figure out how the user ran us. +# When invoked from letsencrypt-auto, sys.argv[0] is something like: +# /home/user/.local/share/letsencrypt/bin/letsencrypt" +fragment = os.path.join(".local", "share", "letsencrypt") +cli_command = "letsencrypt-auto" if fragment in sys.argv[0] else "letsencrypt" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want # to replace as much of it as we can... @@ -51,7 +56,7 @@ logger = logging.getLogger(__name__) # This is the stub to include in help generated by argparse SHORT_USAGE = """ - letsencrypt [SUBCOMMAND] [options] [-d domain] [-d domain] ... + {0} [SUBCOMMAND] [options] [-d domain] [-d domain] ... The Let's Encrypt agent can obtain and install HTTPS/TLS/SSL certificates. By default, it will attempt to use a webserver both for obtaining and installing @@ -65,7 +70,7 @@ the cert. Major SUBCOMMANDS are: config_changes Show changes made to server config during installation plugins Display information about installed plugins -""" +""".format(cli_command) # This is the short help for letsencrypt --help, where we disable argparse # altogether From ef09362a9ff52a53ea5fc148a651bba105bc81a2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 30 Dec 2015 15:48:59 -0800 Subject: [PATCH 050/201] bugfix --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 196c64f98..037188bb2 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -529,7 +529,7 @@ def choose_configurator_plugins(args, config, plugins, verb): '{1} {2} certonly --{0}{1}{1}' '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins' '{1} and "--help plugins" for more information.)'.format( - req_auth, os.linesep, cmd)) + req_auth, os.linesep, cli_command)) raise errors.MissingCommandlineFlag, msg else: From d358c8d1c0a037cedb18b88f077533e60a84be34 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 30 Dec 2015 15:52:51 -0800 Subject: [PATCH 051/201] Correct docstrings & associated typing confusion --- letsencrypt/display/util.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 0d2dbc9d1..a0ebaffa0 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -77,9 +77,9 @@ class NcursesDisplay(object): :param str help_label: label of the help button :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (`code`, `tag`) where - `code` - `str` display_util exit code - `tag` - `int` index corresponding to the item chosen + :returns: tuple of the form (code, tag) where + code - int display exit code + tag - str corresponding to the item chosen :rtype: tuple """ @@ -217,9 +217,10 @@ class FileDisplay(object): list of descriptions (tags will be enumerated) :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, tag) where - code - int display exit code - tag - str corresponding to the item chosen + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection + :rtype: tuple """ @@ -452,11 +453,12 @@ class NoninteractiveDisplay(object): :param choices: Menu lines, len must be > 0 :type choices: list of tuples (tag, item) or list of descriptions (tags will be enumerated) + :param int default: the default choice :param dict kwargs: absorbs various irrelevant labelling arguments - :returns: tuple of the form (code, tag) where - code - int display exit code - tag - str corresponding to the item chosen + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection :rtype: tuple :raises errors.MissingCommandlineFlag: if there was no default @@ -464,7 +466,7 @@ class NoninteractiveDisplay(object): if default is None: self._interaction_fail(message, cli_flag, "Choices: " + repr(choices)) - return OK, choices.index(default) + return OK, default def input(self, message, default=None, cli_flag=None): """Accept input from the user. From b5828d92ad2c33d58ab9b24c9a6e42c1f4265200 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 30 Dec 2015 16:07:25 -0800 Subject: [PATCH 052/201] Test cases for NoninteractiveDisplay (and one of the associated bugfixes :) --- letsencrypt/display/util.py | 2 +- letsencrypt/tests/display/util_test.py | 36 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index a0ebaffa0..d02f74681 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -501,7 +501,7 @@ class NoninteractiveDisplay(object): if default is None: self._interaction_fail(message, cli_flag) else: - return OK, default + return default def checklist(self, message, tags, default=None, cli_flag=None, **kwargs): # pylint: disable=unused-argument diff --git a/letsencrypt/tests/display/util_test.py b/letsencrypt/tests/display/util_test.py index 1d0c10d31..8759a7faa 100644 --- a/letsencrypt/tests/display/util_test.py +++ b/letsencrypt/tests/display/util_test.py @@ -4,6 +4,8 @@ import unittest import mock +import letsencrypt.errors as errors + from letsencrypt.display import util as display_util @@ -278,6 +280,40 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer._get_valid_int_ans(3), (display_util.CANCEL, -1)) +class NoninteractiveDisplayTest(unittest.TestCase): + """Test non-interactive display. + + These tests are pretty easy! + + """ + def setUp(self): + super(NoninteractiveDisplayTest, self).setUp() + self.mock_stdout = mock.MagicMock() + self.displayer = display_util.NoninteractiveDisplay(self.mock_stdout) + + def test_input(self): + d = "an incomputable value" + ret = self.displayer.input("message", default=d) + self.assertEqual(ret, (display_util.OK, d)) + self.assertRaises(errors.MissingCommandlineFlag, self.displayer.input, "message") + + def test_menu(self): + ret = self.displayer.menu("message", CHOICES, default=1) + self.assertEqual(ret, (display_util.OK, 1)) + self.assertRaises(errors.MissingCommandlineFlag, self.displayer.menu, "message", CHOICES) + + def test_yesno(self): + d = False + ret = self.displayer.yesno("message", default=d) + self.assertEqual(ret, d) + self.assertRaises(errors.MissingCommandlineFlag, self.displayer.yesno, "message") + + def test_checklist(self): + d = [1, 3] + ret = self.displayer.menu("message", TAGS, default=d) + self.assertEqual(ret, (display_util.OK, d)) + self.assertRaises(errors.MissingCommandlineFlag, self.displayer.checklist, "message", TAGS) + class SeparateListInputTest(unittest.TestCase): """Test Module functions.""" From 96f704f5770a422ce6ad1b88e1330326540aa9a0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 1 Jan 2016 15:29:34 -0800 Subject: [PATCH 053/201] Test cases for various error cases (and associtated bugfixes) --- letsencrypt/cli.py | 27 +++++++++++++++++---------- letsencrypt/display/util.py | 2 +- letsencrypt/tests/cli_test.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 037188bb2..18e7a96b2 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -160,12 +160,14 @@ def _determine_account(args, config): "must agree in order to register with the ACME " "server at {1}".format( regr.terms_of_service, config.server)) - return zope.component.getUtility(interfaces.IDisplay).yesno( - msg, "Agree", "Cancel", cli_flag="--agree-tos") + obj = zope.component.getUtility(interfaces.IDisplay) + return obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) + except errors.MissingCommandlineFlag: + raise except errors.Error as error: logger.debug(error, exc_info=True) raise errors.Error( @@ -636,6 +638,8 @@ def obtain_cert(args, config, plugins): def install(args, config, plugins): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert + # FIXME: be consistent about whether errors are raised or returned from + # this function ... try: installer, _ = choose_configurator_plugins(args, config, @@ -643,14 +647,17 @@ def install(args, config, plugins): except errors.PluginSelectionError, e: return e.message - domains = _find_domains(args, installer) - le_client = _init_le_client( - args, config, authenticator=None, installer=installer) - assert args.cert_path is not None # required=True in the subparser - le_client.deploy_certificate( - domains, args.key_path, args.cert_path, args.chain_path, - args.fullchain_path) - le_client.enhance_config(domains, config) + try: + domains = _find_domains(args, installer) + le_client = _init_le_client( + args, config, authenticator=None, installer=installer) + assert args.cert_path is not None # required=True in the subparser + le_client.deploy_certificate( + domains, args.key_path, args.cert_path, args.chain_path, + args.fullchain_path) + le_client.enhance_config(domains, config) + except errors.MissingCommandlineFlag, e: + return e.message def revoke(args, config, unused_plugins): # TODO: coop with renewal config diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index d02f74681..2ea6b9bfe 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -486,7 +486,7 @@ class NoninteractiveDisplay(object): return OK, default - def yesno(self, message, default=None, cli_flag=None, **kwargs): + def yesno(self, message, yes_label=None, no_label=None, default=None, cli_flag=None): # pylint: disable=unused-argument """Decide Yes or No, without asking anybody diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index ccf16f5b5..688e92ad3 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -81,7 +81,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(1, mock_run.call_count) def _help_output(self, args): - "Run a help command, and return the help string for scrutiny" + "Run a command, and return the ouput string for scrutiny" output = StringIO.StringIO() with mock.patch('letsencrypt.cli.sys.stdout', new=output): self.assertRaises(SystemExit, self._call_stdout, args) @@ -105,6 +105,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue("--checkpoints" not in out) out = self._help_output(['-h']) + self.assertTrue("letsencrypt-auto" not in out) # test cli.cli_command if "nginx" in plugins: self.assertTrue("Use the Nginx plugin" in out) else: @@ -130,6 +131,31 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods out = self._help_output(['-h']) self.assertTrue(cli.usage_strings(plugins)[0] in out) + + def _cli_missing_flag(self, args, message): + "Ensure that a particular error raises a missing cli flag error containing message" + exc = None + try: + #self._call_no_clientmock(args) + with mock.patch('letsencrypt.cli.sys.stderr') as stderr: + out = cli.main(self.standard_args + args[:]) # NOTE: parser can alter its args! + #out = self._help_output(args) + print out + except errors.MissingCommandlineFlag, exc: + #print "checking for " + message + " in\n"+ str(exc) + self.assertTrue(message in str(exc)) + self.assertTrue(exc is not None) + + def test_noninteractive(self): + args = ['-n', 'certonly'] + self._cli_missing_flag(args, "specify a plugin") + args.extend(['--apache', '-d', 'eg.is']) + self._cli_missing_flag(args, "register before running") + with mock.patch('letsencrypt.cli._auth_from_domains'): + with mock.patch('letsencrypt.cli.client.acme_from_config_key'): + args.extend(['--email', 'io@io.is']) + self._cli_missing_flag(args, "--agree-tos") + @mock.patch('letsencrypt.cli.client.acme_client.Client') @mock.patch('letsencrypt.cli._determine_account') @mock.patch('letsencrypt.cli.client.Client.obtain_and_enroll_certificate') @@ -209,6 +235,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods ret, _, _, _ = self._call(args) self.assertTrue("--webroot-path must be set" in ret) + self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") + with mock.patch("letsencrypt.cli._init_le_client") as mock_init: with mock.patch("letsencrypt.cli._auth_from_domains"): self._call(["certonly", "--manual", "-d", "foo.bar"]) From 59f68d074fb16877ee7a70411e315c70cf8ac778 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 1 Jan 2016 15:31:43 -0800 Subject: [PATCH 054/201] Clean-ups & lint chasing --- letsencrypt/tests/cli_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 688e92ad3..5c5e7f8bd 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -136,13 +136,10 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods "Ensure that a particular error raises a missing cli flag error containing message" exc = None try: - #self._call_no_clientmock(args) - with mock.patch('letsencrypt.cli.sys.stderr') as stderr: + with mock.patch('letsencrypt.cli.sys.stderr'): out = cli.main(self.standard_args + args[:]) # NOTE: parser can alter its args! - #out = self._help_output(args) print out except errors.MissingCommandlineFlag, exc: - #print "checking for " + message + " in\n"+ str(exc) self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) From 77ec944da11eebb659f755e4508d838b65551ab0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 1 Jan 2016 17:10:57 -0800 Subject: [PATCH 055/201] One more apache unit test --- .../letsencrypt_apache/tests/display_ops_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py index 6db319d87..590144372 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py @@ -6,6 +6,7 @@ import mock import zope.component from letsencrypt.display import util as display_util +from letsencrypt import errors from letsencrypt_apache import obj @@ -31,6 +32,14 @@ class SelectVhostTest(unittest.TestCase): mock_util().menu.return_value = (display_util.OK, 3) self.assertEqual(self.vhosts[3], self._call(self.vhosts)) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") + def test_noninteractive(self, mock_util): + mock_util().menu.side_effect = errors.MissingCommandlineFlag("no vhost default") + try: + self._call(self.vhosts) + except errors.MissingCommandlineFlag, e: + self.assertTrue("VirtualHost directives" in e.message) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") def test_more_info_cancel(self, mock_util): mock_util().menu.side_effect = [ From d9cde2b9d3a3a96b2e2f64be8eb17da4fd26f299 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 17 Dec 2015 23:45:51 -0500 Subject: [PATCH 056/201] Get the first end-to-end test of letsencrypt-auto passing. To run it, cd letsencrypt_auto && ./build.py && docker build -t lea . && docker run --rm -t -i lea So as not to depend on the state of the host machine, the test runs within an Ubuntu Docker image. This lets us sidestep interaction challenges by setting up passwordless sudo. It also lets us insert our own local CA for the mock HTTPS server. (openssl's SSL_CERT_FILE env var replaces rather than adds to the accepted CAs, meaning later connections to PyPI within the same process chain fail. SSL_CERT_DIR seems not to work at all on OS X.) This also demonstrates a way to test across various Linux distros, even within Travis if we like, Also... * Switch to an official release of ConfigArgParse. * Don't redundantly re-bootstrap on --no-self-upgrade (that is, phase 2). --- letsencrypt_auto/Dockerfile | 33 +++ letsencrypt_auto/__init__.py | 0 letsencrypt_auto/build.py | 34 ++- letsencrypt_auto/letsencrypt-auto.template | 21 +- letsencrypt_auto/pieces/fetch.py | 5 +- .../pieces/letsencrypt-auto-requirements.txt | 4 +- letsencrypt_auto/tests/__init__.py | 7 + letsencrypt_auto/tests/auto_test.py | 247 ++++++++++++++++++ .../tests/certs/ca/my-root-ca.crt.pem | 23 ++ .../tests/certs/ca/my-root-ca.key.pem | 27 ++ .../tests/certs/ca/my-root-ca.srl | 1 + .../tests/certs/localhost/cert.pem | 19 ++ .../tests/certs/localhost/localhost.csr.pem | 17 ++ .../tests/certs/localhost/privkey.pem | 27 ++ .../tests/certs/localhost/server.pem | 46 ++++ letsencrypt_auto/tests/signing.key | 27 ++ 16 files changed, 515 insertions(+), 23 deletions(-) create mode 100644 letsencrypt_auto/Dockerfile create mode 100644 letsencrypt_auto/__init__.py create mode 100644 letsencrypt_auto/tests/__init__.py create mode 100644 letsencrypt_auto/tests/auto_test.py create mode 100644 letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem create mode 100644 letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem create mode 100644 letsencrypt_auto/tests/certs/ca/my-root-ca.srl create mode 100644 letsencrypt_auto/tests/certs/localhost/cert.pem create mode 100644 letsencrypt_auto/tests/certs/localhost/localhost.csr.pem create mode 100644 letsencrypt_auto/tests/certs/localhost/privkey.pem create mode 100644 letsencrypt_auto/tests/certs/localhost/server.pem create mode 100644 letsencrypt_auto/tests/signing.key diff --git a/letsencrypt_auto/Dockerfile b/letsencrypt_auto/Dockerfile new file mode 100644 index 000000000..4bdb1426f --- /dev/null +++ b/letsencrypt_auto/Dockerfile @@ -0,0 +1,33 @@ +# For running tests, build a docker image with a passwordless sudo and a trust +# store we can manipulate. + +FROM ubuntu:trusty + +# Add an unprivileged user: +RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea + +# Let that user sudo: +RUN adduser lea sudo +RUN sed -i.bkp -e \ + 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \ + /etc/sudoers + +# Install pip and nose: +RUN apt-get update && \ + apt-get -q -y install python-pip && \ + apt-get clean +RUN pip install nose + +RUN mkdir -p /home/lea/letsencrypt/letsencrypt + +# Install fake testing CA: +COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/ +RUN update-ca-certificates + +# Copy code: +COPY . /home/lea/letsencrypt/letsencrypt_auto + +USER lea +WORKDIR /home/lea + +CMD ["nosetests", "-s", "letsencrypt/letsencrypt_auto/tests"] diff --git a/letsencrypt_auto/__init__.py b/letsencrypt_auto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/letsencrypt_auto/build.py b/letsencrypt_auto/build.py index d89c81470..9a5fc46a7 100755 --- a/letsencrypt_auto/build.py +++ b/letsencrypt_auto/build.py @@ -6,11 +6,14 @@ contents of the file at ./pieces/some/file except for certain tokens which have other, special definitions. """ -from os.path import dirname, join +from os.path import abspath, dirname, join import re from sys import argv +DIR = dirname(abspath(__file__)) + + def le_version(build_script_dir): """Return the version number stamped in letsencrypt/__init__.py.""" return re.search('''^__version__ = ['"](.+)['"].*''', @@ -25,25 +28,36 @@ def file_contents(path): return file.read() -def main(): - dir = dirname(argv[0]) +def build(version=None, requirements=None): + """Return the built contents of the letsencrypt-auto script. + :arg version: The version to attach to the script. Default: the version of + the letsencrypt package + :arg requirements: The contents of the requirements file to embed. Default: + contents of letsencrypt-auto-requirements.txt + + """ special_replacements = { - 'LE_AUTO_VERSION': le_version(dir) + 'LE_AUTO_VERSION': version or le_version(DIR) } + if requirements: + special_replacements['letsencrypt-auto-requirements.txt'] = requirements def replacer(match): token = match.group(1) if token in special_replacements: return special_replacements[token] else: - return file_contents(join(dir, 'pieces', token)) + return file_contents(join(DIR, 'pieces', token)) - result = re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', - replacer, - file_contents(join(dir, 'letsencrypt-auto.template'))) - with open(join(dir, 'letsencrypt-auto'), 'w') as out: - out.write(result) + return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', + replacer, + file_contents(join(DIR, 'letsencrypt-auto.template'))) + + +def main(): + with open(join(DIR, 'letsencrypt-auto'), 'w') as out: + out.write(build()) if __name__ == '__main__': diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 07adef413..ee18e8366 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -155,13 +155,7 @@ else SUDO= fi -if [ ! -f "$VENV_BIN/letsencrypt" ]; then - # If it looks like we've never bootstrapped before, bootstrap: - Bootstrap -fi -if [ "$1" = "--os-packages-only" ]; then - echo "OS packages installed." -elif [ "$1" = "--no-self-upgrade" ]; then +if [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. shift 1 # the --no-self-upgrade arg @@ -184,7 +178,7 @@ elif [ "$1" = "--no-self-upgrade" ]; then echo "Installing Python packages..." TEMP_DIR=$(TempDir) - # There is no $ interpolation due to quotes on heredoc delimiters. + # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" {{ letsencrypt-auto-requirements.txt }} @@ -216,7 +210,16 @@ else # upgrading. Phase 1 checks the version of the latest release of # letsencrypt-auto (which is always the same as that of the letsencrypt # package). Phase 2 checks the version of the locally installed letsencrypt. - + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$1" = "--os-packages-only" ]; then + echo "OS packages installed." + exit 0 + fi + echo "Checking for new version..." TEMP_DIR=$(TempDir) # --------------------------------------------------------------------------- diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index 9625c224a..d094e6347 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -93,7 +93,7 @@ def verified_new_le_auto(get, tag, temp_dir): """ le_auto_dir = environ.get( - 'LE_AUTO_DOWNLOAD_TEMPLATE', + 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' 'letsencrypt-auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') @@ -129,4 +129,5 @@ def main(): return 0 -exit(main()) +if __name__ == '__main__': + exit(main()) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 820b44396..70b005d2d 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -20,8 +20,8 @@ # sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA cffi==1.3.1 -# sha256: 0ayQAF3qd2CBys5QjLnHMi4EONHA82AN8auXEZEBJME -https://github.com/kuba/ConfigArgParse/archive/a58b35d75a10e8b8fbee7f3c69163b63bb506325.zip#egg=ConfigArgParse +# sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc +ConfigArgParse == 0.10.0 # sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI configobj==5.0.6 diff --git a/letsencrypt_auto/tests/__init__.py b/letsencrypt_auto/tests/__init__.py new file mode 100644 index 000000000..13180b5f5 --- /dev/null +++ b/letsencrypt_auto/tests/__init__.py @@ -0,0 +1,7 @@ +"""Tests for letsencrypt-auto + +For now, run these by saying... :: + + ./build.py && docker build -t lea . && docker run --rm -t -i lea + +""" diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py new file mode 100644 index 000000000..c2b9fc378 --- /dev/null +++ b/letsencrypt_auto/tests/auto_test.py @@ -0,0 +1,247 @@ +"""Tests for letsencrypt-auto""" + +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from contextlib import contextmanager +from functools import partial +from json import dumps +from os import environ +from os.path import abspath, dirname, join +import re +from shutil import rmtree +import socket +import ssl +from subprocess import CalledProcessError, check_output, Popen, PIPE +from tempfile import mkdtemp +from threading import Thread +from unittest import TestCase + +from nose.tools import eq_, nottest, ok_ + +from ..build import build as build_le_auto + + +class RequestHandler(BaseHTTPRequestHandler): + """An HTTPS request handler which is quiet and serves a specific folder.""" + + def __init__(self, resources, *args, **kwargs): + """ + :arg resources: A dict of resource paths pointing to content bytes + + """ + self.resources = resources + BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + """Don't log each request to the terminal.""" + + def do_GET(self): + """Serve a GET request.""" + content = self.send_head() + if content is not None: + self.wfile.write(content) + + def send_head(self): + """Common code for GET and HEAD commands + + This sends the response code and MIME headers and returns either a + bytestring of content or, if none is found, None. + + """ + path = self.path[1:] # Strip leading slash. + content = self.resources.get(path) + if content is None: + self.send_error(404, 'Path "%s" not found in self.resources' % path) + else: + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.send_header('Content-Length', str(len(content))) + self.end_headers() + return content + + +def server_and_port(resources): + """Return an unstarted HTTPS server and the port it will use.""" + # Find a port, and bind to it. I can't get the OS to close the socket + # promptly after we shut down the server, so we typically need to try + # a couple ports after the first test case. Setting + # TCPServer.allow_reuse_address = True seems to have nothing to do + # with this behavior. + worked = False + for port in xrange(4443, 4543): + try: + server = HTTPServer(('localhost', port), + partial(RequestHandler, resources)) + except socket.error: + pass + else: + worked = True + server.socket = ssl.wrap_socket( + server.socket, + certfile=join(tests_dir(), 'certs', 'localhost', 'server.pem'), + server_side=True) + break + if not worked: + raise RuntimeError("Couldn't find an unused socket for the testing HTTPS server.") + return server, port + + +@contextmanager +def serving(resources): + """Spin up a local HTTPS server, and yield its base URL. + + Use a self-signed cert generated as outlined by + https://coolaj86.com/articles/create-your-own-certificate-authority-for- + testing/. + + """ + server, port = server_and_port(resources) + thread = Thread(target=server.serve_forever) + try: + thread.start() + yield 'https://localhost:{port}/'.format(port=port) + finally: + server.shutdown() + thread.join() + + +@nottest +def tests_dir(): + """Return a path to the "tests" directory.""" + return dirname(abspath(__file__)) + + +@contextmanager +def ephemeral_dir(): + dir = mkdtemp(prefix='le-test-') + try: + yield dir + finally: + rmtree(dir) + + +def out_and_err(command, input=None, shell=False, env=None): + """Run a shell command, and return stderr and stdout as string. + + If the command returns nonzero, raise CalledProcessError. + + :arg command: A list of commandline args + :arg input: Data to pipe to stdin. Omit for none. + + Remaining args have the same meaning as for Popen. + + """ + process = Popen(command, + stdout=PIPE, + stdin=PIPE, + stderr=PIPE, + shell=shell, + env=env) + out, err = process.communicate(input=input) + status = process.poll() # same as in check_output(), though wait() sounds better + if status: + raise CalledProcessError(status, command, output=out) + return out, err + + +def signed(content, private_key_name='signing.key'): + """Return the signed SHA-256 hash of ``content``, using the given key file.""" + command = ['openssl', 'dgst', '-sha256', '-sign', + join(tests_dir(), private_key_name)] + out, err = out_and_err(command, input=content) + return out + + +def run_le_auto(venv_dir, base_url): + """Run the prebuilt version of letsencrypt-auto, returning stdout and + stderr strings. + + If the command returns other than 0, raise CalledProcessError. + + """ + env = environ.copy() + d = dict(XDG_DATA_HOME=venv_dir, + LE_AUTO_JSON_URL=base_url + 'letsencrypt/json', + LE_AUTO_DIR_TEMPLATE=base_url + '%s/', + # The public key corresponding to signing_keys/test.key: + LE_AUTO_PUBLIC_KEY="""-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg +tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G +hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT +uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl +LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 +Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 +iQIDAQAB +-----END PUBLIC KEY-----""") + env.update(d) + return out_and_err( + join(dirname(tests_dir()), 'letsencrypt-auto') + ' --version', + shell=True, + env=env) + + +class AutoTests(TestCase): + def test_all(self): + """Exercise most branches of letsencrypt-auto. + + The branches: + + * An le-auto upgrade is needed. + * An le-auto upgrade is not needed. + * There was an out-of-date LE script installed. + * There was a current LE script installed. + * There was no LE script installed. (not that important) + * Peep verification passes. + * Peep has a hash mismatch. + * The OpenSSL sig mismatches. + + I violate my usual rule of having small, decoupled tests, because... + + 1. We shouldn't need to run a Cartesian product of the branches: the + phases run in separate shell processes, containing state leakage + pretty effectively. The only shared state is FS state, and it's + limited to a temp dir, assuming (if we dare) all functions properly. + 2. One combination of branches happens to set us up nicely for testing + the next, saving code. + + At the moment, we let bootstrapping run. We probably wanted those + packages installed anyway for local development. + + For tests which get this far, we run merely ``letsencrypt --version``. + The functioning of the rest of the letsencrypt script is covered by + other test suites. + + """ + NEW_LE_AUTO = build_le_auto(version='99.9.9') + NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) + + with ephemeral_dir() as venv_dir: + # This serves a PyPI page with a higher version, a GitHub-alike + # with a corresponding le-auto script, and a matching signature. + resources = {'': """ + Directory listing for / + +

Directory listing for /

+
+ +
+ + """, # TODO: Cut this down. + 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG} + with serving(resources) as base_url: + # Test when a phase-1 upgrade is needed, there's no LE binary + # installed, and peep verifies: + out, err = run_le_auto(venv_dir, base_url) + ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', + err.strip().splitlines()[-1])) + + + # This conveniently sets us up to test the next 2 cases: + # Test when no phase-1 upgrade is needed and no LE upgrade is needed (probably a common case). + + # Test (when no phase-1 upgrade is needed), there's an out-of-date LE script installed, (and peep works). + # Test when peep has a hash mismatch. + # Test when the OpenSSL sig mismatches. diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem b/letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem new file mode 100644 index 000000000..4e4d29bd2 --- /dev/null +++ b/letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID5jCCAs6gAwIBAgIJAI1Qkfyw88REMA0GCSqGSIb3DQEBBQUAMFUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMRswGQYDVQQKExJNeSBCb2d1cyBS +b290IENlcnQxFDASBgNVBAMTC2V4YW1wbGUuY29tMB4XDTE1MTIwNDIwNTIxNVoX +DTQwMTIwMzIwNTIxNVowVTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3Rh +dGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJvb3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBs +ZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQVQpQ2EH4gTJB +NJP6+ocT3xJwT8mSXYUnvzjj6iv+JxZiXRGzAPziNzrrSRKY0yDHF+UiJwuOerLa +n8laZkLb1Ogqzs2u64rKeb0xWv90Qp+eXG0J/1xb4dw+GExqe5QFo1JUJzO/eK7m +1S04SeFkN1qV9mD5yJUy7DGiTUzDHgCxM2tXMLusXYqkxsQQ9+2EJ7BEOK4YJGEx +Sign5FuSxb64PiNow6OA97CaLl7tV4INP4w195ueDRIaS4poeOep4s8U7IAdMjIZ +EryJgKNCij50xK92vPBBJSj0NOitltBlwoEqkOZpQCOZamFd6nvt78LQ6W8Am+l6 +y6oCON5JAgMBAAGjgbgwgbUwHQYDVR0OBBYEFAlrdStDhaayLLj89Whe3Gc+HE8y +MIGFBgNVHSMEfjB8gBQJa3UrQ4Wmsiy4/PVoXtxnPhxPMqFZpFcwVTELMAkGA1UE +BhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJv +b3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBsZS5jb22CCQCNUJH8sPPERDAMBgNVHRME +BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQC7KAQfDTiNM3QO8Ic3x21CAPJUavkH +zshifN+Ei0+nmseHDTCTgsGfGDOToLUpUEZ4PuiHnz08UwRfd9wotc3SgY9ZaXMe +vRs8KUAF9EoyTvESzPyv2b6cS9NNMpj5y7KyXSyP17VoGbNavtiGQ4dwgEH6VgNl +0RtBvcSBv/tqxIIx1tWzL74tVEm0Kbd9BAZsYpQNKL8e6WXP35/j0PvCCvtofGrA +E8LTqMz4kCwnX+QaJIMJhBophRCsjXdAkvFbFxX0DGPztQtzIwBPcdMjsft7AFeE +0XchhDDXxw8YsbpvPfCvrD8XiiVuBycbnB1zt0LLVwB/QsCzUW9ImpLC +-----END CERTIFICATE----- diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem b/letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem new file mode 100644 index 000000000..9caa7ddaa --- /dev/null +++ b/letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0FUKUNhB+IEyQTST+vqHE98ScE/Jkl2FJ7844+or/icWYl0R +swD84jc660kSmNMgxxflIicLjnqy2p/JWmZC29ToKs7NruuKynm9MVr/dEKfnlxt +Cf9cW+HcPhhManuUBaNSVCczv3iu5tUtOEnhZDdalfZg+ciVMuwxok1Mwx4AsTNr +VzC7rF2KpMbEEPfthCewRDiuGCRhMUooJ+RbksW+uD4jaMOjgPewmi5e7VeCDT+M +Nfebng0SGkuKaHjnqeLPFOyAHTIyGRK8iYCjQoo+dMSvdrzwQSUo9DTorZbQZcKB +KpDmaUAjmWphXep77e/C0OlvAJvpesuqAjjeSQIDAQABAoIBAH+qbVzneV3wxjwh +HUHi/p3VyHXc3xh7iNq3mwRH/1eK2nPCttLsGwwBbnC64dOXJfH7maWZKcLRPAMv +gfOM0RHn4bJB8tdrbizv91lke0DihvBDkWpb+1wvB4lh2Io0Wpwt3ojFUTfXm87G ++iQRWjbQmQlm5zyKh6uiBDSCjDTQdb9omZEBMAwlGPTZwt8TRUEtWd8QgW8FCHoB +iLER2WBwXdvn3PBtocI3VE6IYDSeZ81Xv+d7925RtVintT8Suk4toYwX+jfSz+wZ +sgHd5V6PSv9a7GUlWoUihD99D9wqDZE8IvMDZ5ofSAUd1KfICDtmsEyugY7u2yYZ +tYt49AECgYEA73f7ITMHg8JsUipqb6eG10gCRtRhkqrrO1g/TNeTBh3CTrQGb56e +y6kmUivn5gK46t3T2N4Ht4IR8fpLcJcbPYPQNulSjmWm5y6WduafXW/VCW1NA9Lc +FyGPkMxFCIVJTLFxfLFepBVvtUzLLDKGGtQxru/GNbBzjdtmVfDPIoECgYEA3rbM +cTfvj+jWrV1YsRbphyjy+k3OJEIVx6KA4s5d7Tp12UfYQp/B3HPhXXm5wqeo1Nos +UAEWZIMi1VoE8iu6jjeJ6uERtbKKQVed25Us/ff0jUPbxlXgiBOtRcllq9d9Srjm +ybHUgfjLsZ2/xpIcOl+oI5pDM9JvD8Sq4ZCFR8kCgYBK/H0tFjeiML2OtS2DLShy +PWBJIbQ0I0Vp3eZkf5TQc30m/ASP61G6YItZa9pAElYpZbEy1cQA2MAZz9DTvt2O +07ndmA57/KTY+6OuM+Vvctd5DjrxmZPFwoKcSvrLAkHDvETXUQtbwkKquRNeEawg +tpWgPAELSufEYhGXk8KpAQKBgBDCqPgMQZcO6rj5QWdyVfi5+C8mE9Fet8ziSdjH +twHXWG8VnQzGgQxaHCewtW4Ut/vsv1D2A/1kcQalU6H18IArZdGrRm3qFcV9FoAj +5dLnChxncu6mH9Odx3htA52/BcrNx3B+VYPCeXHQcVI8RKuP71NelJgdygXhwwpe +mekhAoGBAOUovnqylciYa9HRqo+xZk59eyX+ehhnlV8SeJ2K0PwaQkzQ0KYtCmE7 +kdSdhcv8h/IQKGaFfc/LyFMM/a26PfAeY5bj41UjkT0K5hQrYuL/52xaT401YLcb +Xo+bZz9K0hrdP7TdZFuTY/WxojXgjsVAuAN1NwnJumqxhzPh+hfl +-----END RSA PRIVATE KEY----- diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.srl b/letsencrypt_auto/tests/certs/ca/my-root-ca.srl new file mode 100644 index 000000000..ad6d262b4 --- /dev/null +++ b/letsencrypt_auto/tests/certs/ca/my-root-ca.srl @@ -0,0 +1 @@ +D613482D0EF95DD0 diff --git a/letsencrypt_auto/tests/certs/localhost/cert.pem b/letsencrypt_auto/tests/certs/localhost/cert.pem new file mode 100644 index 000000000..ac83535ce --- /dev/null +++ b/letsencrypt_auto/tests/certs/localhost/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD +ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy +MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw +HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH +a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y +DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 +SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T +Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn +ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM +V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P +NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n +v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN +AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond +3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi +uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== +-----END CERTIFICATE----- diff --git a/letsencrypt_auto/tests/certs/localhost/localhost.csr.pem b/letsencrypt_auto/tests/certs/localhost/localhost.csr.pem new file mode 100644 index 000000000..8a6189f88 --- /dev/null +++ b/letsencrypt_auto/tests/certs/localhost/localhost.csr.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx +ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j +YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZYgiLzoyKzh +RAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/ZfeUJ5aEqavcIlhdWADur/bc85FACK5 +XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGseR7+IDxOQO5ltYbNUtvxMHzeKkE4 +PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E+IJCXDI5rKMeZ2WHxyp9UTytYSbn +/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAdvQJoUQb1C65QM8mXkrvhGvoicxBk +o+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrKVzYV25ruj/B/RLBKHFLgDUOoD8dY +sQxXoxIQXwIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAFbg3WrAokoPx7iAYG6z +PqeDd4/XanXjeL4Ryxv6LoGhu69mmBAd3N5ILPyQJjnkWpIjEmJDzEcPMzhQjRh5 +GlWTyvKWO4zClYU840KZk7crVkpzNZ+HP0YeM/Agz6sab00ffRcq5m1wEF9MCvDE +8FUXk1HBHRAb/6t9QV/7axsPOkGT8SjQ1v2SCaiB0HQL3sYChYLi5zu4dfmQNPGq +ar9Xm5a0YqOQIFfmy8RSwxk0Q/ipNFTGN1uvlIRkgbT9zPnodxjWZsSI9BF+q5Af +uiE/oAk7MxfJ0LyLfhOWB+T98bKIOVtFT3wMLS1IIgMogwqCEXFf30Q9p2iTEzqT +6UE= +-----END CERTIFICATE REQUEST----- diff --git a/letsencrypt_auto/tests/certs/localhost/privkey.pem b/letsencrypt_auto/tests/certs/localhost/privkey.pem new file mode 100644 index 000000000..18feba403 --- /dev/null +++ b/letsencrypt_auto/tests/certs/localhost/privkey.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe +UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs +eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E ++IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd +vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK +VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 +16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK +46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 +K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P +EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 +Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP +0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x +h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk +JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX +lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K +Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX +nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji +5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl +UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K +fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR +tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G +Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO +mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX +qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB +okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow +-----END RSA PRIVATE KEY----- diff --git a/letsencrypt_auto/tests/certs/localhost/server.pem b/letsencrypt_auto/tests/certs/localhost/server.pem new file mode 100644 index 000000000..c5765dd89 --- /dev/null +++ b/letsencrypt_auto/tests/certs/localhost/server.pem @@ -0,0 +1,46 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe +UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs +eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E ++IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd +vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK +VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 +16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK +46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 +K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P +EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 +Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP +0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x +h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk +JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX +lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K +Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX +nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji +5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl +UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K +fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR +tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G +Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO +mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX +qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB +okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD +ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy +MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw +HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH +a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y +DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 +SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T +Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn +ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM +V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P +NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n +v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN +AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond +3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi +uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== +-----END CERTIFICATE----- diff --git a/letsencrypt_auto/tests/signing.key b/letsencrypt_auto/tests/signing.key new file mode 100644 index 000000000..b9964d00c --- /dev/null +++ b/letsencrypt_auto/tests/signing.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAsMoSzLYQ7E1sdSOkwelgtzKIh2qi3bpXuYtcfFC0XrvWig07 +1NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7GhFW0VdbxL6JdGzS2ShNWkX9hE9z+ +j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTTuUtJmmGcuk3a9Aq/sCT6DdfmTSdP +5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVglLsIVPBuy9IcgHidUQ96hJnoPsDCW +sHwX62495QKEarauyKQrJzFes0EY95orDM47Z5o/NDiQB11m91yNB0MmPYY9QSbn +OA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68iQIDAQABAoIBAQCJE3W2Mqk2f+XL +geKa1BjAkzcXQJCduYGRhUQlw/HGzoBPtGki56Tf53MeHTAkIGfIq3CAr1zRhiNv +8SQzvrLQIx/buvhxhcQJdzqsfwgNcqXT3/OliF34P3LMx8GUfPy/6xq2Qdv4fvwA +nLJH8wyDTKP6RxtdvUY7GSZ+Ln2QQv/3Nco7tax4GHNGom8iSgeH/YKTDnvitdqh +a0fr930QzU39TfOftLmasdmKUOIg8G2wr4Sy6Kn060+OUoQr1fZF5mnLvvQeILCK +uav91JkIeMLggzk+t88IJUFWdOoxv5hWTnNzHyt+/GYfovyRz2fKQMwzdh1F8iM5 ++867rEb9AoGBANn1ncemJBedDshStdCBUH0+2ExPrawveaXOZKnx8/VGFXNi0hAf +KzkntMWd5g5kB077FtKO9CYTBvK4pZBWIFLcJEqAz88JeXME6dfUbRucDr72ko+l +rcLHXj7F0IDVzj/9CphMGAhC9J/4YW9SPcSbMw6dQ6xOk73f1Vowve0DAoGBAM+k +/F+hVqCS3f22Bg9KuDtx+zCydaZxC842DgIkV1SO2iFhNHjnpQ5EIR0WrSYeV2n+ +rD7kVs5OH1HvnGScHaQKtAVqZClSwF14jzE+Aj8XDwxiHLSOhJgKlzfVX7h1ymMh +7fsslDl6xNGQ+40gubhkCLT5qABFKy1mrZ8b+3yDAoGAGLGUI6d2FVrM7vM3+Bx+ +gwIYvWSVl5l1XcypaPupmRNMoNsEU6FEY2BVQcJm6yB4F4GpD0f0709ejSdQUq7/ +UIPydKJtaNZ49QgMelBt4B/pJ8eFyVKLAjNWQSRmQAJ5MJS5m5Gbc2wqjOk2GMen +idvPiAtXPHFWmb9/S42UJwMCgYEAjymAe2qgcGtyNNfIC8kHhqzKdEPGi/ALJKzu +MZnewEURrcv4QpfrnA9rCUQ2Mz7eJA1bsqz6EJmaTIK4wEFGynA6uDUnQ7pzOL7D +cz7+i4MZc/89LVvJnY5Hvk4WBfboiDq/etq8g3jatGaSmTYD9la6DhTHORB3eYD+ +meHQHYMCgYEA18y9hnx2k4vNeBei4YXF4pAvKdwKLQD+CcP9ljb3VT+kXktjRA1C +aWj3HhMwvcxtttfkQzEnwwGRAkTEtNewJ8KFxhmc9nYElZTNZ+SuHD5Dkv8xqoj8 +NvG8rU1eiEyPwE2wQxpM5JLqbo7IWtR0dmptjKoF1gRxn6Wh4TwEiHA= +-----END RSA PRIVATE KEY----- From 7e04f52b90b859bf8fd83bf716731fdaf37af0ba Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 15:30:18 -0500 Subject: [PATCH 057/201] Add built letsencrypt-auto. We're going to keep the built artifact in the tree as per https://github.com/letsencrypt/letsencrypt/issues/1572#issuecomment-161379131 so that... 1. People's current behavior of cloning from git and running the le-auto script still works. 2. We don't have a deprecation timeline and process to babysit. We'll enforce its up-to-dateness with a test. --- letsencrypt_auto/letsencrypt-auto | 1731 +++++++++++++++++++++++++++++ 1 file changed, 1731 insertions(+) create mode 100755 letsencrypt_auto/letsencrypt-auto diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto new file mode 100755 index 000000000..1038ffd04 --- /dev/null +++ b/letsencrypt_auto/letsencrypt-auto @@ -0,0 +1,1731 @@ +#!/bin/sh +# +# Download and run the latest release version of the Let's Encrypt client. +# +# WARNING: "letsencrypt-auto" IS A GENERATED FILE. EDIT +# letsencrypt-auto.template INSTEAD. + +set -e # Work even if somebody does "sh thisscript.sh". + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN=${VENV_PATH}/bin +LE_AUTO_VERSION="0.1.0.dev0" + +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + if command -v python2.7 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2.7} + elif command -v python27 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python27} + elif command -v python2 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2} + elif command -v python > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python} + else + echo "Cannot find any Pythons... please install one!" + fi + + PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ $PYVER -eq 26 ]; then + ExperimentalBootstrap "Python 2.6" + elif [ $PYVER -lt 26 ]; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null ; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null ; then + virtualenv="$virtualenv python-virtualenv" + fi + + $SUDO apt-get install -y --no-install-recommends \ + git \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + libaugeas0 \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 22, 23 (x64) + # - Centos 7 (x64: onD igitalOcean droplet) + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if ! $SUDO $tool install -y \ + python \ + python-devel \ + python-virtualenv + then + if ! $SUDO $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + fi + + # "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) + if ! $SUDO $tool install -y \ + git-core \ + gcc \ + dialog \ + augeas-libs \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} + +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + $SUDO zypper -nq in -l git-core \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates +} + +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./bootstrap/dev/_common_venv.sh + + deps=" + git + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + missing=$("$SUDO" pacman -T $deps) + + if [ "$missing" ]; then + "$SUDO" pacman -S --needed $missing + fi +} + +BootstrapGentooCommon() { + PACKAGES="dev-vcs/git + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + "$SUDO" pmerge --noreplace $PACKAGES + ;; + (portage|*) + "$SUDO" emerge --noreplace $PACKAGES + ;; + esac +} + +BootstrapFreeBsd() { + "$SUDO" pkg install -Ay \ + git \ + python \ + py27-virtualenv \ + augeas \ + libffi +} + +BootstrapMac() { + if ! hash brew 2>/dev/null; then + echo "Homebrew Not Installed\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + brew install augeas + brew install dialog + + if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv + fi +} + + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ]; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/redhat-release ]; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchCommon + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ]; then + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon + elif [ -f /etc/gentoo-release ]; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a peep install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + fi +} + +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then + VERBOSE=1 + elif [ "$arg" = "--debug" ]; then + DEBUG=1 + fi +done + +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrap in a pair of `'`, then append to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings followed it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +if [ "$1" = "--no-self-upgrade" ]; then + # Phase 2: Create venv, install LE, and run. + + shift 1 # the --no-self-upgrade arg + if [ -f "$VENV_BIN/letsencrypt" ]; then + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) + else + INSTALLED_VERSION="none" + fi + if [ "$LE_AUTO_VERSION" = "$INSTALLED_VERSION" ]; then + echo "Reusing old virtual environment." + else + echo "Creating virtual environment..." + DeterminePythonVersion + rm -rf "$VENV_PATH" + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi + + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + # There is no $ interpolation due to quotes on starting heredoc delimiter. + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" +# This is the flattened list of packages letsencrypt-auto installs. To generate +# this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, +# `pip freeze`, and then gather the hashes. + +# sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 +# sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo +# sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 +# sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY +# sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc +# sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U +# sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis +# sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU +# sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M +# sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA +# sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs +# sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA +cffi==1.3.1 + +# sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc +ConfigArgParse == 0.10.0 + +# sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI +configobj==5.0.6 + +# sha256: rvaUNVR6WdmmY0V7hb2CtQXOOC24gvhAeWskGV6QjUM +# sha256: F-Cyopbq5TqKIyBxLA2fXjJkUjnlCuLAxwiToL88ZnI +# sha256: agSFbNkcDV2-0d-J8qsKBo4kGMbChiqXjTOaPpaXEsM +# sha256: Gvlc-QCXZIRSkVyi1i3echM_hL5HRbjGF7iE64Ozmho +# sha256: AL5UcOcPhyZBjf18qhTaS0oNH-eAk0UEzyqkfEkLbrQ +# sha256: jg2Kw83Grq8C5ZK_lDNJQ3OHzpNlEve64O4rPv6guYk +# sha256: 2WqBQR437XCoZdk2lyq5guhPVklKDUmC8sGLCA7E16c +# sha256: PWl5CSvtvm3ZEMr1jnbiMTEWAKDcCoKYprdaUIHC7-w +# sha256: e5vcpKkOmI7IYbvzsXB5LstBnzVUq1PWy5v0Tn6ojfc +# sha256: 1vc40Cac149QSCQ9qPFNN9YIR5gZ6tb_tZ5rZMdSZyI +# sha256: 7Tuo9Y-M71rHKXB4W61mQ4OyQ7yhraOhAB8mceCEv2A +# sha256: LC2rxmUOLqj82E-FqbbTcCZMWiO2tL1aAHZv7ASzMaQ +# sha256: gMNj9i7awTcvu1HUqNRR8-BmuvEGRdjBmdCnFlAxPtA +# sha256: B88JF-T8_-mH_DZxErK9gYy6l1YmOfsTv_moOgOKClQ +# sha256: VYSnU5WgJE790QPYe5jnq68Q8C4h_V21yOf52N-g2bE +# sha256: tz2PKTF-1lK0s2Dvw-IPLM5X5EYTk_STfMIx0IOtLyw +# sha256: J4eqAyfnD8bQbQsDvgp97hkUMLE9j1hHoCQMbmjMpeE +# sha256: WiBGvL2G_GOQ2vlSJ1rLSITuPQ2lTH-Baq2la745U8U +# sha256: lkC04TkXiWgAUtJZFcrDsPdNC8l8tj_26atSc9IwFmA +# sha256: vjY4IvCRJqUvHyI81AesY9bcPjXH4oPeipLqJiXN_64 +# sha256: KRKSOvdFX7LSQ5oDckJQfBLK7OfdZlnWL6gqYe2yuuA +cryptography==1.1.1 + +# sha256: nUqSIOTrq9f_YNhT5pw92J3rrV3euaxedor4EezncI4 +# sha256: TTNFnDMlThutz9tHo0CoAc2ARm5G--NdJdMIvxSNrXA +enum34==1.1.1 + +# sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 +# sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM +funcsigs==0.4 + +# sha256: my_FC9PEujBrllG2lBHvIgJtTYM1uTr8IhTO8SRs5wc +# sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs +idna==2.0 + +# sha256: WJWfvsOuQ49axxC7YcQrYQ2Ju05bzDJHswd-I0hzTa0 +# sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY +ipaddress==1.0.15 + +# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs +# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY +mock==1.3.0 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs +# sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs +parsedatetime==1.5 + +# sha256: Rsjbda51oFa9HMB_ohc0_i5gPRGgeDPswe63TDXHLgw +# sha256: 4hJ2JqkebIhduJZol22zECDwry2nKJJLVkgPx8zwlkk +pbr==1.8.1 + +# sha256: WE8LKfzF1SO0M8uJGLL8dNZ-MO4LRKlbrwMVKPQkYZ8 +# sha256: KMoLbp2Zqo3Chuh0ekRxNitpgSolKR3im2qNcKFUWg0 +# sha256: FnrV__UqZyxN3BwaCyUUbWgT67CKmqsKOsRfiltmnDs +# sha256: 5t6mFzqYhye7Ij00lzSa1c3vXAsoLv8tg-X5BlxT-F8 +# sha256: KvXgpKrWYEmVXQc0qk49yMqhep6vi0waJ6Xx7m5A9vw +# sha256: 2YhNwNwuVeJEjklXeNyYmcHIvzeusvQ0wb6nSvk8JoM +# sha256: 4nwv5t_Mhzi-PSxaAi94XrcpcQV-Gp4eNPunO86KcaY +# sha256: Za_W_syPOu0J7kvmNYO8jrRy8GzqpP4kxNHVoaPA4T8 +# sha256: uhxVj7_N-UUVwjlLEVXB3FbivCqcF9MDSYJ8ntimfkY +# sha256: upXqACLctk028MEzXAYF-uNb3z4P6o2S9dD2RWo15Vs +# sha256: QhtlkdFrUJqqjYwVgh1mu5TLSo3EOFytXFG4XUoJbYU +# sha256: MmswXL22-U2vv-LCaxHaiLCrB7igf4GIq511_wxuhBo +# sha256: mu3lsrb-RrN0jqjlIURDiQ0WNAJ77z0zt9rRZVaDAng +# sha256: c77R24lNGqnDx-YR0wLN6reuig3A7q92cnh42xrFzYc +# sha256: k1td1tVYr1EvQlAafAj0HXr_E5rxuzlZ2qOruFkjTWw +# sha256: TKARHPFX3MDy9poyPFtUeHGNaNRfyUNdhL4OwPGGIVs +# sha256: tvE8lTmKP88CJsTc-kSFYLpYZSWc2W7CgQZYZR6TIYk +# sha256: 7mvjDRY1u96kxDJdUH3IoNu95-HBmL1i3bn0MZi54hQ +# sha256: 36eGhYwmjX-74bYXXgAewCc418-uCnzne_m2Ua9nZyk +# sha256: qnf53nKvnBbMKIzUokz1iCQ4j1fXqB5ADEYWRXYphw4 +# sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus +psutil==3.3.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + +# sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M +pycparser==2.14 + +# sha256: iORea7Jd_tJyoe8ucoRh1EtjTCzWiemJtuVqNJxaOuU +# sha256: 8KJgcNbbCIHei8x4RpNLfDyTDY-cedRYg-5ImEvA1nI +pyOpenSSL==0.15.1 + +# sha256: 7qMYNcVuIJavQ2OldFp4SHimHQQ-JH06bWoKMql0H1Y +# sha256: jfvGxFi42rocDzYgqMeACLMjomiye3NZ6SpK5BMl9TU +pyRFC3339==1.0 + +# sha256: Z9WdZs26jWJOA4m4eyqDoXbyHxaodVO1D1cDsj8pusI +python-augeas==0.5.0 + +# sha256: BOk_JJlcQ92Q8zjV2GXKcs4_taU1jU2qSWVXHbNfw-w +# sha256: Pm9ZP-rZj4pSa8PjBpM1MyNuM3KfVS9SiW6lBPVTE_o +python2-pythondialog==3.3.0 + +# sha256: Or5qbT_C-75MYBRCEfRdou2-MYKm9lEa9ru6BZix-ZI +# sha256: k575weEiTZgEBWial__PeCjFbRUXsx1zRkNWwfK3dp4 +# sha256: 6tSu-nAHJJ4F5RsBCVcZ1ajdlXYAifVzCqxWmLGTKRg +# sha256: PMoN8IvQ7ZhDI5BJTOPe0AP15mGqRgvnpzS__jWYNgU +# sha256: Pt5HDT0XujwHY436DRBFK8G25a0yYSemW6d-aq6xG-w +# sha256: aMR5ZPcYbuwwaxNilidyK5B5zURH7Z5eyuzU6shMpzQ +# sha256: 3V05kZUKrkCmyB3hV4lC5z1imAjO_FHRLNFXmA5s_Bg +# sha256: p3xSBiwH63x7MFRdvHPjKZW34Rfup1Axe1y1x6RhjxQ +# sha256: ga-a7EvJYKmgEnxIjxh3La5GNGiSM_BvZUQ-exHr61E +# sha256: 4Hmx2txcBiRswbtv4bI6ULHRFz8u3VEE79QLtzoo9AY +# sha256: -9JnRncsJMuTyLl8va1cueRshrvbG52KdD7gDi-x_F0 +# sha256: mSZu8wo35Dky3uwrfKc-g8jbw7n_cD7HPsprHa5r7-o +# sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM +pytz==2015.7 + +# sha256: ifGx8l3Ne2j1FOjTQaWy60ZvlgrnVoIuqrSAo8GoHCg +# sha256: hP6NW_Tc3MSQAkRsR6FG0XrBD6zwDZCGZZBkrEO2wls +requests==2.8.1 + +# sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE +# sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo +six==1.10.0 + +# sha256: tqk-kJsx4-FGWVhP9zKYZCdZBd3BuGU4Yb3-HllyUPQ +# sha256: 60-YmUtAqOLtziiegRyaOIgK5T65_28DHQ4kOmmw_L8 +Werkzeug==0.11.2 + +# sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo +zope.component==4.2.2 + +# sha256: 3HpZov2Rcw03kxMaXSYbKek-xOKpfxvEh86N7-4v54Y +zope.event==4.1.0 + +# sha256: 8HtjH3pgHNjL0zMtVPQxQscIioMpn4WTVvCNHU1CWbM +# sha256: 3lzKCDuUOdgAL7drvmtJmMWlpyH6sluEKYln8ALfTJQ +# sha256: Z4hBb36n9bipe-lIJTd6ol6L3HNGPge6r5hYsp5zcHc +# sha256: bzIw9yVFGCAeWjcIy7LemMhIME8G497Yv7OeWCXLouE +# sha256: X6V1pSQPBCAMMIhCfQ1Le3N_bpAYgYpR2ND5J6aiUXo +# sha256: UiGUrWpUVzXt11yKg_SNZdGvBk5DKn0yDWT1a6_BLpk +# sha256: 6Mey1AlD9xyZFIyX9myqf1E0FH9XQj-NtbSCUJnOmgk +# sha256: J5Ak8CCGAcPKqQfFOHbjetiGJffq8cs4QtvjYLIocBc +# sha256: LiIanux8zFiImieOoT3P7V75OdgLB4Gamos8scaBSE8 +# sha256: aRGJZUEOyG1E3GuQF-4929WC4MCr7vYrOhnb9sitEys +# sha256: 0E34aG7IZNDK3ozxmff4OuzUFhCaIINNVo-DEN7RLeo +# sha256: 51qUfhXul-fnHgLqMC_rL8YtOiu0Zov5377UOlBqx-c +# sha256: TkXSL7iDIipaufKCoRb-xe4ujRpWjM_2otdbvQ62vPw +# sha256: vOkzm7PHpV4IA7Y9IcWDno5Hm8hcSt9CrkFbcvlPrLI +# sha256: koE4NlJFoOiGmlmZ-8wqRUdaCm7VKklNYNvcVAM1_t0 +# sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro +# sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I +zope.interface==4.1.3 + +# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 +# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo +acme==0.1.0 + +# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 +# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 +letsencrypt==0.1.0 + +# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU +# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI +letsencrypt-apache==0.1.0 + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" +#!/usr/bin/env python +"""peep ("prudently examine every package") verifies that packages conform to a +trusted, locally stored hash and only then installs them:: + + peep install -r requirements.txt + +This makes your deployments verifiably repeatable without having to maintain a +local PyPI mirror or use a vendor lib. Just update the version numbers and +hashes in requirements.txt, and you're all set. + +""" +# This is here so embedded copies of peep.py are MIT-compliant: +# Copyright (c) 2013 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +try: + xrange = xrange +except NameError: + xrange = range +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify +import cgi +from collections import defaultdict +from functools import wraps +from hashlib import sha256 +from itertools import chain, islice +import mimetypes +from optparse import OptionParser +from os.path import join, basename, splitext, isdir +from pickle import dumps, loads +import re +import sys +from shutil import rmtree, copy +from sys import argv, exit +from tempfile import mkdtemp +import traceback +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 +# TODO: Probably use six to make urllib stuff work across 2/3. + +from pkg_resources import require, VersionConflict, DistributionNotFound + +# We don't admit our dependency on pip in setup.py, lest a naive user simply +# say `pip install peep.tar.gz` and thus pull down an untrusted copy of pip +# from PyPI. Instead, we make sure it's installed and new enough here and spit +# out an error message if not: + + +def activate(specifier): + """Make a compatible version of pip importable. Raise a RuntimeError if we + couldn't.""" + try: + for distro in require(specifier): + distro.activate() + except (VersionConflict, DistributionNotFound): + raise RuntimeError('The installed version of pip is too old; peep ' + 'requires ' + specifier) + +# Before 0.6.2, the log module wasn't there, so some +# of our monkeypatching fails. It probably wouldn't be +# much work to support even earlier, though. +activate('pip>=0.6.2') + +import pip +from pip.commands.install import InstallCommand +try: + from pip.download import url_to_path # 1.5.6 +except ImportError: + try: + from pip.util import url_to_path # 0.7.0 + except ImportError: + from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.index import PackageFinder, Link +try: + from pip.log import logger +except ImportError: + from pip import logger # 6.0 +from pip.req import parse_requirements +try: + from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner +except ImportError: + class NullProgressBar(object): + def __init__(self, *args, **kwargs): + pass + + def iter(self, ret, *args, **kwargs): + return ret + + DownloadProgressBar = DownloadProgressSpinner = NullProgressBar + +__version__ = 2, 5, 0 + +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True + + +ITS_FINE_ITS_FINE = 0 +SOMETHING_WENT_WRONG = 1 +# "Traditional" for command-line errors according to optparse docs: +COMMAND_LINE_ERROR = 2 + +ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') + +MARKER = object() + + +class PipException(Exception): + """When I delegated to pip, it exited with an error.""" + + def __init__(self, error_code): + self.error_code = error_code + + +class UnsupportedRequirementError(Exception): + """An unsupported line was encountered in a requirements file.""" + + +class DownloadError(Exception): + def __init__(self, link, exc): + self.link = link + self.reason = str(exc) + + def __str__(self): + return 'Downloading %s failed: %s' % (self.link, self.reason) + + +def encoded_hash(sha): + """Return a short, 7-bit-safe representation of a hash. + + If you pass a sha256, this results in the hash algorithm that the Wheel + format (PEP 427) uses, except here it's intended to be run across the + downloaded archive before unpacking. + + """ + return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') + + +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + +def run_pip(initial_args): + """Delegate to pip the given args (starting with the subcommand), and raise + ``PipException`` if something goes wrong.""" + status_code = pip.main(initial_args) + + # Clear out the registrations in the pip "logger" singleton. Otherwise, + # loggers keep getting appended to it with every run. Pip assumes only one + # command invocation will happen per interpreter lifetime. + logger.consumers = [] + + if status_code: + raise PipException(status_code) + + +def hash_of_file(path): + """Return the hash of a downloaded file.""" + with open(path, 'rb') as archive: + sha = sha256() + while True: + data = archive.read(2 ** 20) + if not data: + break + sha.update(data) + return encoded_hash(sha) + + +def is_git_sha(text): + """Return whether this is probably a git sha""" + # Handle both the full sha as well as the 7-character abbreviation + if len(text) in (40, 7): + try: + int(text, 16) + return True + except ValueError: + pass + return False + + +def filename_from_url(url): + parsed = urlparse(url) + path = parsed.path + return path.split('/')[-1] + + +def requirement_args(argv, want_paths=False, want_other=False): + """Return an iterable of filtered arguments. + + :arg argv: Arguments, starting after the subcommand + :arg want_paths: If True, the returned iterable includes the paths to any + requirements files following a ``-r`` or ``--requirement`` option. + :arg want_other: If True, the returned iterable includes the args that are + not a requirement-file path or a ``-r`` or ``--requirement`` flag. + + """ + was_r = False + for arg in argv: + # Allow for requirements files named "-r", don't freak out if there's a + # trailing "-r", etc. + if was_r: + if want_paths: + yield arg + was_r = False + elif arg in ['-r', '--requirement']: + was_r = True + else: + if want_other: + yield arg + +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') + +HASH_COMMENT_RE = re.compile( + r""" + \s*\#\s+ # Lines that start with a '#' + (?Psha256):\s+ # Hash type is hardcoded to be sha256 for now. + (?P[^\s]+) # Hashes can be anything except '#' or spaces. + \s* # Suck up whitespace before the comment or + # just trailing whitespace if there is no + # comment. Also strip trailing newlines. + (?:\#(?P.*))? # Comments can be anything after a whitespace+# + # and are optional. + $""", re.X) + + +def peep_hash(argv): + """Return the peep hash of one or more files, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + parser = OptionParser( + usage='usage: %prog hash file [file ...]', + description='Print a peep hash line for one or more files: for ' + 'example, "# sha256: ' + 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY".') + _, paths = parser.parse_args(args=argv) + if paths: + for path in paths: + print('# sha256:', hash_of_file(path)) + return ITS_FINE_ITS_FINE + else: + parser.print_usage() + return COMMAND_LINE_ERROR + + +class EmptyOptions(object): + """Fake optparse options for compatibility with pip<1.2 + + pip<1.2 had a bug in parse_requirements() in which the ``options`` kwarg + was required. We work around that by passing it a mock object. + + """ + default_vcs = None + skip_requirements_regex = None + isolated_mode = False + + +def memoize(func): + """Memoize a method that should return the same result every time on a + given instance. + + """ + @wraps(func) + def memoizer(self): + if not hasattr(self, '_cache'): + self._cache = {} + if func.__name__ not in self._cache: + self._cache[func.__name__] = func(self) + return self._cache[func.__name__] + return memoizer + + +def package_finder(argv): + """Return a PackageFinder respecting command-line options. + + :arg argv: Everything after the subcommand + + """ + # We instantiate an InstallCommand and then use some of its private + # machinery--its arg parser--for our own purposes, like a virus. This + # approach is portable across many pip versions, where more fine-grained + # ones are not. Ignoring options that don't exist on the parser (for + # instance, --use-wheel) gives us a straightforward method of backward + # compatibility. + try: + command = InstallCommand() + except TypeError: + # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 + # given)" error. In that version, InstallCommand takes a top=level + # parser passed in from outside. + from pip.baseparser import create_main_parser + command = InstallCommand(create_main_parser()) + # The downside is that it essentially ruins the InstallCommand class for + # further use. Calling out to pip.main() within the same interpreter, for + # example, would result in arguments parsed this time turning up there. + # Thus, we deepcopy the arg parser so we don't trash its singletons. Of + # course, deepcopy doesn't work on these objects, because they contain + # uncopyable regex patterns, so we pickle and unpickle instead. Fun! + options, _ = loads(dumps(command.parser)).parse_args(argv) + + # Carry over PackageFinder kwargs that have [about] the same names as + # options attr names: + possible_options = [ + 'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified', + 'allow_all_external', ('allow_all_prereleases', 'pre'), + 'process_dependency_links'] + kwargs = {} + for option in possible_options: + kw, attr = option if isinstance(option, tuple) else (option, option) + value = getattr(options, attr, MARKER) + if value is not MARKER: + kwargs[kw] = value + + # Figure out index_urls: + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + index_urls = [] + index_urls += getattr(options, 'mirrors', []) + + # If pip is new enough to have a PipSession, initialize one, since + # PackageFinder requires it: + if hasattr(command, '_build_session'): + kwargs['session'] = command._build_session(options) + + return PackageFinder(index_urls=index_urls, **kwargs) + + +class DownloadedReq(object): + """A wrapper around InstallRequirement which offers additional information + based on downloading and examining a corresponding package archive + + These are conceptually immutable, so we can get away with memoizing + expensive things. + + """ + def __init__(self, req, argv, finder): + """Download a requirement, compare its hashes, and return a subclass + of DownloadedReq depending on its state. + + :arg req: The InstallRequirement I am based on + :arg argv: The args, starting after the subcommand + + """ + self._req = req + self._argv = argv + self._finder = finder + + # We use a separate temp dir for each requirement so requirements + # (from different indices) that happen to have the same archive names + # don't overwrite each other, leading to a security hole in which the + # latter is a hash mismatch, the former has already passed the + # comparison, and the latter gets installed. + self._temp_path = mkdtemp(prefix='peep-') + # Think of DownloadedReq as a one-shot state machine. It's an abstract + # class that ratchets forward to being one of its own subclasses, + # depending on its package status. Then it doesn't move again. + self.__class__ = self._class() + + def dispose(self): + """Delete temp files and dirs I've made. Render myself useless. + + Do not call further methods on me after calling dispose(). + + """ + rmtree(self._temp_path) + + def _version(self): + """Deduce the version number of the downloaded package from its filename.""" + # TODO: Can we delete this method and just print the line from the + # reqs file verbatim instead? + def version_of_archive(filename, package_name): + # Since we know the project_name, we can strip that off the left, strip + # any archive extensions off the right, and take the rest as the + # version. + for ext in ARCHIVE_EXTENSIONS: + if filename.endswith(ext): + filename = filename[:-len(ext)] + break + # Handle github sha tarball downloads. + if is_git_sha(filename): + filename = package_name + '-' + filename + if not filename.lower().replace('_', '-').startswith(package_name.lower()): + # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -? + give_up(filename, package_name) + return filename[len(package_name) + 1:] # Strip off '-' before version. + + def version_of_wheel(filename, package_name): + # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file- + # name-convention) we know the format bits are '-' separated. + whl_package_name, version, _rest = filename.split('-', 2) + # Do the alteration to package_name from PEP 427: + our_package_name = re.sub(r'[^\w\d.]+', '_', package_name, re.UNICODE) + if whl_package_name != our_package_name: + give_up(filename, whl_package_name) + return version + + def give_up(filename, package_name): + raise RuntimeError("The archive '%s' didn't start with the package name " + "'%s', so I couldn't figure out the version number. " + "My bad; improve me." % + (filename, package_name)) + + get_version = (version_of_wheel + if self._downloaded_filename().endswith('.whl') + else version_of_archive) + return get_version(self._downloaded_filename(), self._project_name()) + + def _is_always_unsatisfied(self): + """Returns whether this requirement is always unsatisfied + + This would happen in cases where we can't determine the version + from the filename. + + """ + # If this is a github sha tarball, then it is always unsatisfied + # because the url has a commit sha in it and not the version + # number. + url = self._url() + if url: + filename = filename_from_url(url) + if filename.endswith(ARCHIVE_EXTENSIONS): + filename, ext = splitext(filename) + if is_git_sha(filename): + return True + return False + + @memoize # Avoid hitting the file[cache] over and over. + def _expected_hashes(self): + """Return a list of known-good hashes for this package.""" + return hashes_above(*path_and_line(self._req)) + + def _download(self, link): + """Download a file, and return its name within my temp dir. + + This does no verification of HTTPS certs, but our checking hashes + makes that largely unimportant. It would be nice to be able to use the + requests lib, which can verify certs, but it is guaranteed to be + available only in pip >= 1.5. + + This also drops support for proxies and basic auth, though those could + be added back in. + + """ + # Based on pip 1.4.1's URLOpener but with cert verification removed + def opener(is_https): + if is_https: + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + else: + opener = build_opener() + return opener + + # Descended from unpack_http_url() in pip 1.4.1 + def best_filename(link, response): + """Return the most informative possible filename for a download, + ideally with a proper extension. + + """ + content_type = response.info().get('content-type', '') + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess: + content_disposition = response.info().get('content-disposition') + if content_disposition: + type, params = cgi.parse_header(content_disposition) + # We use ``or`` here because we don't want to use an "empty" value + # from the filename param: + filename = params.get('filename') or filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != response.geturl(): + ext = splitext(response.geturl())[1] + if ext: + filename += ext + return filename + + # Descended from _download_url() in pip 1.4.1 + def pipe_to_file(response, path, size=0): + """Pull the data off an HTTP response, shove it in a new file, and + show progress. + + :arg response: A file-like object to read from + :arg path: The path of the new file + :arg size: The expected size, in bytes, of the download. 0 for + unknown or to suppress progress indication (as for cached + downloads) + + """ + def response_chunks(chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + print('Downloading %s%s...' % ( + self._req.req, + (' (%sK)' % (size / 1000)) if size > 1000 else '')) + progress_indicator = (DownloadProgressBar(max=size).iter if size + else DownloadProgressSpinner().iter) + with open(path, 'wb') as file: + for chunk in progress_indicator(response_chunks(4096), 4096): + file.write(chunk) + + url = link.url.split('#', 1)[0] + try: + response = opener(urlparse(url).scheme != 'http').open(url) + except (HTTPError, IOError) as exc: + raise DownloadError(link, exc) + filename = best_filename(link, response) + try: + size = int(response.headers['content-length']) + except (ValueError, KeyError, TypeError): + size = 0 + pipe_to_file(response, join(self._temp_path, filename), size=size) + return filename + + # Based on req_set.prepare_files() in pip bb2a8428d4aebc8d313d05d590f386fa3f0bbd0f + @memoize # Avoid re-downloading. + def _downloaded_filename(self): + """Download the package's archive if necessary, and return its + filename. + + --no-deps is implied, as we have reimplemented the bits that would + ordinarily do dependency resolution. + + """ + # Peep doesn't support requirements that don't come down as a single + # file, because it can't hash them. Thus, it doesn't support editable + # requirements, because pip itself doesn't support editable + # requirements except for "local projects or a VCS url". Nor does it + # support VCS requirements yet, because we haven't yet come up with a + # portable, deterministic way to hash them. In summary, all we support + # is == requirements and tarballs/zips/etc. + + # TODO: Stop on reqs that are editable or aren't ==. + + # If the requirement isn't already specified as a URL, get a URL + # from an index: + link = self._link() or self._finder.find_requirement(self._req, upgrade=False) + + if link: + lower_scheme = link.scheme.lower() # pip lower()s it for some reason. + if lower_scheme == 'http' or lower_scheme == 'https': + file_path = self._download(link) + return basename(file_path) + elif lower_scheme == 'file': + # The following is inspired by pip's unpack_file_url(): + link_path = url_to_path(link.url_without_fragment) + if isdir(link_path): + raise UnsupportedRequirementError( + "%s: %s is a directory. So that it can compute " + "a hash, peep supports only filesystem paths which " + "point to files" % + (self._req, link.url_without_fragment)) + else: + copy(link_path, self._temp_path) + return basename(link_path) + else: + raise UnsupportedRequirementError( + "%s: The download link, %s, would not result in a file " + "that can be hashed. Peep supports only == requirements, " + "file:// URLs pointing to files (not folders), and " + "http:// and https:// URLs pointing to tarballs, zips, " + "etc." % (self._req, link.url)) + else: + raise UnsupportedRequirementError( + "%s: couldn't determine where to download this requirement from." + % (self._req,)) + + def install(self): + """Install the package I represent, without dependencies. + + Obey typical pip-install options passed in on the command line. + + """ + other_args = list(requirement_args(self._argv, want_other=True)) + archive_path = join(self._temp_path, self._downloaded_filename()) + # -U so it installs whether pip deems the requirement "satisfied" or + # not. This is necessary for GitHub-sourced zips, which change without + # their version numbers changing. + run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path]) + + @memoize + def _actual_hash(self): + """Download the package's archive if necessary, and return its hash.""" + return hash_of_file(join(self._temp_path, self._downloaded_filename())) + + def _project_name(self): + """Return the inner Requirement's "unsafe name". + + Raise ValueError if there is no name. + + """ + name = getattr(self._req.req, 'project_name', '') + if name: + return name + raise ValueError('Requirement has no project_name.') + + def _name(self): + return self._req.name + + def _link(self): + try: + return self._req.link + except AttributeError: + # The link attribute isn't available prior to pip 6.1.0, so fall + # back to the now deprecated 'url' attribute. + return Link(self._req.url) if self._req.url else None + + def _url(self): + link = self._link() + return link.url if link else None + + @memoize # Avoid re-running expensive check_if_exists(). + def _is_satisfied(self): + self._req.check_if_exists() + return (self._req.satisfied_by and + not self._is_always_unsatisfied()) + + def _class(self): + """Return the class I should be, spanning a continuum of goodness.""" + try: + self._project_name() + except ValueError: + return MalformedReq + if self._is_satisfied(): + return SatisfiedReq + if not self._expected_hashes(): + return MissingReq + if self._actual_hash() not in self._expected_hashes(): + return MismatchedReq + return InstallableReq + + @classmethod + def foot(cls): + """Return the text to be printed once, after all of the errors from + classes of my type are printed. + + """ + return '' + + +class MalformedReq(DownloadedReq): + """A requirement whose package name could not be determined""" + + @classmethod + def head(cls): + return 'The following requirements could not be processed:\n' + + def error(self): + return '* Unable to determine package name from URL %s; add #egg=' % self._url() + + +class MissingReq(DownloadedReq): + """A requirement for which no hashes were specified in the requirements file""" + + @classmethod + def head(cls): + return ('The following packages had no hashes specified in the requirements file, which\n' + 'leaves them open to tampering. Vet these packages to your satisfaction, then\n' + 'add these "sha256" lines like so:\n\n') + + def error(self): + if self._url(): + # _url() always contains an #egg= part, or this would be a + # MalformedRequest. + line = self._url() + else: + line = '%s==%s' % (self._name(), self._version()) + return '# sha256: %s\n%s\n' % (self._actual_hash(), line) + + +class MismatchedReq(DownloadedReq): + """A requirement for which the downloaded file didn't match any of my hashes.""" + @classmethod + def head(cls): + return ("THE FOLLOWING PACKAGES DIDN'T MATCH THE HASHES SPECIFIED IN THE REQUIREMENTS\n" + "FILE. If you have updated the package versions, update the hashes. If not,\n" + "freak out, because someone has tampered with the packages.\n\n") + + def error(self): + preamble = ' %s: expected' % self._project_name() + if len(self._expected_hashes()) > 1: + preamble += ' one of' + padding = '\n' + ' ' * (len(preamble) + 1) + return '%s %s\n%s got %s' % (preamble, + padding.join(self._expected_hashes()), + ' ' * (len(preamble) - 4), + self._actual_hash()) + + @classmethod + def foot(cls): + return '\n' + + +class SatisfiedReq(DownloadedReq): + """A requirement which turned out to be already installed""" + + @classmethod + def head(cls): + return ("These packages were already installed, so we didn't need to download or build\n" + "them again. If you installed them with peep in the first place, you should be\n" + "safe. If not, uninstall them, then re-attempt your install with peep.\n") + + def error(self): + return ' %s' % (self._req,) + + +class InstallableReq(DownloadedReq): + """A requirement whose hash matched and can be safely installed""" + + +# DownloadedReq subclasses that indicate an error that should keep us from +# going forward with installation, in the order in which their errors should +# be reported: +ERROR_CLASSES = [MismatchedReq, MissingReq, MalformedReq] + + +def bucket(things, key): + """Return a map of key -> list of things.""" + ret = defaultdict(list) + for thing in things: + ret[key(thing)].append(thing) + return ret + + +def first_every_last(iterable, first, every, last): + """Execute something before the first item of iter, something else for each + item, and a third thing after the last. + + If there are no items in the iterable, don't execute anything. + + """ + did_first = False + for item in iterable: + if not did_first: + did_first = True + first(item) + every(item) + if did_first: + last(item) + + +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + +def downloaded_reqs_from_path(path, argv): + """Return a list of DownloadedReqs representing the requirements parsed + out of a given requirements file. + + :arg path: The path to the requirements file + :arg argv: The commandline args, starting after the subcommand + + """ + finder = package_finder(argv) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] + + +def peep_install(argv): + """Perform the ``peep install`` subcommand, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + output = [] + out = output.append + reqs = [] + try: + req_paths = list(requirement_args(argv, want_paths=True)) + if not req_paths: + out("You have to specify one or more requirements files with the -r option, because\n" + "otherwise there's nowhere for peep to look up the hashes.\n") + return COMMAND_LINE_ERROR + + # We're a "peep install" command, and we have some requirement paths. + reqs = list(chain.from_iterable( + downloaded_reqs_from_path(path, argv) + for path in req_paths)) + buckets = bucket(reqs, lambda r: r.__class__) + + # Skip a line after pip's "Cleaning up..." so the important stuff + # stands out: + if any(buckets[b] for b in ERROR_CLASSES): + out('\n') + + printers = (lambda r: out(r.head()), + lambda r: out(r.error() + '\n'), + lambda r: out(r.foot())) + for c in ERROR_CLASSES: + first_every_last(buckets[c], *printers) + + if any(buckets[b] for b in ERROR_CLASSES): + out('-------------------------------\n' + 'Not proceeding to installation.\n') + return SOMETHING_WENT_WRONG + else: + for req in buckets[InstallableReq]: + req.install() + + first_every_last(buckets[SatisfiedReq], *printers) + + return ITS_FINE_ITS_FINE + except (UnsupportedRequirementError, DownloadError) as exc: + out(str(exc)) + return SOMETHING_WENT_WRONG + finally: + for req in reqs: + req.dispose() + print(''.join(output)) + + +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + +def main(): + """Be the top-level entrypoint. Return a shell status code.""" + commands = {'hash': peep_hash, + 'install': peep_install, + 'port': peep_port} + try: + if len(argv) >= 2 and argv[1] in commands: + return commands[argv[1]](argv[2:]) + else: + # Fall through to top-level pip main() for everything else: + return pip.main() + except PipException as exc: + return exc.error_code + + +def exception_handler(exc_type, exc_value, exc_tb): + print('Oh no! Peep had a problem while trying to do stuff. Please write up a bug report') + print('with the specifics so we can fix it:') + print() + print('https://github.com/erikrose/peep/issues/new') + print() + print('Here are some particulars you can copy and paste into the bug report:') + print() + print('---') + print('peep:', repr(__version__)) + print('python:', repr(sys.version)) + print('pip:', repr(getattr(pip, '__version__', 'no __version__ attr'))) + print('Command line: ', repr(sys.argv)) + print( + ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))) + print('---') + + +if __name__ == '__main__': + try: + exit(main()) + except Exception: + exception_handler(*sys.exc_info()) + exit(SOMETHING_WENT_WRONG) + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + set +e + PEEP_OUT=`"$VENV_BIN/python" "$TEMP_DIR/peep.py" install -r "$TEMP_DIR/letsencrypt-auto-requirements.txt"` + PEEP_STATUS=$? + set -e + rm -rf "$TEMP_DIR" + if [ "$PEEP_STATUS" != 0 ]; then + # Report error. (Otherwise, be quiet.) + echo "Had a problem while downloading and verifying Python packages:" + echo $PEEP_OUT + exit 1 + fi + fi + echo "Running letsencrypt..." + echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" + $SUDO "$VENV_BIN/letsencrypt" "$@" +else + # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # letsencrypt-auto (which is always the same as that of the letsencrypt + # package). Phase 2 checks the version of the locally installed letsencrypt. + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$1" = "--os-packages-only" ]; then + echo "OS packages installed." + exit 0 + fi + + echo "Checking for new version..." + TEMP_DIR=$(TempDir) + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 + +On failure, return non-zero. + +""" +from distutils.version import LooseVersion +from json import loads +from os import devnull, environ +from os.path import dirname, join +import re +from subprocess import check_call, CalledProcessError +from sys import argv, exit +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + + +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe +4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B +2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww +s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T +QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE +33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP +rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 ++E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK +EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu +q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 +3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn +I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== +-----END PUBLIC KEY----- +""") # TODO: Replace with real one. + + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/letsencrypt/json'))) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp_dir): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. + + """ + le_auto_dir = environ.get( + 'LE_AUTO_DIR_TEMPLATE', + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'letsencrypt-auto/') % tag + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp_dir, 'public_key.pem'), + '-signature', + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + + +def main(): + get = HttpsGetter().get + flag = argv[1] + try: + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) + +UNLIKELY_EOF + # --------------------------------------------------------------------------- + DeterminePythonVersion + REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` + if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Replacing letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + rm -rf "$TEMP_DIR" + fi # should upgrade + "$0" --no-self-upgrade "$@" +fi From e6cece580d03753874acc863ea3eb3fdeceb24a1 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 15:39:34 -0500 Subject: [PATCH 058/201] Document le-auto env vars. --- letsencrypt_auto/tests/auto_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index c2b9fc378..c9c3d3c88 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -160,9 +160,12 @@ def run_le_auto(venv_dir, base_url): """ env = environ.copy() d = dict(XDG_DATA_HOME=venv_dir, + # URL to PyPI-style JSON that tell us the latest released version + # of LE: LE_AUTO_JSON_URL=base_url + 'letsencrypt/json', + # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: LE_AUTO_DIR_TEMPLATE=base_url + '%s/', - # The public key corresponding to signing_keys/test.key: + # The public key corresponding to signing.key: LE_AUTO_PUBLIC_KEY="""-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G From fa306259228b91863d0c1a55a61e108638e8292f Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 17:35:17 -0500 Subject: [PATCH 059/201] Update the built version of letsencrypt-auto. --- letsencrypt_auto/letsencrypt-auto | 65 ++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index 1038ffd04..54c2218e1 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -13,7 +13,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.1.0.dev0" +LE_AUTO_VERSION="0.2.0.dev0" ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name @@ -41,6 +41,7 @@ DeterminePythonVersion() { export LE_PYTHON=${LE_PYTHON:-python} else echo "Cannot find any Pythons... please install one!" + exit 1 fi PYVER=`"$LE_PYTHON" --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` @@ -78,26 +79,56 @@ BootstrapDebCommon() { # distro version (#346) virtualenv= - if apt-cache show virtualenv > /dev/null ; then + if apt-cache show virtualenv > /dev/null 2>&1; then virtualenv="virtualenv" fi - if apt-cache show python-virtualenv > /dev/null ; then + if apt-cache show python-virtualenv > /dev/null 2>&1; then virtualenv="$virtualenv python-virtualenv" fi + augeas_pkg=libaugeas0 + AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then + /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." + sleep 1s + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + + sudo sh -c 'echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list' + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 + augeas_pkg= + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs + fi + $SUDO apt-get install -y --no-install-recommends \ - git \ python \ python-dev \ $virtualenv \ gcc \ dialog \ - libaugeas0 \ + $augeas_pkg \ libssl-dev \ libffi-dev \ ca-certificates \ + + if ! command -v virtualenv > /dev/null ; then echo Failed to install a working \"virtualenv\" command, exiting exit 1 @@ -107,7 +138,7 @@ BootstrapDebCommon() { BootstrapRpmCommon() { # Tested with: # - Fedora 22, 23 (x64) - # - Centos 7 (x64: onD igitalOcean droplet) + # - Centos 7 (x64: on DigitalOcean droplet) if type dnf 2>/dev/null then @@ -138,9 +169,7 @@ BootstrapRpmCommon() { fi fi - # "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails) if ! $SUDO $tool install -y \ - git-core \ gcc \ dialog \ augeas-libs \ @@ -152,12 +181,20 @@ BootstrapRpmCommon() { echo "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi + + + if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then + if ! $SUDO $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi + fi } BootstrapSuseCommon() { # SLE12 don't have python-virtualenv - $SUDO zypper -nq in -l git-core \ + $SUDO zypper -nq in -l \ python \ python-devel \ python-virtualenv \ @@ -178,7 +215,6 @@ BootstrapArchCommon() { # ./bootstrap/dev/_common_venv.sh deps=" - git python2 python-virtualenv gcc @@ -198,7 +234,7 @@ BootstrapArchCommon() { } BootstrapGentooCommon() { - PACKAGES="dev-vcs/git + PACKAGES=" dev-lang/python:2.7 dev-python/virtualenv dev-util/dialog @@ -223,7 +259,6 @@ BootstrapGentooCommon() { BootstrapFreeBsd() { "$SUDO" pkg install -Ay \ - git \ python \ py27-virtualenv \ augeas \ @@ -325,13 +360,13 @@ if test "`id -u`" -ne "0" ; then args="" # This `while` loop iterates over all parameters given to this function. # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrap in a pair of `'`, then append to `$args` string + # will be wrapped in a pair of `'`, then appended to `$args` string # For example, `echo "It's only 1\$\!"` will be escaped to: # 'echo' 'It'"'"'s only 1$!' # │ │└┼┘│ # │ │ │ └── `'s only 1$!'` the literal string # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings followed it + # │ └── `'It'`, to be concatenated with the strings following it # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself while [ $# -ne 0 ]; do args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " @@ -1548,7 +1583,7 @@ UNLIKELY_EOF exit 1 fi fi - echo "Running letsencrypt..." + echo "Requesting root privileges to run letsencrypt..." echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" $SUDO "$VENV_BIN/letsencrypt" "$@" else From 5f694e338185720e4fc1986bf25ae32e3dbfa50f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 14:39:02 -0800 Subject: [PATCH 060/201] Create a fake release in a PyPI-style json file --- pypi.json | 504 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 pypi.json diff --git a/pypi.json b/pypi.json new file mode 100644 index 000000000..389d44790 --- /dev/null +++ b/pypi.json @@ -0,0 +1,504 @@ +{ + "info": { + "maintainer": "", + "docs_url": null, + "requires_python": "", + "maintainer_email": "", + "cheesecake_code_kwalitee_id": null, + "keywords": "", + "package_url": "http://pypi.python.org/pypi/letsencrypt", + "author": "Let's Encrypt Project", + "author_email": "client-dev@letsencrypt.org", + "download_url": "", + "platform": "UNKNOWN", + "version": "0.1.1", + "cheesecake_documentation_id": null, + "_pypi_hidden": false, + "description": ".. notice for github users\n\nDisclaimer\n==========\n\nThe Let's Encrypt Client is **BETA SOFTWARE**. It contains plenty of bugs and\nrough edges, and should be tested thoroughly in staging environments before use\non production systems.\n\nFor more information regarding the status of the project, please see\nhttps://letsencrypt.org. Be sure to checkout the\n`Frequently Asked Questions (FAQ) `_.\n\nAbout the Let's Encrypt Client\n==============================\n\nThe Let's Encrypt Client is a fully-featured, extensible client for the Let's\nEncrypt CA (or any other CA that speaks the `ACME\n`_\nprotocol) that can automate the tasks of obtaining certificates and\nconfiguring webservers to use them.\n\nInstallation\n------------\n\nIf ``letsencrypt`` is packaged for your OS, you can install it from there, and\nrun it by typing ``letsencrypt``. Because not all operating systems have\npackages yet, we provide a temporary solution via the ``letsencrypt-auto``\nwrapper script, which obtains some dependencies from your OS and puts others\nin a python virtual environment::\n\n user@webserver:~$ git clone https://github.com/letsencrypt/letsencrypt\n user@webserver:~$ cd letsencrypt\n user@webserver:~/letsencrypt$ ./letsencrypt-auto --help\n\nOr for full command line help, type::\n\n ./letsencrypt-auto --help all\n\n``letsencrypt-auto`` updates to the latest client release automatically. And\nsince ``letsencrypt-auto`` is a wrapper to ``letsencrypt``, it accepts exactly\nthe same command line flags and arguments. More details about this script and\nother installation methods can be found `in the User Guide\n`_.\n\nHow to run the client\n---------------------\n\nIn many cases, you can just run ``letsencrypt-auto`` or ``letsencrypt``, and the\nclient will guide you through the process of obtaining and installing certs\ninteractively.\n\nYou can also tell it exactly what you want it to do from the command line.\nFor instance, if you want to obtain a cert for ``thing.com``,\n``www.thing.com``, and ``otherthing.net``, using the Apache plugin to both\nobtain and install the certs, you could do this::\n\n ./letsencrypt-auto --apache -d thing.com -d www.thing.com -d otherthing.net\n\n(The first time you run the command, it will make an account, and ask for an\nemail and agreement to the Let's Encrypt Subscriber Agreement; you can\nautomate those with ``--email`` and ``--agree-tos``)\n\nIf you want to use a webserver that doesn't have full plugin support yet, you\ncan still use \"standalone\" or \"webroot\" plugins to obtain a certificate::\n\n ./letsencrypt-auto certonly --standalone --email admin@thing.com -d thing.com -d www.thing.com -d otherthing.net\n\n\nUnderstanding the client in more depth\n--------------------------------------\n\nTo understand what the client is doing in detail, it's important to\nunderstand the way it uses plugins. Please see the `explanation of\nplugins `_ in\nthe User Guide.\n\nLinks\n=====\n\nDocumentation: https://letsencrypt.readthedocs.org\n\nSoftware project: https://github.com/letsencrypt/letsencrypt\n\nNotes for developers: https://letsencrypt.readthedocs.org/en/latest/contributing.html\n\nMain Website: https://letsencrypt.org/\n\nIRC Channel: #letsencrypt on `Freenode`_\n\nCommunity: https://community.letsencrypt.org\n\nMailing list: `client-dev`_ (to subscribe without a Google account, send an\nemail to client-dev+subscribe@letsencrypt.org)\n\n|build-status| |coverage| |docs| |container|\n\n\n\n.. |build-status| image:: https://travis-ci.org/letsencrypt/letsencrypt.svg?branch=master\n :target: https://travis-ci.org/letsencrypt/letsencrypt\n :alt: Travis CI status\n\n.. |coverage| image:: https://coveralls.io/repos/letsencrypt/letsencrypt/badge.svg?branch=master\n :target: https://coveralls.io/r/letsencrypt/letsencrypt\n :alt: Coverage status\n\n.. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/\n :target: https://readthedocs.org/projects/letsencrypt/\n :alt: Documentation status\n\n.. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status\n :target: https://quay.io/repository/letsencrypt/letsencrypt\n :alt: Docker Repository on Quay.io\n\n.. _`installation instructions`:\n https://letsencrypt.readthedocs.org/en/latest/using.html\n\n.. _watch demo video: https://www.youtube.com/watch?v=Gas_sSB-5SU\n\nSystem Requirements\n===================\n\nThe Let's Encrypt Client presently only runs on Unix-ish OSes that include\nPython 2.6 or 2.7; Python 3.x support will be added after the Public Beta\nlaunch. The client requires root access in order to write to\n``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to\nbind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and\nmodify webserver configurations (if you use the ``apache`` or ``nginx``\nplugins). If none of these apply to you, it is theoretically possible to run\nwithout root privileges, but for most users who want to avoid running an ACME\nclient as root, either `letsencrypt-nosudo\n`_ or `simp_le\n`_ are more appropriate choices.\n\nThe Apache plugin currently requires a Debian-based OS with augeas version\n1.0; this includes Ubuntu 12.04+ and Debian 7+.\n\n\nCurrent Features\n================\n\n* Supports multiple web servers:\n\n - apache/2.x (working on Debian 8+ and Ubuntu 12.04+)\n - standalone (runs its own simple webserver to prove you control a domain)\n - webroot (adds files to webroot directories in order to prove control of\n domains and obtain certs)\n - nginx/0.8.48+ (highly experimental, not included in letsencrypt-auto)\n\n* The private key is generated locally on your system.\n* Can talk to the Let's Encrypt CA or optionally to other ACME\n compliant services.\n* Can get domain-validated (DV) certificates.\n* Can revoke certificates.\n* Adjustable RSA key bit-length (2048 (default), 4096, ...).\n* Can optionally install a http -> https redirect, so your site effectively\n runs https only (Apache only)\n* Fully automated.\n* Configuration changes are logged and can be reverted.\n* Supports ncurses and text (-t) UI, or can be driven entirely from the\n command line.\n* Free and Open Source Software, made with Python.\n\n\n.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt\n.. _client-dev: https://groups.google.com/a/letsencrypt.org/forum/#!forum/client-dev", + "release_url": "http://pypi.python.org/pypi/letsencrypt/0.1.1", + "downloads": { + "last_month": 95687, + "last_week": 19418, + "last_day": 2007 + }, + "_pypi_ordering": 14, + "requires_dist": [ + "ConfigArgParse", + "PyOpenSSL", + "acme (==0.1.1)", + "configobj", + "cryptography (>=0.7)", + "mock", + "parsedatetime", + "psutil (>=2.1.0)", + "pyrfc3339", + "python2-pythondialog (>=3.2.2rc1)", + "pytz", + "requests", + "setuptools", + "six", + "zope.component", + "zope.interface", + "astroid (==1.3.5); extra == 'dev'", + "pylint (==1.4.2); extra == 'dev'", + "twine; extra == 'dev'", + "wheel; extra == 'dev'", + "Sphinx (>=1.0); extra == 'docs'", + "repoze.sphinx.autointerface; extra == 'docs'", + "sphinx-rtd-theme; extra == 'docs'", + "sphinxcontrib-programoutput; extra == 'docs'", + "coverage; extra == 'testing'", + "nose; extra == 'testing'", + "nosexcover; extra == 'testing'", + "pep8; extra == 'testing'", + "tox; extra == 'testing'" + ], + "classifiers": [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Environment :: Console :: Curses", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Security", + "Topic :: System :: Installation/Setup", + "Topic :: System :: Networking", + "Topic :: System :: Systems Administration", + "Topic :: Utilities" + ], + "name": "letsencrypt", + "bugtrack_url": "", + "license": "Apache License 2.0", + "summary": "Let's Encrypt client", + "home_page": "https://github.com/letsencrypt/letsencrypt", + "cheesecake_installability_id": null + }, + "releases": { + "0.1.22": [ + { + "comment_text" : "fake release for 0.2.0 leauto testing" + } + ], + "0.0.0.dev20151108": [ + { + "has_sig": true, + "upload_time": "2015-11-08T07:54:45", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151108-py2-none-any.whl", + "md5_digest": "fb4eeb29ed528fca81037854c8c98e3c", + "downloads": 8896, + "filename": "letsencrypt-0.0.0.dev20151108-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 163047 + }, + { + "has_sig": true, + "upload_time": "2015-11-08T07:55:32", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151108.tar.gz", + "md5_digest": "0f7ab3da27f96c637a94092a431c571f", + "downloads": 801, + "filename": "letsencrypt-0.0.0.dev20151108.tar.gz", + "packagetype": "sdist", + "size": 154879 + } + ], + "0.0.0.dev20151123": [ + { + "has_sig": true, + "upload_time": "2015-11-23T21:48:35", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151123-py2-none-any.whl", + "md5_digest": "57dfc30fa49516789411346199f11d7a", + "downloads": 6708, + "filename": "letsencrypt-0.0.0.dev20151123-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 169232 + }, + { + "has_sig": true, + "upload_time": "2015-11-23T21:49:24", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151123.tar.gz", + "md5_digest": "d78867dcf88e0e6a4201d3d733e09d50", + "downloads": 715, + "filename": "letsencrypt-0.0.0.dev20151123.tar.gz", + "packagetype": "sdist", + "size": 159702 + } + ], + "0.0.0.dev20151114": [ + { + "has_sig": true, + "upload_time": "2015-11-14T12:24:20", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151114-py2-none-any.whl", + "md5_digest": "b570611a6e0e04c8082744a1509dcdde", + "downloads": 7945, + "filename": "letsencrypt-0.0.0.dev20151114-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 167306 + }, + { + "has_sig": true, + "upload_time": "2015-11-14T12:24:49", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151114.tar.gz", + "md5_digest": "c1c49f145bd32151b2ba991693d01467", + "downloads": 616, + "filename": "letsencrypt-0.0.0.dev20151114.tar.gz", + "packagetype": "sdist", + "size": 158894 + } + ], + "0.0.0.dev20151017": [ + { + "has_sig": true, + "upload_time": "2015-10-17T10:49:30", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151017-py2-none-any.whl", + "md5_digest": "bc43ffc9c8e5a4a73f15ca9587134538", + "downloads": 1414, + "filename": "letsencrypt-0.0.0.dev20151017-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 159823 + }, + { + "has_sig": true, + "upload_time": "2015-10-17T10:49:54", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151017.tar.gz", + "md5_digest": "0836e416e6acfd795af07f2c45f57153", + "downloads": 460, + "filename": "letsencrypt-0.0.0.dev20151017.tar.gz", + "packagetype": "sdist", + "size": 144361 + } + ], + "0.0.0.dev20151021": [ + { + "has_sig": true, + "upload_time": "2015-10-21T20:05:47", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151021-py2-none-any.whl", + "md5_digest": "c692b30e57862345383c3b0b415c4e0e", + "downloads": 1034, + "filename": "letsencrypt-0.0.0.dev20151021-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 161328 + }, + { + "has_sig": true, + "upload_time": "2015-10-21T20:06:24", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151021.tar.gz", + "md5_digest": "14cc485de72856a2413639674406d7f8", + "downloads": 361, + "filename": "letsencrypt-0.0.0.dev20151021.tar.gz", + "packagetype": "sdist", + "size": 145857 + } + ], + "0.0.0.dev20151020": [ + { + "has_sig": true, + "upload_time": "2015-10-20T21:51:35", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151020-py2-none-any.whl", + "md5_digest": "6f54a898ed2fb5a8cd36fb3278fc7827", + "downloads": 1027, + "filename": "letsencrypt-0.0.0.dev20151020-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 161290 + }, + { + "has_sig": true, + "upload_time": "2015-10-20T21:52:05", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151020.tar.gz", + "md5_digest": "f17fb4cb37175b80cfb52bb18d741b70", + "downloads": 377, + "filename": "letsencrypt-0.0.0.dev20151020.tar.gz", + "packagetype": "sdist", + "size": 145810 + } + ], + "0.0.0.dev20151030": [ + { + "has_sig": true, + "upload_time": "2015-10-30T07:59:24", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151030-py2-none-any.whl", + "md5_digest": "3e5f929e5bdab9f205fbdd55fa815369", + "downloads": 3815, + "filename": "letsencrypt-0.0.0.dev20151030-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 163194 + }, + { + "has_sig": true, + "upload_time": "2015-10-30T08:00:00", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151030.tar.gz", + "md5_digest": "eb4bd8c6eed5f80f22c9cfb8cf67bbd1", + "downloads": 603, + "filename": "letsencrypt-0.0.0.dev20151030.tar.gz", + "packagetype": "sdist", + "size": 151205 + } + ], + "0.0.0.dev20151008": [ + { + "has_sig": true, + "upload_time": "2015-10-08T19:40:21", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151008-py2-none-any.whl", + "md5_digest": "8e4fde8cbbb64b39bf5daec22eb30a19", + "downloads": 952, + "filename": "letsencrypt-0.0.0.dev20151008-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 159536 + }, + { + "has_sig": true, + "upload_time": "2015-10-08T19:40:46", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151008.tar.gz", + "md5_digest": "c3191f8add4dc1b7f51ff6477b719f5c", + "downloads": 413, + "filename": "letsencrypt-0.0.0.dev20151008.tar.gz", + "packagetype": "sdist", + "size": 145734 + } + ], + "0.0.0.dev20151006": [ + { + "has_sig": true, + "upload_time": "2015-10-06T06:48:16", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151006-py2-none-any.whl", + "md5_digest": "b0b4f25f23a68a561379ea407d3b332d", + "downloads": 403, + "filename": "letsencrypt-0.0.0.dev20151006-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 159892 + }, + { + "has_sig": true, + "upload_time": "2015-10-06T06:57:21", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151006.tar.gz", + "md5_digest": "b830f757206d668d7892f23a7c934108", + "downloads": 404, + "filename": "letsencrypt-0.0.0.dev20151006.tar.gz", + "packagetype": "sdist", + "size": 131184 + } + ], + "0.0.0.dev20151104": [ + { + "has_sig": true, + "upload_time": "2015-11-04T07:44:29", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151104-py2-none-any.whl", + "md5_digest": "897b3fbadd5c5966921e690fae0bfd83", + "downloads": 6601, + "filename": "letsencrypt-0.0.0.dev20151104-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 163690 + }, + { + "has_sig": true, + "upload_time": "2015-11-04T07:45:00", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151104.tar.gz", + "md5_digest": "41f2b5d8a99f19a46a98f1cd2f574e84", + "downloads": 793, + "filename": "letsencrypt-0.0.0.dev20151104.tar.gz", + "packagetype": "sdist", + "size": 154943 + } + ], + "0.0.0.dev20151107": [ + { + "has_sig": true, + "upload_time": "2015-11-07T11:49:26", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151107-py2-none-any.whl", + "md5_digest": "b33f11f9601c5d2e90b4feec37e3d406", + "downloads": 915, + "filename": "letsencrypt-0.0.0.dev20151107-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 164010 + }, + { + "has_sig": true, + "upload_time": "2015-11-07T11:50:17", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151107.tar.gz", + "md5_digest": "bbc8d651da6e12b2fa8fa74b42d8ce7c", + "downloads": 409, + "filename": "letsencrypt-0.0.0.dev20151107.tar.gz", + "packagetype": "sdist", + "size": 155434 + } + ], + "0.0.0.dev20151201": [ + { + "has_sig": true, + "upload_time": "2015-12-02T04:22:14", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151201-py2-none-any.whl", + "md5_digest": "850a340e5099cd6d0f1a50320f66c7dd", + "downloads": 1693, + "filename": "letsencrypt-0.0.0.dev20151201-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 173130 + }, + { + "has_sig": true, + "upload_time": "2015-12-02T04:22:41", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151201.tar.gz", + "md5_digest": "7faee02c89c9d7a732d8e6a7911a7f67", + "downloads": 252, + "filename": "letsencrypt-0.0.0.dev20151201.tar.gz", + "packagetype": "sdist", + "size": 164577 + } + ], + "0.0.0.dev20151024": [ + { + "has_sig": true, + "upload_time": "2015-10-24T17:57:09", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.0.0.dev20151024-py2-none-any.whl", + "md5_digest": "f36722b0181c2024d1a7c2d5714a8c7d", + "downloads": 3422, + "filename": "letsencrypt-0.0.0.dev20151024-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 161792 + }, + { + "has_sig": true, + "upload_time": "2015-10-24T17:57:35", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.0.0.dev20151024.tar.gz", + "md5_digest": "86b8ffc407af81b66a69d1dad29c0705", + "downloads": 473, + "filename": "letsencrypt-0.0.0.dev20151024.tar.gz", + "packagetype": "sdist", + "size": 147776 + } + ], + "0.1.0": [ + { + "has_sig": true, + "upload_time": "2015-12-03T07:49:17", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.1.0-py2-none-any.whl", + "md5_digest": "0c84b4c0f426714256ddb5349e670f6d", + "downloads": 51458, + "filename": "letsencrypt-0.1.0-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 173233 + }, + { + "has_sig": true, + "upload_time": "2015-12-03T07:49:22", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.1.0.tar.gz", + "md5_digest": "b3bc1f8a49e4953771d2fa77bf0fe9d2", + "downloads": 1426, + "filename": "letsencrypt-0.1.0.tar.gz", + "packagetype": "sdist", + "size": 167375 + } + ], + "0.1.1": [ + { + "has_sig": true, + "upload_time": "2015-12-16T00:54:48", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.1.1-py2-none-any.whl", + "md5_digest": "07218b6e5792a3de089891f1227c6ac2", + "downloads": 48253, + "filename": "letsencrypt-0.1.1-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 175923 + }, + { + "has_sig": true, + "upload_time": "2015-12-16T00:55:37", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.1.1.tar.gz", + "md5_digest": "ddd393563f3d5835d678b7478148e3d9", + "downloads": 801, + "filename": "letsencrypt-0.1.1.tar.gz", + "packagetype": "sdist", + "size": 166683 + } + ] + }, + "urls": [ + { + "has_sig": true, + "upload_time": "2015-12-16T00:54:48", + "comment_text": "", + "python_version": "py2", + "url": "https://pypi.python.org/packages/py2/l/letsencrypt/letsencrypt-0.1.1-py2-none-any.whl", + "md5_digest": "07218b6e5792a3de089891f1227c6ac2", + "downloads": 48253, + "filename": "letsencrypt-0.1.1-py2-none-any.whl", + "packagetype": "bdist_wheel", + "size": 175923 + }, + { + "has_sig": true, + "upload_time": "2015-12-16T00:55:37", + "comment_text": "", + "python_version": "source", + "url": "https://pypi.python.org/packages/source/l/letsencrypt/letsencrypt-0.1.1.tar.gz", + "md5_digest": "ddd393563f3d5835d678b7478148e3d9", + "downloads": 801, + "filename": "letsencrypt-0.1.1.tar.gz", + "packagetype": "sdist", + "size": 166683 + } + ] +} From 7d182c210d92078de9328e906020d16f4c08e2bb Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 17:53:36 -0500 Subject: [PATCH 061/201] Substitute test-only values for the env vars. To come: a test-only public key for letsencrypt-auto.sig. --- letsencrypt_auto/letsencrypt-auto.template | 4 ++-- letsencrypt_auto/pieces/fetch.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index b3d708ede..bb48b39fe 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -9,7 +9,7 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share-test} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin @@ -236,7 +236,7 @@ UNLIKELY_EOF # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. - "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "letsencrypt-auto-release-testing-v$REMOTE_VERSION" # Install new copy of letsencrypt-auto. This preserves permissions and # ownership from the old copy. diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index d094e6347..95f5d612a 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -75,7 +75,7 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/letsencrypt/json'))) + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/letsencrypt-auto-release-testing/pypi.json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most From 726376f288c9fadce7e83cec76792afeb011d471 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 14:57:18 -0800 Subject: [PATCH 062/201] A real release signture key --- letsencrypt_auto/pieces/fetch.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index d094e6347..323001ac3 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -21,20 +21,15 @@ from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe -4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B -2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww -s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T -QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE -33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP -rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 -+E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK -EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu -q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 -3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn -I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB -----END PUBLIC KEY----- -""") # TODO: Replace with real one. +""") class ExpectedError(Exception): From d0bbe447575ea7e57abc27a20f4cac572d6bb322 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 15:37:07 -0800 Subject: [PATCH 063/201] Switch to a throwaway testing key --- letsencrypt_auto/pieces/fetch.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index b9c4e5ab9..008fe75e1 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -21,13 +21,13 @@ from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvWrG8oyI2FlCWEEEo1+Q ++VmDgUdMKGWlThHm5oM6XODDpllY8gUGWoYn//jCUMQuQmDTtvPz1V6s5uoESnG3 +PUjEj539Dt3bQmfm5eRN17DXb3FR4l4eKkYE/bDHvGvWsI3B1b2ek0mK88XEoUxg +hx7tre19X8Q5N4ssii1+HW51e6NHO6S2fa7mko85RcF0ZHSvOVwMELbVYg+GVlmz +5K39QNqcBr2RcWmTR9XpRkV6F7DPm4XsSKd51McHiatG4vCzMpMw5R96aY4Y/wg+ +lLMOJYYAQEv7ii8exClMCTUiTzVevI0mSXyHxRcILFNRYrgc5OBNtf1w2ZjcHKAr +9QIDAQAB -----END PUBLIC KEY----- """) From b2a0142e07a5915c219379ed3bf93dfab4556f83 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 15:40:23 -0800 Subject: [PATCH 064/201] First attempt at signing with a throwaway key --- letsencrypt_auto/letsencrypt-auto | 27 +++++++++++--------------- letsencrypt_auto/letsencrypt-auto.sig | Bin 0 -> 256 bytes 2 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 letsencrypt_auto/letsencrypt-auto.sig diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index 54c2218e1..d7a9addab 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -9,7 +9,7 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share-test} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin @@ -1630,20 +1630,15 @@ from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe -4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B -2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww -s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T -QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE -33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP -rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 -+E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK -EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu -q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 -3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn -I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvWrG8oyI2FlCWEEEo1+Q ++VmDgUdMKGWlThHm5oM6XODDpllY8gUGWoYn//jCUMQuQmDTtvPz1V6s5uoESnG3 +PUjEj539Dt3bQmfm5eRN17DXb3FR4l4eKkYE/bDHvGvWsI3B1b2ek0mK88XEoUxg +hx7tre19X8Q5N4ssii1+HW51e6NHO6S2fa7mko85RcF0ZHSvOVwMELbVYg+GVlmz +5K39QNqcBr2RcWmTR9XpRkV6F7DPm4XsSKd51McHiatG4vCzMpMw5R96aY4Y/wg+ +lLMOJYYAQEv7ii8exClMCTUiTzVevI0mSXyHxRcILFNRYrgc5OBNtf1w2ZjcHKAr +9QIDAQAB -----END PUBLIC KEY----- -""") # TODO: Replace with real one. +""") class ExpectedError(Exception): @@ -1684,7 +1679,7 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/letsencrypt/json'))) + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/letsencrypt-auto-release-testing/pypi.json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most @@ -1751,7 +1746,7 @@ UNLIKELY_EOF # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. - "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "letsencrypt-auto-release-testing-v$REMOTE_VERSION" # Install new copy of letsencrypt-auto. This preserves permissions and # ownership from the old copy. diff --git a/letsencrypt_auto/letsencrypt-auto.sig b/letsencrypt_auto/letsencrypt-auto.sig new file mode 100644 index 0000000000000000000000000000000000000000..11d69c5280da3a5ecfd0dfd14435a419be7f0414 GIT binary patch literal 256 zcmV+b0ssEIdGBhO%iflh@lbEelBKG*O3II%@y(P`&#+lk-{HfXUV9Zkc+o*wt>`K= zn3rBPOKSLV3In>d+rpa+zB#wO6*dUJ%H9fQ>Z(#1Q;EI@$jUn?KOv;H*fH|k^zc>5 z&Gf=}hD5UqC8iLs=#8v`4aN0mIn#j;&o51dj%ZpLr{|^lEC*Z#vN^)=3>aSlG_@388Gz1KBB1k!uB_5c_+imRmC*EYu`seq zH+@x$v0hO1tDDw-VBTHIf3)+3cX2ro-M6qmt Date: Tue, 5 Jan 2016 18:47:23 -0500 Subject: [PATCH 065/201] Change version of le-auto script to the one published in the pypi.json. --- letsencrypt_auto/letsencrypt-auto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index d7a9addab..b285cbfe2 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -13,7 +13,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share-test} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.2.0.dev0" +LE_AUTO_VERSION="0.1.22" ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name From 0f787533c38dfab13a5bb45d6eb5af163438d8c9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 18:50:43 -0500 Subject: [PATCH 066/201] Swap _ for - so the phase-1 upgrade doesn't 404. --- letsencrypt_auto/letsencrypt-auto | 2 +- letsencrypt_auto/pieces/fetch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index b285cbfe2..88003ff6e 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -1699,7 +1699,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt-auto/') % tag + 'letsencrypt_auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index 008fe75e1..6f2570890 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -90,7 +90,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt-auto/') % tag + 'letsencrypt_auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') From 1ad21f9d1a12633af387df9924b9de1b3c73f1d7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 15:51:49 -0800 Subject: [PATCH 067/201] Update signature! --- letsencrypt_auto/letsencrypt-auto.sig | Bin 256 -> 256 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.sig b/letsencrypt_auto/letsencrypt-auto.sig index 11d69c5280da3a5ecfd0dfd14435a419be7f0414..cc62255bfb7a8633be2a80e56556fcaf497c2afa 100644 GIT binary patch literal 256 zcmV+b0ssC`8K^$&B2%Ucl~OL5THm-j69R&1ZDD#CXk?ut3ur~{b)d{H&OQViq2OIT zs}5e=18nU8V`zDCNmxw(wL6^+affVwBd9a+L%+Nk8Ecj)-2loWhUKABpQO5=KysB);q~l_6ja!|(~vq$3iUEqrKR=QE^+EiB1`N*Ls>hiD}>75-kbcr;g?Q5 z2aH1@*_*b_mw3fUnxf~Hbkwgc9ugMiqq%|d&b$1*mNJY2=C))jj*DZ@Dds`t@>x7* z9jz??H)rfwOY;iZb+Sc+z_-3{3v2N6<`I~W7?350Or%a;C%wmeI=wN3HX6rmJY#La Ga$+s2KYK#} literal 256 zcmV+b0ssEIdGBhO%iflh@lbEelBKG*O3II%@y(P`&#+lk-{HfXUV9Zkc+o*wt>`K= zn3rBPOKSLV3In>d+rpa+zB#wO6*dUJ%H9fQ>Z(#1Q;EI@$jUn?KOv;H*fH|k^zc>5 z&Gf=}hD5UqC8iLs=#8v`4aN0mIn#j;&o51dj%ZpLr{|^lEC*Z#vN^)=3>aSlG_@388Gz1KBB1k!uB_5c_+imRmC*EYu`seq zH+@x$v0hO1tDDw-VBTHIf3)+3cX2ro-M6qmt Date: Tue, 5 Jan 2016 16:17:05 -0800 Subject: [PATCH 068/201] Survive unsuccessful apt-get update... --- letsencrypt_auto/pieces/bootstrappers/deb_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh index 0ed3b6f79..d581ac2eb 100644 --- a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh @@ -17,7 +17,7 @@ BootstrapDebCommon() { # # - Debian 6.0.10 "squeeze" (x64) - $SUDO apt-get update + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) From f4011cc5c962dbd199c6b8460ce2d2a78b1701d7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 16:25:48 -0800 Subject: [PATCH 069/201] Move sudo to the top (So that people who want to audit can see it quickly) --- letsencrypt_auto/letsencrypt-auto.template | 72 +++++++++++----------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index bb48b39fe..5334e1f26 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -15,6 +15,42 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then @@ -120,42 +156,6 @@ for arg in "$@" ; do fi done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - if [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. From 8bb0631ab6b869d7007248e1722f5efc1b7cd50b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 Jan 2016 19:25:54 -0500 Subject: [PATCH 070/201] Updated versions --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index ba2c88394..77d42e44f 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.0.1.22' install_requires = [ # load_pem_private/public_key (>=0.6) diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 58008e1e4..a0f78170f 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.1.22' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 1d42fe488..226b3c317 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.1.22' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 1c7815f78..46a18db71 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.0.dev0' +__version__ = '0.1.22' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index d487e556d..6ec8bb3a7 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.1.22' install_requires = [ 'setuptools', # pkg_resources From b00877059f81d72f1165172bf626c65f1561733e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 16:26:56 -0800 Subject: [PATCH 071/201] Resign --- letsencrypt_auto/letsencrypt-auto | 74 +++++++++++++------------- letsencrypt_auto/letsencrypt-auto.sig | Bin 256 -> 256 bytes 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index 88003ff6e..a6c0b3f7a 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -15,6 +15,42 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="0.1.22" +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then @@ -73,7 +109,7 @@ BootstrapDebCommon() { # # - Debian 6.0.10 "squeeze" (x64) - $SUDO apt-get update + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -344,42 +380,6 @@ for arg in "$@" ; do fi done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - if [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. diff --git a/letsencrypt_auto/letsencrypt-auto.sig b/letsencrypt_auto/letsencrypt-auto.sig index cc62255bfb7a8633be2a80e56556fcaf497c2afa..1338334cade86a7bfa2f50198264bf8befbe12d6 100644 GIT binary patch literal 256 zcmV+b0ssDbF_WTw3+=Mlrrx^&i=8f!v96O{2u^m_&E>&@Z~^+++Cc!kI1Gu&(zLg| z@9?at5I~qoDZNPQ2L)Zl#~ORFnXvKGhyY#$bFlAsn7CAP zTJHBPS@7FC@wKnaR%$WZ(I?3&cNUe5|J7j*!*yB3M(n}s4G_#x G?O$r9F@>T4 literal 256 zcmV+b0ssC`8K^$&B2%Ucl~OL5THm-j69R&1ZDD#CXk?ut3ur~{b)d{H&OQViq2OIT zs}5e=18nU8V`zDCNmxw(wL6^+affVwBd9a+L%+Nk8Ecj)-2loWhUKABpQO5=KysB);q~l_6ja!|(~vq$3iUEqrKR=QE^+EiB1`N*Ls>hiD}>75-kbcr;g?Q5 z2aH1@*_*b_mw3fUnxf~Hbkwgc9ugMiqq%|d&b$1*mNJY2=C))jj*DZ@Dds`t@>x7* z9jz??H)rfwOY;iZb+Sc+z_-3{3v2N6<`I~W7?350Or%a;C%wmeI=wN3HX6rmJY#La Ga$+s2KYK#} From 0a122cbf4cdbea40d51798d7a947b8372db2a7cb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 18:05:10 -0800 Subject: [PATCH 072/201] Try baking in this 0.1.22 thing --- letsencrypt_auto/letsencrypt-auto | 2 ++ letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index a6c0b3f7a..f11893561 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -410,6 +410,8 @@ if [ "$1" = "--no-self-upgrade" ]; then # this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, # `pip freeze`, and then gather the hashes. +-i https://isnot.org/pip/0.1.22/ + # sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 70b005d2d..7097186e0 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -2,6 +2,8 @@ # this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, # `pip freeze`, and then gather the hashes. +-i https://isnot.org/pip/0.1.22/ + # sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 From 404de8429a162c965a96074951533bb9d2b43465 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 12:38:02 -0500 Subject: [PATCH 073/201] Update le-auto requirements file to fake LE 0.1.22 release. --- .../pieces/letsencrypt-auto-requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 7097186e0..ea1a8da90 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -188,14 +188,14 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 -# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo -acme==0.1.0 +# sha256: 37nVTMxyDwYMA4A1RSG63_uXw9hlsfHIWs2PnoC8OQc +# sha256: 43jSqZb6YepyCV-ULKAn7AvZnnAaZ8f-TsU2gg4gVwM +acme==0.1.22 -# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 -# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 -letsencrypt==0.1.0 +# sha256: MgQM63Brm18vm4UEdMUbZV6JToRspxRUUhVn7OtgY1Y +# sha256: f-fS-LnLY_S-XW8oMqFZCKSgtBDHRAllLx9kwhYWsrE +letsencrypt==0.1.22 -# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU -# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI -letsencrypt-apache==0.1.0 +# sha256: eAm3upSfC7PSRZj_PELZ_vF9UO5ATZilb8iShqD1M40 +# sha256: 0nS2F0GCtbOXx-EIWz-A5RkMoVODpO8iVMASgm0WVBc +letsencrypt-apache==0.1.22 From 484b0321ae4547d6415c49c4b5b5ddeeb18624b2 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 12:44:00 -0500 Subject: [PATCH 074/201] Add hashes for new cffi 1.3.1 packages. New ones for OS X 10.6 were released on 2015-12-16. --- letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index ea1a8da90..9fd70ee0d 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -8,15 +8,19 @@ # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 # sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 # sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY # sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc # sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM # sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U # sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis # sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U # sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU # sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M # sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y # sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA # sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs # sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA From d83dda815c6169e101ea7c54ed05a250e6a8ec12 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 13:07:47 -0500 Subject: [PATCH 075/201] Rebuild and re-sign le-auto. --- letsencrypt_auto/letsencrypt-auto | 22 +++++++++++++--------- letsencrypt_auto/letsencrypt-auto.sig | Bin 256 -> 256 bytes 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index f11893561..ad5c4acad 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -416,15 +416,19 @@ if [ "$1" = "--no-self-upgrade" ]; then # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 # sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 # sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY # sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc # sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM # sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U # sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis # sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U # sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU # sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M # sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y # sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA # sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs # sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA @@ -596,17 +600,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 -# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo -acme==0.1.0 +# sha256: 37nVTMxyDwYMA4A1RSG63_uXw9hlsfHIWs2PnoC8OQc +# sha256: 43jSqZb6YepyCV-ULKAn7AvZnnAaZ8f-TsU2gg4gVwM +acme==0.1.22 -# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 -# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 -letsencrypt==0.1.0 +# sha256: MgQM63Brm18vm4UEdMUbZV6JToRspxRUUhVn7OtgY1Y +# sha256: f-fS-LnLY_S-XW8oMqFZCKSgtBDHRAllLx9kwhYWsrE +letsencrypt==0.1.22 -# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU -# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI -letsencrypt-apache==0.1.0 +# sha256: eAm3upSfC7PSRZj_PELZ_vF9UO5ATZilb8iShqD1M40 +# sha256: 0nS2F0GCtbOXx-EIWz-A5RkMoVODpO8iVMASgm0WVBc +letsencrypt-apache==0.1.22 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt_auto/letsencrypt-auto.sig b/letsencrypt_auto/letsencrypt-auto.sig index 1338334cade86a7bfa2f50198264bf8befbe12d6..f9c2da60e0a1320d20d7ffd700d9e6820ed6e344 100644 GIT binary patch literal 256 zcmV+b0ssDCv95rBODHuSYvW3ZDw^)(|!>L zMQXw);%dHNR3sNHhYE5|c3eU*z3xN5rcZpM#fytD3}{B`NR`V|MX@eMmW(lb>Inv7 z1S>p?^mSVC+`=WCv#9sXTF-d-;&i9J;=J~%rs4l>MBrwj8?Kgttw61rIgrVppdcl( zGU|QH$5|q(3OT`t+f4qSuOE@*T2qpR5?N<`32D7iK<9I7_}U-u1-@;Rm zz>&@Z~^+++Cc!kI1Gu&(zLg| z@9?at5I~qoDZNPQ2L)Zl#~ORFnXvKGhyY#$bFlAsn7CAP zTJHBPS@7FC@wKnaR%$WZ(I?3&cNUe5|J7j*!*yB3M(n}s4G_#x G?O$r9F@>T4 From 275d3b4c689e02103ca9ec37642d9e59fcaa85f6 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 5 Jan 2016 18:50:43 -0500 Subject: [PATCH 076/201] Swap _ for - so the phase-1 upgrade doesn't 404. --- letsencrypt_auto/letsencrypt-auto | 2 +- letsencrypt_auto/pieces/fetch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index 54c2218e1..d1b974bbf 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -1704,7 +1704,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt-auto/') % tag + 'letsencrypt_auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt_auto/pieces/fetch.py index d094e6347..2ba296ca6 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt_auto/pieces/fetch.py @@ -95,7 +95,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt-auto/') % tag + 'letsencrypt_auto/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') From 5aa9fe9e7f153830455ec95f29bc2ed84664ea4b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 16:17:05 -0800 Subject: [PATCH 077/201] Survive unsuccessful apt-get update... --- letsencrypt_auto/pieces/bootstrappers/deb_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh index 0ed3b6f79..d581ac2eb 100644 --- a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt_auto/pieces/bootstrappers/deb_common.sh @@ -17,7 +17,7 @@ BootstrapDebCommon() { # # - Debian 6.0.10 "squeeze" (x64) - $SUDO apt-get update + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) From 8a3bbf97e0056192e8da11fe18fff14730d1a650 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 5 Jan 2016 16:25:48 -0800 Subject: [PATCH 078/201] Move sudo to the top (So that people who want to audit can see it quickly) --- letsencrypt_auto/letsencrypt-auto.template | 72 +++++++++++----------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index b3d708ede..f170fb45e 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -15,6 +15,42 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then @@ -120,42 +156,6 @@ for arg in "$@" ; do fi done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - if [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. From 4940ee2fcf7099e18e4e0b2adebe7ed8b03586b5 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 12:44:00 -0500 Subject: [PATCH 079/201] Add hashes for new cffi 1.3.1 packages. New ones for OS X 10.6 were released on 2015-12-16. --- letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 70b005d2d..9fe996008 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -6,15 +6,19 @@ # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 # sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 # sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY # sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc # sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM # sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U # sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis # sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U # sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU # sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M # sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y # sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA # sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs # sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA From ba6bf45753693f9f7590c0e3cff9b6c42704c37a Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 16:29:51 -0500 Subject: [PATCH 080/201] Update pinning of LE packages to 0.1.1. If we keep these at the latest release, something sane should happen if someone runs le-auto from master. --- .../pieces/letsencrypt-auto-requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 9fe996008..6be690043 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -190,14 +190,14 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 -# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo -acme==0.1.0 +# sha256: xlKw86fsP3VtbdljTTTboXgIw1QFHdCR1p8ff_zKnQ0 +# sha256: HCCDzax5ts-yAZDWhpMiQDUQS0bSXcnFLRerqZB_YPs +acme==0.1.1 -# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 -# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 -letsencrypt==0.1.0 +# sha256: F6dWBwnljkI2IASGO4VwCV05Nc3t0w8U0kWZsllpC8s +# sha256: jmeEAx7qchLhKOF75RqPuOSH2Wk6XficiL5TYyYfKs4 +letsencrypt==0.1.1 -# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU -# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI -letsencrypt-apache==0.1.0 +# sha256: 06zn8Fstsyfw58RmLaaODObDzR2jfZVCor0Hc65LYus +# sha256: 8ZO1CwBNE8eunQXAoXKVoGiaeAmj2BkRTMy5tXrBuYY +letsencrypt-apache==0.1.1 From 762709aa534d3701d8024d98e9337f3d83820ae7 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 17:10:44 -0500 Subject: [PATCH 081/201] Remove needless message about reusing venv. Rebuild le-auto. --- letsencrypt_auto/letsencrypt-auto | 100 +++++++++++---------- letsencrypt_auto/letsencrypt-auto.template | 4 +- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index d1b974bbf..452fb13b4 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -15,6 +15,42 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="0.2.0.dev0" +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then @@ -73,7 +109,7 @@ BootstrapDebCommon() { # # - Debian 6.0.10 "squeeze" (x64) - $SUDO apt-get update + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -344,42 +380,6 @@ for arg in "$@" ; do fi done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - if [ "$1" = "--no-self-upgrade" ]; then # Phase 2: Create venv, install LE, and run. @@ -389,9 +389,7 @@ if [ "$1" = "--no-self-upgrade" ]; then else INSTALLED_VERSION="none" fi - if [ "$LE_AUTO_VERSION" = "$INSTALLED_VERSION" ]; then - echo "Reusing old virtual environment." - else + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then echo "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" @@ -414,15 +412,19 @@ if [ "$1" = "--no-self-upgrade" ]; then # sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo # sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 # sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc +# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 # sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY # sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc # sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg +# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM # sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U # sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis # sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 +# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U # sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU # sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M # sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 +# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y # sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA # sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs # sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA @@ -594,17 +596,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: 9yQWjdAK9JWFHcodsdFotAiYqXVC1coj3rChq7kDlg8 -# sha256: w-u_7NH3h-9uMJ3cxBLGXQ7qMY0A0K_2EL23_wmoQDo -acme==0.1.0 +# sha256: xlKw86fsP3VtbdljTTTboXgIw1QFHdCR1p8ff_zKnQ0 +# sha256: HCCDzax5ts-yAZDWhpMiQDUQS0bSXcnFLRerqZB_YPs +acme==0.1.1 -# sha256: etdo3FQXbmpBSn60YkfKItjU2isypbo-N854YkyIhG8 -# sha256: gfYnYXbSVLfPX5W1ZHALxx8SsRA01AIDicpMMX43af0 -letsencrypt==0.1.0 +# sha256: F6dWBwnljkI2IASGO4VwCV05Nc3t0w8U0kWZsllpC8s +# sha256: jmeEAx7qchLhKOF75RqPuOSH2Wk6XficiL5TYyYfKs4 +letsencrypt==0.1.1 -# sha256: k8qUreGlW03BJz1FP-51brlSEef7QC8rGrljroQTZkU -# sha256: QvYP9hJhs-I_BG9SSma7vX115a_TET4qOWommmJC0lI -letsencrypt-apache==0.1.0 +# sha256: 06zn8Fstsyfw58RmLaaODObDzR2jfZVCor0Hc65LYus +# sha256: 8ZO1CwBNE8eunQXAoXKVoGiaeAmj2BkRTMy5tXrBuYY +letsencrypt-apache==0.1.1 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index f170fb45e..6e7cf784a 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -165,9 +165,7 @@ if [ "$1" = "--no-self-upgrade" ]; then else INSTALLED_VERSION="none" fi - if [ "$LE_AUTO_VERSION" = "$INSTALLED_VERSION" ]; then - echo "Reusing old virtual environment." - else + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then echo "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" From 4b075df871a22b4dd014b8329b830f82bf91afa5 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 6 Jan 2016 21:40:10 -0500 Subject: [PATCH 082/201] Cut down mock PyPI dir listing HTML. --- letsencrypt_auto/tests/auto_test.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index c9c3d3c88..814c37a30 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -220,17 +220,7 @@ class AutoTests(TestCase): with ephemeral_dir() as venv_dir: # This serves a PyPI page with a higher version, a GitHub-alike # with a corresponding le-auto script, and a matching signature. - resources = {'': """ - Directory listing for / - -

Directory listing for /

-
- -
- - """, # TODO: Cut this down. + resources = {'': 'letsencrypt/', 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), 'v99.9.9/letsencrypt-auto': NEW_LE_AUTO, 'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG} From fea4b24fb85c3afccaf8d9add091cc752522e4ff Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 20:14:51 +0000 Subject: [PATCH 083/201] Add test to discover "global" max_attempt bug (#1719) --- acme/acme/client_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 58f55b293..9a32303e1 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -310,7 +310,10 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime) + csr, authzrs, mintime=mintime, + # make sure that max_attempts is per-authorization, rather + # than global + max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') From 4d04d14b2038d0364c26e49164eff6cf7624a593 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 20:25:07 +0000 Subject: [PATCH 084/201] Fix "global" max_attempt bug (#1719) --- acme/acme/client.py | 30 ++++++++++++++++++++---------- acme/acme/errors.py | 17 ++++++++--------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3e28ef47..9970e2046 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,4 +1,5 @@ """ACME client API.""" +import collections import datetime import heapq import logging @@ -336,8 +337,9 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts before - `PollError` with non-empty ``waiting`` is raised. + :param int max_attempts: Maximum number of attempts (per + authorization) before `PollError` with non-empty ``waiting`` + is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -351,6 +353,11 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ + # pylint: disable=too-many-locals + assert max_attempts > 0 + attempts = collections.defaultdict(int) + exhausted = set() + # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -358,8 +365,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting and max_attempts: - max_attempts -= 1 + while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -373,16 +379,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr + attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + if attempts[authzr] < max_attempts: + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) + else: + exhausted.add(authzr) - if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(waiting, updated) + if exhausted or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 0385667c7..77d47c522 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,26 +56,25 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`waiting` will be non-empty) + This might be caused by either timeout (`exhausted` will be non-empty) or by some authorization being invalid. - :ivar waiting: Priority queue with `datetime.datatime` (based on - ``Retry-After``) as key, and original `.AuthorizationResource` - as value. + :ivar exhausted: Set of `.AuthorizationResource` that didn't finish + within max allowed attempts. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, waiting, updated): - self.waiting = waiting + def __init__(self, exhausted, updated): + self.exhausted = exhausted self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.waiting) + return bool(self.exhausted) def __repr__(self): - return '{0}(waiting={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.waiting, self.updated) + return '{0}(exhausted={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.exhausted, self.updated) From a36a59ba6c7f221c5ef4beec053a1752376439b3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 20:31:40 +0000 Subject: [PATCH 085/201] Fix waiting->exhausted in PollError tests --- acme/acme/errors_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 45b269a0b..966be8f1e 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,5 +1,4 @@ """Tests for acme.errors.""" -import datetime import unittest import mock @@ -36,9 +35,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], + exhausted=set([mock.sentinel.AR]), updated={}) - self.invalid = PollError(waiting=[], updated={ + self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -46,7 +45,7 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' + self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' 'sentinel.AR2})', repr(self.invalid)) From 98b3c41f2b6d96f578a8475eb3a0bdbc4fd5aa62 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 7 Jan 2016 16:18:23 -0500 Subject: [PATCH 086/201] Add le-auto tests for "no upgrade needed" and "only a phase-2 upgrade needed". --- letsencrypt_auto/tests/auto_test.py | 52 ++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index 814c37a30..a522f6b9a 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -182,7 +182,35 @@ iQIDAQAB env=env) +def set_le_script_version(venv_dir, version): + """Tell the letsencrypt script to report a certain version. + + We actually replace the script with a dummy version that knows only how to + print its version. + + """ + with open(join(venv_dir, 'letsencrypt', 'bin', 'letsencrypt'), 'w') as script: + script.write("#!/usr/bin/env python\n" + "from sys import stderr\n" + "stderr.write('letsencrypt %s\\n')" % version) + + class AutoTests(TestCase): + # Remove these helpers when we no longer need to support Python 2.6: + def assertIn(self, member, container, msg=None): + """Just like self.assertTrue(a in b), but with a nicer default message.""" + if member not in container: + standardMsg = '%s not found in %s' % (safe_repr(member), + safe_repr(container)) + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIn(self, member, container, msg=None): + """Just like self.assertTrue(a not in b), but with a nicer default message.""" + if member in container: + standardMsg = '%s unexpectedly found in %s' % (safe_repr(member), + safe_repr(container)) + self.fail(self._formatMessage(msg, standardMsg)) + def test_all(self): """Exercise most branches of letsencrypt-auto. @@ -228,13 +256,27 @@ class AutoTests(TestCase): # Test when a phase-1 upgrade is needed, there's no LE binary # installed, and peep verifies: out, err = run_le_auto(venv_dir, base_url) - ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', - err.strip().splitlines()[-1])) + ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', + err.strip().splitlines()[-1])) + # Make a few assertions to test the validity of the next tests: + self.assertIn('Upgrading letsencrypt-auto ', out) + self.assertIn('Creating virtual environment...', out) + # This conveniently sets us up to test the next 2 cases. - # This conveniently sets us up to test the next 2 cases: - # Test when no phase-1 upgrade is needed and no LE upgrade is needed (probably a common case). + # Test when neither phase-1 upgrade nor phase-2 upgrade is + # needed (probably a common case): + set_le_script_version(venv_dir, '99.9.9') + out, err = run_le_auto(venv_dir, base_url) + self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertNotIn('Creating virtual environment...', out) + + # Test when a phase-1 upgrade is not needed but a phase-2 + # upgrade is: + set_le_script_version(venv_dir, '0.0.1') + out, err = run_le_auto(venv_dir, base_url) + self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertIn('Creating virtual environment...', out) - # Test (when no phase-1 upgrade is needed), there's an out-of-date LE script installed, (and peep works). # Test when peep has a hash mismatch. # Test when the OpenSSL sig mismatches. From e5e5c2d65bc4b1f781fb9dff0841d7d305d3d2cf Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 7 Jan 2016 16:45:27 -0500 Subject: [PATCH 087/201] Don't stomp on the in-tree le-auto during tests. --- letsencrypt_auto/tests/auto_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index a522f6b9a..9cb8ebfeb 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -7,7 +7,7 @@ from json import dumps from os import environ from os.path import abspath, dirname, join import re -from shutil import rmtree +from shutil import copy, rmtree import socket import ssl from subprocess import CalledProcessError, check_output, Popen, PIPE @@ -177,7 +177,7 @@ iQIDAQAB -----END PUBLIC KEY-----""") env.update(d) return out_and_err( - join(dirname(tests_dir()), 'letsencrypt-auto') + ' --version', + join(venv_dir, 'letsencrypt-auto') + ' --version', shell=True, env=env) @@ -255,6 +255,7 @@ class AutoTests(TestCase): with serving(resources) as base_url: # Test when a phase-1 upgrade is needed, there's no LE binary # installed, and peep verifies: + copy(join(dirname(tests_dir()), 'letsencrypt-auto'), venv_dir) out, err = run_le_auto(venv_dir, base_url) ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) From 134b7ab8de29951eff4f01d05abf5951d2c93757 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 7 Jan 2016 17:04:32 -0500 Subject: [PATCH 088/201] Add a test for when openssl signature verification fails during phase-1 upgrade. --- letsencrypt_auto/tests/auto_test.py | 39 ++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index 9cb8ebfeb..c94dea292 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -110,6 +110,8 @@ def tests_dir(): return dirname(abspath(__file__)) +LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto') + @contextmanager def ephemeral_dir(): dir = mkdtemp(prefix='le-test-') @@ -211,9 +213,11 @@ class AutoTests(TestCase): safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) - def test_all(self): + def test_successes(self): """Exercise most branches of letsencrypt-auto. + They just happen to be the branches in which everything goes well. + The branches: * An le-auto upgrade is needed. @@ -234,12 +238,9 @@ class AutoTests(TestCase): 2. One combination of branches happens to set us up nicely for testing the next, saving code. - At the moment, we let bootstrapping run. We probably wanted those - packages installed anyway for local development. - - For tests which get this far, we run merely ``letsencrypt --version``. - The functioning of the rest of the letsencrypt script is covered by - other test suites. + For tests which get to the end, we run merely ``letsencrypt + --version``. The functioning of the rest of the letsencrypt script is + covered by other test suites. """ NEW_LE_AUTO = build_le_auto(version='99.9.9') @@ -255,7 +256,7 @@ class AutoTests(TestCase): with serving(resources) as base_url: # Test when a phase-1 upgrade is needed, there's no LE binary # installed, and peep verifies: - copy(join(dirname(tests_dir()), 'letsencrypt-auto'), venv_dir) + copy(LE_AUTO_PATH, venv_dir) out, err = run_le_auto(venv_dir, base_url) ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) @@ -279,5 +280,25 @@ class AutoTests(TestCase): self.assertNotIn('Upgrading letsencrypt-auto ', out) self.assertIn('Creating virtual environment...', out) + def test_openssl_failure(self): + """Make sure we stop if the openssl signature check fails.""" + with ephemeral_dir() as venv_dir: + # Serve an unrelated hash signed with the good key (easier than + # making a bad key, and a mismatch is a mismatch): + resources = {'': 'letsencrypt/', + 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), + 'v99.9.9/letsencrypt-auto.sig': signed('something else')} + with serving(resources) as base_url: + copy(LE_AUTO_PATH, venv_dir) + try: + out, err = run_le_auto(venv_dir, base_url) + except CalledProcessError as exc: + eq_(exc.returncode, 1) + self.assertIn("Couldn't verify signature of downloaded " + "letsencrypt-auto.", + exc.output) + else: + self.fail('Signature check on letsencrypt-auto erroneously passed!') + # Test when peep has a hash mismatch. - # Test when the OpenSSL sig mismatches. From bb31d71fe652f5105144ed94aa039c4c41b676e8 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 7 Jan 2016 23:41:02 -0500 Subject: [PATCH 089/201] Add a test for failed hash verification during phase-2 upgrade. --- letsencrypt_auto/tests/auto_test.py | 62 ++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt_auto/tests/auto_test.py index c94dea292..36a23e9c6 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt_auto/tests/auto_test.py @@ -4,12 +4,13 @@ from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler from contextlib import contextmanager from functools import partial from json import dumps -from os import environ +from os import chmod, environ from os.path import abspath, dirname, join import re from shutil import copy, rmtree import socket import ssl +from stat import S_IRUSR, S_IXUSR from subprocess import CalledProcessError, check_output, Popen, PIPE from tempfile import mkdtemp from threading import Thread @@ -198,6 +199,23 @@ def set_le_script_version(venv_dir, version): class AutoTests(TestCase): + """Test the major branch points of letsencrypt-auto: + + * An le-auto upgrade is needed. + * An le-auto upgrade is not needed. + * There was an out-of-date LE script installed. + * There was a current LE script installed. + * There was no LE script installed (less important). + * Peep verification passes. + * Peep has a hash mismatch. + * The OpenSSL sig matches. + * The OpenSSL sig mismatches. + + For tests which get to the end, we run merely ``letsencrypt --version``. + The functioning of the rest of the letsencrypt script is covered by other + test suites. + + """ # Remove these helpers when we no longer need to support Python 2.6: def assertIn(self, member, container, msg=None): """Just like self.assertTrue(a in b), but with a nicer default message.""" @@ -218,17 +236,6 @@ class AutoTests(TestCase): They just happen to be the branches in which everything goes well. - The branches: - - * An le-auto upgrade is needed. - * An le-auto upgrade is not needed. - * There was an out-of-date LE script installed. - * There was a current LE script installed. - * There was no LE script installed. (not that important) - * Peep verification passes. - * Peep has a hash mismatch. - * The OpenSSL sig mismatches. - I violate my usual rule of having small, decoupled tests, because... 1. We shouldn't need to run a Cartesian product of the branches: the @@ -238,10 +245,6 @@ class AutoTests(TestCase): 2. One combination of branches happens to set us up nicely for testing the next, saving code. - For tests which get to the end, we run merely ``letsencrypt - --version``. The functioning of the rest of the letsencrypt script is - covered by other test suites. - """ NEW_LE_AUTO = build_le_auto(version='99.9.9') NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) @@ -299,6 +302,29 @@ class AutoTests(TestCase): "letsencrypt-auto.", exc.output) else: - self.fail('Signature check on letsencrypt-auto erroneously passed!') + self.fail('Signature check on letsencrypt-auto erroneously passed.') - # Test when peep has a hash mismatch. + def test_peep_failure(self): + """Make sure peep stops us if there is a hash mismatch.""" + with ephemeral_dir() as venv_dir: + resources = {'': 'letsencrypt/', + 'letsencrypt/json': dumps({'releases': {'99.9.9': None}})} + with serving(resources) as base_url: + # Build a le-auto script embedding a bad requirements file: + venv_le_auto_path = join(venv_dir, 'letsencrypt-auto') + with open(venv_le_auto_path, 'w') as le_auto: + le_auto.write(build_le_auto( + version='99.9.9', + requirements='# sha256: badbadbadbadbadbadbadbadbadbadbadbadbadbadb\n' + 'configobj==5.0.6')) + chmod(venv_le_auto_path, S_IRUSR | S_IXUSR) + try: + out, err = run_le_auto(venv_dir, base_url) + except CalledProcessError as exc: + eq_(exc.returncode, 1) + self.assertIn("THE FOLLOWING PACKAGES DIDN'T MATCH THE " + "HASHES SPECIFIED IN THE REQUIREMENTS", + exc.output) + else: + self.fail("Peep didn't detect a bad hash and stop the " + "installation.") From 1d719bd89c54adb5f8a4f225bee870a1e1e886e9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 8 Jan 2016 15:09:10 -0500 Subject: [PATCH 090/201] Teach le-auto about dependencies that are conditional on the Python version. --- acme/setup.py | 2 + letsencrypt_auto/letsencrypt-auto | 65 +++++++++++++------ letsencrypt_auto/letsencrypt-auto.template | 6 ++ .../pieces/conditional_requirements.py | 39 +++++++++++ .../pieces/letsencrypt-auto-requirements.txt | 20 ------ setup.py | 1 + 6 files changed, 93 insertions(+), 40 deletions(-) create mode 100644 letsencrypt_auto/pieces/conditional_requirements.py diff --git a/acme/setup.py b/acme/setup.py index ba2c88394..d2c74accb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -21,6 +21,7 @@ install_requires = [ ] # env markers in extras_require cause problems with older pip: #517 +# Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying @@ -30,6 +31,7 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') +# Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt_auto/letsencrypt-auto index 452fb13b4..8ab0e1b95 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt_auto/letsencrypt-auto @@ -475,13 +475,6 @@ idna==2.0 # sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY ipaddress==1.0.15 -# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs -# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY -mock==1.3.0 - -# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ -ndg-httpsclient==0.4.0 - # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 @@ -513,19 +506,6 @@ pbr==1.8.1 # sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus psutil==3.3.0 -# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 -# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 -# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A -# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U -# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU -# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg -# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg -# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 -# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 -# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik -# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 -pyasn1==0.1.9 - # sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M pycparser==2.14 @@ -609,6 +589,51 @@ letsencrypt==0.1.1 letsencrypt-apache==0.1.1 UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/conditional_requirements.py" +"""Spit out additional pinned requirements depending on the Python version.""" +from sys import version_info + + +if __name__ == '__main__': + if version_info < (2, 7, 9): + print """ +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 +""" + if version_info < (2, 7): + print """ +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +mock==1.0.1 +""" + else: + print """ +# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs +# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY +mock==1.3.0 +""" + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + "$VENV_BIN/python" "$TEMP_DIR/conditional_requirements.py" >> "$TEMP_DIR/letsencrypt-auto-requirements.txt" # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" #!/usr/bin/env python diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt_auto/letsencrypt-auto.template index 6e7cf784a..b1852079a 100755 --- a/letsencrypt_auto/letsencrypt-auto.template +++ b/letsencrypt_auto/letsencrypt-auto.template @@ -182,6 +182,12 @@ if [ "$1" = "--no-self-upgrade" ]; then cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/conditional_requirements.py" +{{ conditional_requirements.py }} +UNLIKELY_EOF + # ------------------------------------------------------------------------- + "$VENV_BIN/python" "$TEMP_DIR/conditional_requirements.py" >> "$TEMP_DIR/letsencrypt-auto-requirements.txt" # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" {{ peep.py }} diff --git a/letsencrypt_auto/pieces/conditional_requirements.py b/letsencrypt_auto/pieces/conditional_requirements.py new file mode 100644 index 000000000..5194a103b --- /dev/null +++ b/letsencrypt_auto/pieces/conditional_requirements.py @@ -0,0 +1,39 @@ +"""Spit out additional pinned requirements depending on the Python version.""" +from sys import version_info + + +if __name__ == '__main__': + if version_info < (2, 7, 9): + print """ +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 +""" + if version_info < (2, 7): + print """ +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +mock==1.0.1 +""" + else: + print """ +# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs +# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY +mock==1.3.0 +""" diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt index 6be690043..5b800b8be 100644 --- a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt @@ -69,13 +69,6 @@ idna==2.0 # sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY ipaddress==1.0.15 -# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs -# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY -mock==1.3.0 - -# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ -ndg-httpsclient==0.4.0 - # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 @@ -107,19 +100,6 @@ pbr==1.8.1 # sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus psutil==3.3.0 -# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 -# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 -# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A -# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U -# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU -# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg -# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg -# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 -# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 -# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik -# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 -pyasn1==0.1.9 - # sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M pycparser==2.14 diff --git a/setup.py b/setup.py index f95f672ff..494f894d9 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ install_requires = [ ] # env markers in extras_require cause problems with older pip: #517 +# Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying From cd43e9035bf1a368bc72536525910ca1d12367fc Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 8 Jan 2016 16:26:25 -0500 Subject: [PATCH 091/201] Rename letsencrypt_auto dir to match other dirs. Originally, I had it in mind to move letsencrypt-auto inside this dir. However, now we'd like to copy it or link it to the root level, where people are used to finding it (at least for awhile). Since it would be confusing to have a letsencrypt-auto and a letsencrypt_auto right next to each other, we rename this folder. --- Dockerfile | 2 +- Dockerfile-dev | 2 +- .../Dockerfile | 4 ++-- .../build.py | 0 .../letsencrypt-auto | 2 +- .../letsencrypt-auto.template | 0 .../pieces/bootstrappers/arch_common.sh | 0 .../pieces/bootstrappers/deb_common.sh | 0 .../pieces/bootstrappers/free_bsd.sh | 0 .../pieces/bootstrappers/gentoo_common.sh | 0 .../pieces/bootstrappers/mac.sh | 0 .../pieces/bootstrappers/rpm_common.sh | 0 .../pieces/bootstrappers/suse_common.sh | 0 .../pieces/conditional_requirements.py | 0 .../pieces/fetch.py | 2 +- .../pieces/letsencrypt-auto-requirements.txt | 0 .../pieces/peep.py | 0 .../tests/__init__.py | 0 .../tests/auto_test.py | 18 +++++++++++------- .../tests/certs/ca/my-root-ca.crt.pem | 0 .../tests/certs/ca/my-root-ca.key.pem | 0 .../tests/certs/ca/my-root-ca.srl | 0 .../tests/certs/localhost/cert.pem | 0 .../tests/certs/localhost/localhost.csr.pem | 0 .../tests/certs/localhost/privkey.pem | 0 .../tests/certs/localhost/server.pem | 0 .../tests/signing.key | 0 letsencrypt_auto/__init__.py | 0 28 files changed, 17 insertions(+), 13 deletions(-) rename {letsencrypt_auto => letsencrypt-auto-source}/Dockerfile (86%) rename {letsencrypt_auto => letsencrypt-auto-source}/build.py (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/letsencrypt-auto (99%) rename {letsencrypt_auto => letsencrypt-auto-source}/letsencrypt-auto.template (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/arch_common.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/deb_common.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/free_bsd.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/gentoo_common.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/mac.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/rpm_common.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/bootstrappers/suse_common.sh (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/conditional_requirements.py (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/fetch.py (99%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/letsencrypt-auto-requirements.txt (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/pieces/peep.py (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/__init__.py (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/auto_test.py (99%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/ca/my-root-ca.crt.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/ca/my-root-ca.key.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/ca/my-root-ca.srl (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/localhost/cert.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/localhost/localhost.csr.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/localhost/privkey.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/certs/localhost/server.pem (100%) rename {letsencrypt_auto => letsencrypt-auto-source}/tests/signing.key (100%) delete mode 100644 letsencrypt_auto/__init__.py diff --git a/Dockerfile b/Dockerfile index 65e175aec..67043edd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 78ee0c8e0..61908d470 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -22,7 +22,7 @@ WORKDIR /opt/letsencrypt # TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ diff --git a/letsencrypt_auto/Dockerfile b/letsencrypt-auto-source/Dockerfile similarity index 86% rename from letsencrypt_auto/Dockerfile rename to letsencrypt-auto-source/Dockerfile index 4bdb1426f..0969dfce2 100644 --- a/letsencrypt_auto/Dockerfile +++ b/letsencrypt-auto-source/Dockerfile @@ -25,9 +25,9 @@ COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/ RUN update-ca-certificates # Copy code: -COPY . /home/lea/letsencrypt/letsencrypt_auto +COPY . /home/lea/letsencrypt/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-s", "letsencrypt/letsencrypt_auto/tests"] +CMD ["nosetests", "-s", "letsencrypt/letsencrypt-auto-source/tests"] diff --git a/letsencrypt_auto/build.py b/letsencrypt-auto-source/build.py similarity index 100% rename from letsencrypt_auto/build.py rename to letsencrypt-auto-source/build.py diff --git a/letsencrypt_auto/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto similarity index 99% rename from letsencrypt_auto/letsencrypt-auto rename to letsencrypt-auto-source/letsencrypt-auto index 8ab0e1b95..dfefe1c46 100755 --- a/letsencrypt_auto/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1731,7 +1731,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt_auto/') % tag + 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') diff --git a/letsencrypt_auto/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template similarity index 100% rename from letsencrypt_auto/letsencrypt-auto.template rename to letsencrypt-auto-source/letsencrypt-auto.template diff --git a/letsencrypt_auto/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/arch_common.sh rename to letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/deb_common.sh rename to letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/free_bsd.sh rename to letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh rename to letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/mac.sh rename to letsencrypt-auto-source/pieces/bootstrappers/mac.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/rpm_common.sh rename to letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh diff --git a/letsencrypt_auto/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh similarity index 100% rename from letsencrypt_auto/pieces/bootstrappers/suse_common.sh rename to letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh diff --git a/letsencrypt_auto/pieces/conditional_requirements.py b/letsencrypt-auto-source/pieces/conditional_requirements.py similarity index 100% rename from letsencrypt_auto/pieces/conditional_requirements.py rename to letsencrypt-auto-source/pieces/conditional_requirements.py diff --git a/letsencrypt_auto/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py similarity index 99% rename from letsencrypt_auto/pieces/fetch.py rename to letsencrypt-auto-source/pieces/fetch.py index 2ba296ca6..8c38cfb1d 100644 --- a/letsencrypt_auto/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -95,7 +95,7 @@ def verified_new_le_auto(get, tag, temp_dir): le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' - 'letsencrypt_auto/') % tag + 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') write(PUBLIC_KEY, temp_dir, 'public_key.pem') diff --git a/letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt similarity index 100% rename from letsencrypt_auto/pieces/letsencrypt-auto-requirements.txt rename to letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt diff --git a/letsencrypt_auto/pieces/peep.py b/letsencrypt-auto-source/pieces/peep.py similarity index 100% rename from letsencrypt_auto/pieces/peep.py rename to letsencrypt-auto-source/pieces/peep.py diff --git a/letsencrypt_auto/tests/__init__.py b/letsencrypt-auto-source/tests/__init__.py similarity index 100% rename from letsencrypt_auto/tests/__init__.py rename to letsencrypt-auto-source/tests/__init__.py diff --git a/letsencrypt_auto/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py similarity index 99% rename from letsencrypt_auto/tests/auto_test.py rename to letsencrypt-auto-source/tests/auto_test.py index 36a23e9c6..6b6f388d4 100644 --- a/letsencrypt_auto/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -12,13 +12,22 @@ import socket import ssl from stat import S_IRUSR, S_IXUSR from subprocess import CalledProcessError, check_output, Popen, PIPE +import sys from tempfile import mkdtemp from threading import Thread from unittest import TestCase from nose.tools import eq_, nottest, ok_ -from ..build import build as build_le_auto + +@nottest +def tests_dir(): + """Return a path to the "tests" directory.""" + return dirname(abspath(__file__)) + + +sys.path.insert(0, dirname(tests_dir())) +from build import build as build_le_auto class RequestHandler(BaseHTTPRequestHandler): @@ -105,14 +114,9 @@ def serving(resources): thread.join() -@nottest -def tests_dir(): - """Return a path to the "tests" directory.""" - return dirname(abspath(__file__)) - - LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto') + @contextmanager def ephemeral_dir(): dir = mkdtemp(prefix='le-test-') diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem similarity index 100% rename from letsencrypt_auto/tests/certs/ca/my-root-ca.crt.pem rename to letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem similarity index 100% rename from letsencrypt_auto/tests/certs/ca/my-root-ca.key.pem rename to letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem diff --git a/letsencrypt_auto/tests/certs/ca/my-root-ca.srl b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl similarity index 100% rename from letsencrypt_auto/tests/certs/ca/my-root-ca.srl rename to letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl diff --git a/letsencrypt_auto/tests/certs/localhost/cert.pem b/letsencrypt-auto-source/tests/certs/localhost/cert.pem similarity index 100% rename from letsencrypt_auto/tests/certs/localhost/cert.pem rename to letsencrypt-auto-source/tests/certs/localhost/cert.pem diff --git a/letsencrypt_auto/tests/certs/localhost/localhost.csr.pem b/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem similarity index 100% rename from letsencrypt_auto/tests/certs/localhost/localhost.csr.pem rename to letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem diff --git a/letsencrypt_auto/tests/certs/localhost/privkey.pem b/letsencrypt-auto-source/tests/certs/localhost/privkey.pem similarity index 100% rename from letsencrypt_auto/tests/certs/localhost/privkey.pem rename to letsencrypt-auto-source/tests/certs/localhost/privkey.pem diff --git a/letsencrypt_auto/tests/certs/localhost/server.pem b/letsencrypt-auto-source/tests/certs/localhost/server.pem similarity index 100% rename from letsencrypt_auto/tests/certs/localhost/server.pem rename to letsencrypt-auto-source/tests/certs/localhost/server.pem diff --git a/letsencrypt_auto/tests/signing.key b/letsencrypt-auto-source/tests/signing.key similarity index 100% rename from letsencrypt_auto/tests/signing.key rename to letsencrypt-auto-source/tests/signing.key diff --git a/letsencrypt_auto/__init__.py b/letsencrypt_auto/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 55128383776f3e8a65d5ea2c6024325d4c630762 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 8 Jan 2016 16:55:52 -0500 Subject: [PATCH 092/201] Get le-auto tests running on Travis. --- .travis.yml | 5 +++-- letsencrypt-auto-source/Dockerfile | 2 +- letsencrypt-auto-source/tests/__init__.py | 2 +- tox.ini | 10 ++++++++++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5d6d8a85..a40cbb44a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python services: + - docker - rabbitmq - mariadb # apacheconftest @@ -22,6 +23,7 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=le_auto - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration @@ -37,8 +39,7 @@ branches: - master - /^test-.*$/ -# container-based infrastructure -sudo: false +sudo: required addons: # make sure simplehttp simple verification works (custom /etc/hosts) diff --git a/letsencrypt-auto-source/Dockerfile b/letsencrypt-auto-source/Dockerfile index 0969dfce2..667acfe5a 100644 --- a/letsencrypt-auto-source/Dockerfile +++ b/letsencrypt-auto-source/Dockerfile @@ -30,4 +30,4 @@ COPY . /home/lea/letsencrypt/letsencrypt-auto-source USER lea WORKDIR /home/lea -CMD ["nosetests", "-s", "letsencrypt/letsencrypt-auto-source/tests"] +CMD ["nosetests", "-v", "-s", "letsencrypt/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/tests/__init__.py b/letsencrypt-auto-source/tests/__init__.py index 13180b5f5..45db90444 100644 --- a/letsencrypt-auto-source/tests/__init__.py +++ b/letsencrypt-auto-source/tests/__init__.py @@ -1,6 +1,6 @@ """Tests for letsencrypt-auto -For now, run these by saying... :: +Run these locally by saying... :: ./build.py && docker build -t lea . && docker run --rm -t -i lea diff --git a/tox.ini b/tox.ini index dbd6d51fa..eb4368393 100644 --- a/tox.ini +++ b/tox.ini @@ -75,3 +75,13 @@ setenv = commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test --debian-modules + +[testenv:le_auto] +# At the moment, this tests under Python 2.7 only, as only that version is +# readily available on the Trusty Docker image. +commands = + docker build -t lea letsencrypt-auto-source + docker run --rm -t -i lea +whitelist_externals = + docker +passenv = DOCKER_* \ No newline at end of file From 1f45c2ca5ce31d6ded34bf8d4ea277b145e1227b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 9 Jan 2016 15:10:06 -0800 Subject: [PATCH 093/201] s/--apache/--standalong in non-interactive unit tests Since apache may not be installed in travis or other test envs --- letsencrypt/tests/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 5c5e7f8bd..0775bc349 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -146,7 +146,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_noninteractive(self): args = ['-n', 'certonly'] self._cli_missing_flag(args, "specify a plugin") - args.extend(['--apache', '-d', 'eg.is']) + args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") with mock.patch('letsencrypt.cli._auth_from_domains'): with mock.patch('letsencrypt.cli.client.acme_from_config_key'): From 74f09fb7bdd7930f843c62618a41b8cf27a85e63 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 9 Jan 2016 15:43:58 -0800 Subject: [PATCH 094/201] Never auto-select plugins in non-interactive mode * We really want the user to pick one, so that the later addition of a second option doesn't cause -n mode to fail. --- letsencrypt/display/ops.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 90d3d97c3..5568e24b6 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -32,15 +32,7 @@ def choose_plugin(prepared, question): while True: disp = util(interfaces.IDisplay) - try: - code, index = disp.menu(question, opts, help_label="More Info") - except errors.MissingCommandlineFlag: - # use a custom message for this case - raise errors.MissingCommandlineFlag, ("Missing command line flags. For non-interactive " - "execution, you will need to specify a plugin on the command line. Run with " - "'--help plugins' to see a list of options, and see " - " https://eff.org/letsencrypt-plugins for more detail on what the plugins " - "do and how to use them.") + code, index = disp.menu(question, opts, help_label="More Info") if code == display_util.OK: plugin_ep = prepared[index] @@ -82,6 +74,16 @@ def pick_plugin(config, default, plugins, question, ifaces): # throw more UX-friendly error if default not in plugins filtered = plugins.filter(lambda p_ep: p_ep.name == default) else: + if config.noninteractive_mode: + # it's really bad to auto-select the single available plugin in + # non-interactive mode, because an update could later add a second + # available plugin + raise errors.MissingCommandlineFlag, ("Missing command line flags. For non-interactive " + "execution, you will need to specify a plugin on the command line. Run with " + "'--help plugins' to see a list of options, and see " + " https://eff.org/letsencrypt-plugins for more detail on what the plugins " + "do and how to use them.") + filtered = plugins.visible().ifaces(ifaces) filtered.init(config) From 3c70af7da5ab37e2bbe9093ab3d5b9193e18b94e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 9 Jan 2016 15:55:54 -0800 Subject: [PATCH 095/201] Avoid accidentally mocking noninteractivity --- letsencrypt/tests/display/ops_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/tests/display/ops_test.py b/letsencrypt/tests/display/ops_test.py index 31db47cce..8db751e34 100644 --- a/letsencrypt/tests/display/ops_test.py +++ b/letsencrypt/tests/display/ops_test.py @@ -69,7 +69,7 @@ class PickPluginTest(unittest.TestCase): """Tests for letsencrypt.display.ops.pick_plugin.""" def setUp(self): - self.config = mock.Mock() + self.config = mock.Mock(noninteractive_mode=False) self.default = None self.reg = mock.MagicMock() self.question = "Question?" From fac2ed41d8eaf689e0a565f89a3b5993705165d6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 10 Jan 2016 18:17:35 +0000 Subject: [PATCH 096/201] ACME: pylint to 80 chars --- acme/.pylintrc | 2 +- acme/acme/challenges_test.py | 16 ++++++++++------ acme/acme/client_test.py | 9 ++++++--- acme/acme/messages.py | 5 +++-- acme/acme/standalone_test.py | 15 ++++++++------- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/acme/.pylintrc b/acme/.pylintrc index 33650310d..d0d150631 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -100,7 +100,7 @@ comment=no [FORMAT] # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=80 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 4f2d06167..ef78e1eba 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -73,7 +73,8 @@ class KeyAuthorizationChallengeResponseTest(unittest.TestCase): def test_verify_wrong_form(self): from acme.challenges import KeyAuthorizationChallengeResponse response = KeyAuthorizationChallengeResponse( - key_authorization='.foo.oKGqedy-b-acd5eoybm2f-NVFxvyOoET5CNy3xnv8WY') + key_authorization='.foo.oKGqedy-b-acd5eoybm2f-' + 'NVFxvyOoET5CNy3xnv8WY') self.assertFalse(response.verify(self.chall, KEY.public_key())) @@ -273,10 +274,12 @@ class TLSSNI01ResponseTest(unittest.TestCase): @mock.patch('acme.challenges.TLSSNI01Response.verify_cert', autospec=True) def test_simple_verify(self, mock_verify_cert): mock_verify_cert.return_value = mock.sentinel.verification - self.assertEqual(mock.sentinel.verification, self.response.simple_verify( - self.chall, self.domain, KEY.public_key(), - cert=mock.sentinel.cert)) - mock_verify_cert.assert_called_once_with(self.response, mock.sentinel.cert) + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, mock.sentinel.cert) @mock.patch('acme.challenges.TLSSNI01Response.probe_cert') def test_simple_verify_false_on_probe_error(self, mock_probe_cert): @@ -590,7 +593,8 @@ class DNSTest(unittest.TestCase): def test_check_validation_wrong_fields(self): bad_validation = jose.JWS.sign( - payload=self.msg.update(token=b'x' * 20).json_dumps().encode('utf-8'), + payload=self.msg.update( + token=b'x' * 20).json_dumps().encode('utf-8'), alg=jose.RS256, key=KEY) self.assertFalse(self.msg.check_validation( bad_validation, KEY.public_key())) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 58f55b293..863fe350f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -34,8 +34,10 @@ class ClientTest(unittest.TestCase): self.net.get.return_value = self.response self.directory = messages.Directory({ - messages.NewRegistration: 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', }) from acme.client import Client @@ -331,7 +333,8 @@ class ClientTest(unittest.TestCase): self.assertEqual(clock.dt, datetime.datetime(2015, 3, 27, 0, 1, 7)) # CA sets invalid | TODO: move to a separate test - invalid_authzr = mock.MagicMock(times=[], retries=[messages.STATUS_INVALID]) + invalid_authzr = mock.MagicMock( + times=[], retries=[messages.STATUS_INVALID]) self.assertRaises( errors.PollError, self.client.poll_and_request_issuance, csr, authzrs=(invalid_authzr,), mintime=mintime) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 9c6a5f7b9..06b4492d6 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -130,8 +130,9 @@ class Directory(jose.JSONDeSerializable): @classmethod def register(cls, resource_body_cls): """Register resource.""" - assert resource_body_cls.resource_type not in cls._REGISTERED_TYPES - cls._REGISTERED_TYPES[resource_body_cls.resource_type] = resource_body_cls + resource_type = resource_body_cls.resource_type + assert resource_type not in cls._REGISTERED_TYPES + cls._REGISTERED_TYPES[resource_type] = resource_body_cls return resource_body_cls def __init__(self, jobj): diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 2778635f5..85cd9d11d 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -32,11 +32,10 @@ class TLSSNI01ServerTest(unittest.TestCase): """Test for acme.standalone.TLSSNI01Server.""" def setUp(self): - self.certs = { - b'localhost': (test_util.load_pyopenssl_private_key('rsa512_key.pem'), - # pylint: disable=protected-access - test_util.load_cert('cert.pem')), - } + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa512_key.pem'), + test_util.load_cert('cert.pem'), + )} from acme.standalone import TLSSNI01Server self.server = TLSSNI01Server(("", 0), certs=self.certs) # pylint: disable=no-member @@ -49,7 +48,8 @@ class TLSSNI01ServerTest(unittest.TestCase): def test_it(self): host, port = self.server.socket.getsockname()[:2] - cert = crypto_util.probe_sni(b'localhost', host=host, port=port, timeout=1) + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1) self.assertEqual(jose.ComparableX509(cert), jose.ComparableX509(self.certs[b'localhost'][1])) @@ -140,7 +140,8 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): while max_attempts: max_attempts -= 1 try: - cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', self.port) + cert = crypto_util.probe_sni( + b'localhost', b'0.0.0.0', self.port) except errors.Error: self.assertTrue(max_attempts > 0, "Timeout!") time.sleep(1) # wait until thread starts From 2eb3e09ca96b7b186ae2122510a0452036b86363 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 22:57:49 -0800 Subject: [PATCH 097/201] Check correct signature presence for release --- tools/release.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/release.sh b/tools/release.sh index 172f6fea1..2d427d49d 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -34,6 +34,9 @@ else echo Releasing developer version "$version"... fi +if [ "$RELEASE_OPENSSL_KEY" = "" ] ; then + RELEASE_OPENSSL_KEY="`realpath \`dirname $0\``/eff-pubkey.pem" +fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -78,6 +81,14 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" +if ! openssl dgst -sha1 -verify $RELEASE_OPENSSL_KEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; then + echo Failed letsencrypt-auto signature check on "$RELEASE_BRANCH" + echo please fix that and re-run +fi + + SetVersion() { ver="$1" for pkg_dir in $SUBPKGS @@ -112,6 +123,7 @@ do cd - done + mkdir "dist.$version" mv dist "dist.$version/letsencrypt" for pkg_dir in $SUBPKGS From 7cfb10ba27cb7ae322edb8f81fab909867441184 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 23:12:48 -0800 Subject: [PATCH 098/201] These signatures should be in git --- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 0 -> 256 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 letsencrypt-auto-source/letsencrypt-auto.sig diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig new file mode 100644 index 0000000000000000000000000000000000000000..fb506192ef7acbb10108b61c1316e49c95ef61e8 GIT binary patch literal 256 zcmV+b0ssEJGl2Gn_~YU#dGI-^Lc7_m6k4IVwG`UW#cB)BTJLa(xYM*HS1Z9=PfI7J z9Kis?O}poysJeZr+Jn@dD#roxlBvQ>D?hNec99NuaF&*<#Y4;K6HiV1A zo)Bi^(8G;g^KrN#grE3{bUH4GeW?2EibWfc1$Nx?y#4$7>u)wkEWvtWBTi2xB>;B*^iwL{@;Q#+Vgp-*Ij|xj~}FxT*VD z1V~@Hk*+TJr7a@}ZfhBsVqzXW>3@WI9s%Kt;5E&-OLZrc0 Date: Sun, 10 Jan 2016 23:14:44 -0800 Subject: [PATCH 099/201] helpful documentation --- tools/half-sign.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/half-sign.c b/tools/half-sign.c index 454201799..b4ab99e4c 100644 --- a/tools/half-sign.c +++ b/tools/half-sign.c @@ -9,6 +9,9 @@ // This program can be used to perform RSA public key signatures given only // the hash of the file to be signed as input. +// To compile: +// gcc half-sign.c -lssl -lcrypto -o half-sign + // Sign with SHA1 #define HASH_SIZE 20 From bbd53d6d7d803cd02afd6d26b0ad4ed3266bd21b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 23:15:29 -0800 Subject: [PATCH 100/201] Ensure we have an leauto signature before releasing --- tools/release.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/release.sh b/tools/release.sh index 2d427d49d..61506f79e 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -81,11 +81,18 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" -if ! openssl dgst -sha1 -verify $RELEASE_OPENSSL_KEY -signature \ +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +if ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_KEY -signature \ letsencrypt-auto-source/letsencrypt-auto.sig \ letsencrypt-auto-source/letsencrypt-auto ; then echo Failed letsencrypt-auto signature check on "$RELEASE_BRANCH" echo please fix that and re-run + exit 1 +else + echo Signature check on letsencrypt-auto successful fi From 0c09eaff3c8ebeb3bdee1d13761c633abcffec07 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 23:18:19 -0800 Subject: [PATCH 101/201] Switch to real release key (though this is still a test signature) --- letsencrypt-auto-source/letsencrypt-auto | 15 +++++++++++++-- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes letsencrypt-auto-source/pieces/fetch.py | 15 +++++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index dfefe1c46..3ae182853 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1656,6 +1656,7 @@ from sys import argv, exit from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +#test PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe 4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B @@ -1670,8 +1671,18 @@ q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== -----END PUBLIC KEY----- -""") # TODO: Replace with real one. - +""") +# real +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB +-----END PUBLIC KEY----- +""") class ExpectedError(Exception): """A novice-readable exception that also carries the original exception for diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index fb506192ef7acbb10108b61c1316e49c95ef61e8..7db9da58e067d7e48975769a81e467cb06234515 100644 GIT binary patch literal 256 zcmV+b0ssE6R`Nfv8?!-f+CE*@!S0ACLy{E|JU%tqM7iG<#6M@Em3w}HKKYC`bS}vU z?tho}W5otLnsyNAzcq4|hTYh=T4~PtnG(4zn1eAn*g16JvDb7epvvsQrug@k2A43S zy^l@bGyvyzkqA6eWW<{1OppQsGomAa!R!SuP2jT@@T+4;Q9;JX*VG(&_n`oA2v^l( zDnkV5lw?Gtc&~2`EE~cMe(|$75%?VS?Tr<-1V5HGOpA5rpuzv_&_sD?hNec99NuaF&*<#Y4;K6HiV1A zo)Bi^(8G;g^KrN#grE3{bUH4GeW?2EibWfc1$Nx?y#4$7>u)wkEWvtWBTi2xB>;B*^iwL{@;Q#+Vgp-*Ij|xj~}FxT*VD z1V~@Hk*+TJr7a@}ZfhBsVqzXW>3@WI9s%Kt;5E&-OLZrc0 Date: Sun, 10 Jan 2016 23:22:04 -0800 Subject: [PATCH 102/201] Add tool for requesting & handling offline signatures --- tools/offline-sigrequest.sh | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 tools/offline-sigrequest.sh diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh new file mode 100755 index 000000000..ca349f629 --- /dev/null +++ b/tools/offline-sigrequest.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -o errexit + +if ! `which festival > /dev/null` ; then + echo Please install \'festival\'! + exit 1 +fi + +function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL + while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.5)"; \ + echo -n '(SayText "'; \ + sha1sum | cut -c1-40 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + echo '")' ) | festival + done + + echo 'Paste in the data from the QR code, then type Ctrl-D:' + cat > $2 +} + +function offlinesign { # $1 <-- INPFILE ; $2 <---SIGFILE + echo HASH FOR SIGNING: + SIGFILEBALL="$2.lzma.base64" + #echo "(place the resulting raw binary signature in $SIGFILEBALL)" + sha1sum $1 + echo metahash for confirmation only $(sha1sum $1 |cut -d' ' -f1 | tr -d '\n' | sha1sum | cut -c1-6) ... + echo + sayhash $1 $SIGFILEBALL +} + +function oncesigned { # $1 <-- INPFILE ; $2 <--SIGFILE + SIGFILEBALL="$2.lzma.base64" + cat $SIGFILEBALL | tr -d '\r' | base64 -d | unlzma -c > $2 || exit 1 + if ! [ -f $2 ] ; then + echo "Failed to find $2"'!' + exit 1 + fi + + if file $2 | grep -qv " data" ; then + echo "WARNING WARNING $2 does not look like a binary signature:" + echo `file $2` + exit 1 + fi +} + +HERE=`dirname $0` +LEAUTO="`realpath $HERE`/../letsencrypt-auto-source/letsencrypt-auto" +SIGFILE="$LEAUTO".sig +offlinesign $LEAUTO $SIGFILE +oncesigned $LEAUTO $SIGFILE From e17bb2750877801b15fc914599152a36cb592c0b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 11 Jan 2016 09:19:21 -0800 Subject: [PATCH 103/201] Remove test key --- letsencrypt-auto-source/pieces/fetch.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index bf270cdc4..39ff7777c 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -19,24 +19,6 @@ from subprocess import check_call, CalledProcessError from sys import argv, exit from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError - -#test -PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe -4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B -2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww -s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T -QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE -33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP -rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 -+E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK -EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu -q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 -3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn -I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== ------END PUBLIC KEY----- -""") -# real PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 From be653e8e6ba51134310046179ed2ee56d597c8c5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 11 Jan 2016 12:53:39 -0800 Subject: [PATCH 104/201] Use SHA256 openssl signatures --- tools/half-sign.c | 12 ++++++------ tools/release.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/half-sign.c b/tools/half-sign.c index b4ab99e4c..e56bc397c 100644 --- a/tools/half-sign.c +++ b/tools/half-sign.c @@ -12,18 +12,18 @@ // To compile: // gcc half-sign.c -lssl -lcrypto -o half-sign -// Sign with SHA1 -#define HASH_SIZE 20 +// Sign with SHA256 +#define HASH_SIZE 32 void usage() { printf("half-sign [binary hash file]\n"); printf("\n"); - printf(" Computes and prints a binary RSA signature over data given the SHA1 hash of\n"); + printf(" Computes and prints a binary RSA signature over data given the SHA256 hash of\n"); printf(" the data as input.\n"); printf("\n"); printf(" should be PEM encoded.\n"); printf("\n"); - printf(" The input SHA1 hash should be %d bytes in length. If no binary hash file is\n", HASH_SIZE); + printf(" The input SHA256 hash should be %d bytes in length. If no binary hash file is\n", HASH_SIZE); printf(" specified, it will be read from stdin.\n"); exit(1); } @@ -41,7 +41,7 @@ void sign_hashed_data(EVP_PKEY *signing_key, unsigned char *md, size_t mdlen) { if ((!ctx) || (EVP_PKEY_sign_init(ctx) <= 0) || (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) - || (EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha1()) <= 0)) { + || (EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha256()) <= 0)) { fprintf(stderr, "Failure establishing ctx for signature\n"); exit(1); } @@ -108,7 +108,7 @@ int main(int argc, char *argv[]) { exit(1); } if (fread(buffer, HASH_SIZE, 1, input) != 1) { - perror("half-sign: Failed to read SHA1 from input\n"); + perror("half-sign: Failed to read SHA256 from input\n"); exit(1); } diff --git a/tools/release.sh b/tools/release.sh index 61506f79e..0d3aa8808 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -34,8 +34,8 @@ else echo Releasing developer version "$version"... fi -if [ "$RELEASE_OPENSSL_KEY" = "" ] ; then - RELEASE_OPENSSL_KEY="`realpath \`dirname $0\``/eff-pubkey.pem" +if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then + RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry @@ -85,7 +85,7 @@ git checkout "$RELEASE_BRANCH" letsencrypt-auto-source/build.py # and that it's signed correctly -if ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_KEY -signature \ +if ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ letsencrypt-auto-source/letsencrypt-auto.sig \ letsencrypt-auto-source/letsencrypt-auto ; then echo Failed letsencrypt-auto signature check on "$RELEASE_BRANCH" From 916f8916d86f6d9a0c615590f1d79d8131e5bf61 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 11 Jan 2016 12:55:01 -0800 Subject: [PATCH 105/201] Clearer notes about when / how to edit the script --- letsencrypt-auto-source/letsencrypt-auto.template | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b1852079a..23e77de38 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -2,8 +2,14 @@ # # Download and run the latest release version of the Let's Encrypt client. # -# WARNING: "letsencrypt-auto" IS A GENERATED FILE. EDIT -# letsencrypt-auto.template INSTEAD. +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* set -e # Work even if somebody does "sh thisscript.sh". From 1b3c8e87c7b14646fbbf820df1a1c59e78660cc0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 11 Jan 2016 13:57:46 -0800 Subject: [PATCH 106/201] Better processing & documentation of leauto flags - move them to the top for clarity - accept them in any position - shadow & document them in the Python client --- .../letsencrypt-auto.template | 35 +++++++++++-------- letsencrypt/cli.py | 7 ++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 23e77de38..3d2180280 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -21,6 +21,24 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then + VERBOSE=1 + elif [ "$arg" = "--no-self-upgrade" ] ; then + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1 + elif [ "$arg" = "--os-packages-only" ] ; then + OS_PACKAGES_ONLY=1 + elif [ "$arg" = "--debug" ]; then + DEBUG=1 + fi +done + # letsencrypt-auto needs root access to bootstrap OS dependencies, and # letsencrypt itself needs root access for almost all modes of operation # The "normal" case is that sudo is used for the steps that need root, but @@ -150,22 +168,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X } -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) -for arg in "$@" ; do - # This first clause is redundant with the third, but hedging on portability - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then - VERBOSE=1 - elif [ "$arg" = "--debug" ]; then - DEBUG=1 - fi -done -if [ "$1" = "--no-self-upgrade" ]; then + +if [ "$NO_SELF_UPGRADE" = 1 ]; then # Phase 2: Create venv, install LE, and run. - shift 1 # the --no-self-upgrade arg if [ -f "$VENV_BIN/letsencrypt" ]; then INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) else @@ -226,7 +233,7 @@ else # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi - if [ "$1" = "--os-packages-only" ]; then + if [ "$OS_PACKAGES_ONLY" = 1 ]; then echo "OS packages installed." exit 0 fi diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index aba9116f9..1f9504c6e 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -998,6 +998,13 @@ def prepare_and_parse_args(plugins, args): "automation", "--duplicate", dest="duplicate", action="store_true", help="Allow making a certificate lineage that duplicates an existing one " "(both can be renewed in parallel)") + helpful.add( + "automation", "--os-packages-only", action="store_true", + help="(letsencrypt-auto only) install OS package dependencies and then stop") + helpful.add( + "automation", "--no-self-upgrade", action="store_true", + help="(letsencrypt-auto only) prevent the letsencrypt-auto script from" + " upgrading itself to newer released versions") helpful.add_group( "testing", description="The following flags are meant for " From 66ca7449cb8b12cabec090c778e1c5ffa318efdb Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 11 Jan 2016 13:38:43 -0500 Subject: [PATCH 107/201] Take le-auto tests out of Travis until we figure out why sudo:required causes other ones to fail. For now, we'll run them locally with `tox -e le_auto` as we do with the apacheconf tests. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a40cbb44a..a5d6d8a85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python services: - - docker - rabbitmq - mariadb # apacheconftest @@ -23,7 +22,6 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=le_auto - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration @@ -39,7 +37,8 @@ branches: - master - /^test-.*$/ -sudo: required +# container-based infrastructure +sudo: false addons: # make sure simplehttp simple verification works (custom /etc/hosts) From 6c05197a43fffe3dcd2c10f41954f8a61aec2134 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Mon, 11 Jan 2016 13:01:25 -0500 Subject: [PATCH 108/201] Remove mock as an install requirement. The motivation is to free us of a reliance on a rather modern version of setuptools, which caused le-auto failures for people on Wheezy and other older distros. (The alternative would have been to forcibly upgrade setuptools as the old le-auto did, but less is more.) Mock is used only in tests, so we move it to tests_require. It will still be installed automatically when setup.py test is run. Give all packages a test_suite so this works. The "testing" extra remains for optional packages not required for the nose tests but used in tox. However, the extra is much less useful now and is a candidate for deletion. We could roll the list of packages therein into the tox config so as not to favor any particular package. Remove tests_require=install_requires, which I don't think does anything useful, since install requirements are implicitly installed when running setup.py test. Fix tests to pass with mock removed. We had to stop them pulling down LE from PyPI, since the current version there (0.1.1) requires mock and explodes when `letsencrypt` is run. --- acme/setup.py | 4 +- letsencrypt-apache/setup.py | 1 + letsencrypt-auto-source/letsencrypt-auto | 30 +--------- .../letsencrypt-auto.template | 2 +- .../pieces/conditional_requirements.py | 10 ---- letsencrypt-auto-source/tests/auto_test.py | 56 +++++++++++++----- .../tests/fake-letsencrypt/dist/.DS_Store | Bin 0 -> 6148 bytes .../dist/letsencrypt-99.9.9.tar.gz | Bin 0 -> 899 bytes .../tests/fake-letsencrypt/letsencrypt.py | 8 +++ .../tests/fake-letsencrypt/setup.py | 12 ++++ letsencrypt-nginx/setup.py | 1 + letshelp-letsencrypt/setup.py | 1 + setup.cfg | 2 + setup.py | 4 +- tox.ini | 31 +++++----- 15 files changed, 83 insertions(+), 79 deletions(-) create mode 100644 letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store create mode 100644 letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz create mode 100755 letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py create mode 100644 letsencrypt-auto-source/tests/fake-letsencrypt/setup.py diff --git a/acme/setup.py b/acme/setup.py index af66c143e..7b532f28d 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -26,10 +26,7 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', - 'mock<1.1.0', ]) -else: - install_requires.append('mock') # Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7, 9): @@ -75,6 +72,7 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, + tests_require='mock<1.1.0' if sys.version_info < (2, 7) else 'mock', extras_require={ 'docs': docs_extras, 'testing': testing_extras, diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 58008e1e4..bfcce143b 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -62,4 +62,5 @@ setup( 'apache = letsencrypt_apache.configurator:ApacheConfigurator', ], }, + test_suite='letsencrypt_apache', ) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 3ae182853..7000d3027 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -619,16 +619,6 @@ pyasn1==0.1.9 # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ # sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ argparse==1.4.0 - -# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw -# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 -mock==1.0.1 -""" - else: - print """ -# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs -# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY -mock==1.3.0 """ UNLIKELY_EOF @@ -1606,7 +1596,7 @@ UNLIKELY_EOF if [ "$PEEP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while downloading and verifying Python packages:" - echo $PEEP_OUT + echo "$PEEP_OUT" exit 1 fi fi @@ -1655,24 +1645,6 @@ from subprocess import check_call, CalledProcessError from sys import argv, exit from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError - -#test -PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnwHkSuCSy3gIHawaCiIe -4ilJ5kfEmSoiu50uiimBhTESq1JG2gVqXVXFxxVgobGhahSF+/iRVp3imrTtGp1B -2heoHbELnPTTZ8E36WHKf4gkLEo0y0XgOP3oBJ9IM5q8J68x0U3Q3c+kTxd/sgww -s5NVwpjw4aAZhgDPe5u+rvthUYOD1whYUANgYvooCpV4httNv5wuDjo7SG2V797T -QTE8aG3AOhWzdsLm6E6Tl2o/dR6XKJi/RMiXIk53SzArimtAJXe/1GyADe1AgIGE -33Ja3hU3uu9lvnnkowy1VI0qvAav/mu/APahcWVYkBAvSVAhH3zGNAGZUnP2zfcP -rH7OPw/WrxLVGlX4trLnvQr1wzX7aiM2jdikcMiaExrP0JfQXPu00y3c+hjOC5S0 -+E5P+e+8pqz5iC5mmvEqy2aQJ6pV7dSpYX3mcDs8pCYaVXXtCPXS1noWirCcqCMK -EHGGdJCTXXLHaWUaGQ9Gx1An1gU7Ljkkji2Al65ZwYhkFowsLfuniYKuAywRrCNu -q958HnzFpZiQZAqZYtOHaiQiaHPs/36ZN0HuOEy0zM9FEHbp4V/DEn4pNCfAmRY5 -3v+3nIBhgiLdlM7cV9559aDNeutF25n1Uz2kvuSVSS94qTEmlteCPZGBQb9Rr2wn -I2OU8tPRzqKdQ6AwS9wvqscCAwEAAQ== ------END PUBLIC KEY----- -""") -# real PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b1852079a..0181d5b69 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -201,7 +201,7 @@ UNLIKELY_EOF if [ "$PEEP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while downloading and verifying Python packages:" - echo $PEEP_OUT + echo "$PEEP_OUT" exit 1 fi fi diff --git a/letsencrypt-auto-source/pieces/conditional_requirements.py b/letsencrypt-auto-source/pieces/conditional_requirements.py index 5194a103b..d81f03c6a 100644 --- a/letsencrypt-auto-source/pieces/conditional_requirements.py +++ b/letsencrypt-auto-source/pieces/conditional_requirements.py @@ -26,14 +26,4 @@ pyasn1==0.1.9 # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ # sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ argparse==1.4.0 - -# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw -# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 -mock==1.0.1 -""" - else: - print """ -# sha256: P1c6GL6U3ohtEZHyfBaEJ-9pPo3Pzs-VsXBXey62nLs -# sha256: HiR9vsxs4FcpnrfuAZrWgxS7kxUugdmmEQ019NXsoPY -mock==1.3.0 """ diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 6b6f388d4..4fa4b4e27 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -158,7 +158,21 @@ def signed(content, private_key_name='signing.key'): return out -def run_le_auto(venv_dir, base_url): +def install_le_auto(contents, venv_dir): + """Install some given source code as the letsencrypt-auto script at the + root level of a virtualenv. + + :arg contents: The contents of the built letsencrypt-auto script + :arg venv_dir: The path under which to install the script + + """ + venv_le_auto_path = join(venv_dir, 'letsencrypt-auto') + with open(venv_le_auto_path, 'w') as le_auto: + le_auto.write(contents) + chmod(venv_le_auto_path, S_IRUSR | S_IXUSR) + + +def run_le_auto(venv_dir, base_url, **kwargs): """Run the prebuilt version of letsencrypt-auto, returning stdout and stderr strings. @@ -181,7 +195,8 @@ uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB ------END PUBLIC KEY-----""") +-----END PUBLIC KEY-----""", + **kwargs) env.update(d) return out_and_err( join(venv_dir, 'letsencrypt-auto') + ' --version', @@ -250,40 +265,50 @@ class AutoTests(TestCase): the next, saving code. """ - NEW_LE_AUTO = build_le_auto(version='99.9.9') + NEW_LE_AUTO = build_le_auto( + version='99.9.9', + requirements='# sha256: 7NpInQZj4v2dvdCBUYtcBHqVlBfnUmlsKF_oSOzU9zY\n' + 'letsencrypt==99.9.9') NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) with ephemeral_dir() as venv_dir: # This serves a PyPI page with a higher version, a GitHub-alike # with a corresponding le-auto script, and a matching signature. - resources = {'': 'letsencrypt/', - 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + resources = {'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), 'v99.9.9/letsencrypt-auto': NEW_LE_AUTO, 'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG} with serving(resources) as base_url: + run_letsencrypt_auto = partial( + run_le_auto, + venv_dir, + base_url, + PIP_FIND_LINKS=join(tests_dir(), + 'fake-letsencrypt', + 'dist')) + # Test when a phase-1 upgrade is needed, there's no LE binary # installed, and peep verifies: - copy(LE_AUTO_PATH, venv_dir) - out, err = run_le_auto(venv_dir, base_url) + install_le_auto(build_le_auto(version='50.0.0'), venv_dir) + out, err = run_letsencrypt_auto() ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: self.assertIn('Upgrading letsencrypt-auto ', out) self.assertIn('Creating virtual environment...', out) - # This conveniently sets us up to test the next 2 cases. + # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This + # conveniently sets us up to test the next 2 cases. # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): - set_le_script_version(venv_dir, '99.9.9') - out, err = run_le_auto(venv_dir, base_url) + out, err = run_letsencrypt_auto() self.assertNotIn('Upgrading letsencrypt-auto ', out) self.assertNotIn('Creating virtual environment...', out) # Test when a phase-1 upgrade is not needed but a phase-2 # upgrade is: set_le_script_version(venv_dir, '0.0.1') - out, err = run_le_auto(venv_dir, base_url) + out, err = run_letsencrypt_auto() self.assertNotIn('Upgrading letsencrypt-auto ', out) self.assertIn('Creating virtual environment...', out) @@ -315,13 +340,12 @@ class AutoTests(TestCase): 'letsencrypt/json': dumps({'releases': {'99.9.9': None}})} with serving(resources) as base_url: # Build a le-auto script embedding a bad requirements file: - venv_le_auto_path = join(venv_dir, 'letsencrypt-auto') - with open(venv_le_auto_path, 'w') as le_auto: - le_auto.write(build_le_auto( + install_le_auto( + build_le_auto( version='99.9.9', requirements='# sha256: badbadbadbadbadbadbadbadbadbadbadbadbadbadb\n' - 'configobj==5.0.6')) - chmod(venv_le_auto_path, S_IRUSR | S_IXUSR) + 'configobj==5.0.6'), + venv_dir) try: out, err = run_le_auto(venv_dir, base_url) except CalledProcessError as exc: diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T02w>EAZ0 z`qy*|^j`=4A)_gah?@?1n22TRLzkzq5VOI~J2>;*I2`u6k#^^DjMgIuEBbF6rVaY9 zIsULbg8r-ePo{gg#=8H1+ty@+)XK8=qu#2$d;NakNC~y?sk|m$tY~CdRAx&8zpQx7>mOrE;nFWw}fR_h+jC^z#jGUKK1nhb{wqEFQgV7DxKsz4+Uwc-)ztIZ(5d->0h@Y|F6;i zXAu{SMe{TuZ;}2#FB`ejrqUBQWp}L}nK#sB~S00000000000000000000 Z0000000000006)f= 2 and argv[1] == '--version': + stderr.write('letsencrypt 99.9.9\n') diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py new file mode 100644 index 000000000..e5f7fde35 --- /dev/null +++ b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + + +setup( + name='letsencrypt', + version='99.9.9', + description='A mock version of letsencrypt that just prints its version', + py_modules=['letsencrypt'], + entry_points={ + 'console_scripts': ['letsencrypt = letsencrypt:main'] + } +) diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 1d42fe488..e4336f701 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -62,4 +62,5 @@ setup( 'nginx = letsencrypt_nginx.configurator:NginxConfigurator', ], }, + test_suite='letsencrypt_nginx', ) diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index d487e556d..316868fa8 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -55,4 +55,5 @@ setup( 'letshelp-letsencrypt-apache = letshelp_letsencrypt.apache:main', ], }, + test_suite='letshelp_letsencrypt', ) diff --git a/setup.cfg b/setup.cfg index ca4c1b1ca..4c9007edb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,5 @@ nocapture=1 cover-package=letsencrypt,acme,letsencrypt_apache,letsencrypt_nginx cover-erase=1 cover-tests=1 +# More verbose output: allows to detect busy waiting loops, especially on Travis +verbosity=1 \ No newline at end of file diff --git a/setup.py b/setup.py index bf146f3f6..8604f5119 100644 --- a/setup.py +++ b/setup.py @@ -53,12 +53,10 @@ if sys.version_info < (2, 7): # only some distros recognize stdlib argparse as already satisfying 'argparse', 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 - 'mock<1.1.0', ]) else: install_requires.extend([ 'ConfigArgParse', - 'mock', ]) dev_extras = [ @@ -116,13 +114,13 @@ setup( include_package_data=True, install_requires=install_requires, + tests_require='mock<1.1.0' if sys.version_info < (2, 7) else 'mock', extras_require={ 'dev': dev_extras, 'docs': docs_extras, 'testing': testing_extras, }, - tests_require=install_requires, # to test all packages run "python setup.py test -s # {acme,letsencrypt_apache,letsencrypt_nginx}" test_suite='letsencrypt', diff --git a/tox.ini b/tox.ini index eb4368393..81f962259 100644 --- a/tox.ini +++ b/tox.ini @@ -8,23 +8,20 @@ skipsdist = true envlist = py26,py27,py33,py34,py35,cover,lint -# nosetest -v => more verbose output, allows to detect busy waiting -# loops, especially on Travis - [testenv] -# packages installed separately to ensure that dowstream deps problems +# packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 commands = - pip install -e acme[testing] - nosetests -v acme - pip install -e .[testing] - nosetests -v letsencrypt + pip install -e acme + python acme/setup.py test + pip install -e . + python setup.py test pip install -e letsencrypt-apache - nosetests -v letsencrypt_apache + python letsencrypt-apache/setup.py test pip install -e letsencrypt-nginx - nosetests -v letsencrypt_nginx + python letsencrypt-nginx/setup.py test pip install -e letshelp-letsencrypt - nosetests -v letshelp_letsencrypt + python letshelp-letsencrypt/setup.py test setenv = PYTHONPATH = {toxinidir} @@ -33,18 +30,18 @@ setenv = [testenv:py33] commands = - pip install -e acme[testing] - nosetests -v acme + pip install -e acme + python acme/setup.py test [testenv:py34] commands = - pip install -e acme[testing] - nosetests -v acme + pip install -e acme + python acme/setup.py test [testenv:py35] commands = - pip install -e acme[testing] - nosetests -v acme + pip install -e acme + python acme/setup.py test [testenv:cover] basepython = python2.7 From 7ee23b723af6b7cb62f23f2bfdc2e472d80f89ec Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 12 Jan 2016 11:35:58 -0500 Subject: [PATCH 109/201] Get all tests, even le_auto, working on Travis. Switch to a MySQL 5.6 setup based on https://github.com/mozilla/treeherder/pull/1080/files and Travis's beta trusty infra, which runs on Google Compute Engine. Remove MariaDB addon, which conflicts with the socket used by the treeherder approach's mysql package. Remove maria service (which has no effect). --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d9b4cb5ea..ab81f20b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,14 @@ language: python services: - rabbitmq - - mariadb # apacheconftest #- apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq mysql-server-5.6 mysql-client-5.6 mysql-client-core-5.6 - 'dpkg -s libaugeas0' - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5.1)"' @@ -24,6 +25,7 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=py26-oldest BOULDER_INTEGRATION=1 - TOXENV=py27-oldest BOULDER_INTEGRATION=1 + - TOXENV=le_auto - TOXENV=py33 - TOXENV=py34 - TOXENV=lint @@ -45,14 +47,13 @@ branches: - master - /^test-.*$/ -# container-based infrastructure -sudo: false +sudo: required +dist: trusty addons: # make sure simplehttp simple verification works (custom /etc/hosts) hosts: - le.wtf - mariadb: "10.0" apt: sources: - augeas From a3288a92b9ea1dabf369a2059d44fd82ad85745b Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 12 Jan 2016 14:25:36 -0500 Subject: [PATCH 110/201] Disable too-many-instance-attributes for the acme linter. This should make the linter pass and allow us to merge the letsencrypt-auto-release branch when it's ready. IHNI why it passes on master without this disabled. --- acme/.pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/.pylintrc b/acme/.pylintrc index 33650310d..dcb8af713 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -60,7 +60,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=fixme,locally-disabled,abstract-class-not-used +disable=fixme,locally-disabled,abstract-class-not-used,too-many-instance-attributes # bstract-class-not-used cannot be disabled locally (at least in # pylint 1.4.1/2) From cb5beb84c59ea0f83783650d3f91d5e79bec6bef Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 12 Jan 2016 17:06:58 -0500 Subject: [PATCH 111/201] Fix Fedora 23 crasher. This fixes an "OSError: [Errno 2] No such file or directory" on Fedora 23. Note that openssl-devel was not sufficient to install the openssl commandline tool. The current manual-testing build of le-auto now crashes with #1548, but that should have been resolved when we upgraded the cryptography lib and so should go away when we build a new version. --- letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index d10a1b5ff..8a8f1526a 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -36,6 +36,7 @@ BootstrapRpmCommon() { gcc \ dialog \ augeas-libs \ + openssl \ openssl-devel \ libffi-devel \ redhat-rpm-config \ From a7ae4369c8693107b6d393385fab1e0d8bdef48d Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 12 Jan 2016 18:16:08 -0500 Subject: [PATCH 112/201] Bring built le-auto script up to date. --- letsencrypt-auto-source/letsencrypt-auto | 51 +++++++++++++++--------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7000d3027..d080d8800 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -123,28 +123,42 @@ BootstrapDebCommon() { virtualenv="$virtualenv python-virtualenv" fi - augeas_pkg=libaugeas0 + augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + + } + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then if lsb_release -a | grep -q wheezy ; then - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then - /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." - sleep 1s - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - - sudo sh -c 'echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list' - $SUDO apt-get update - fi - fi - $SUDO apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 - augeas_pkg= + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" echo "Let's Encrypt apache plugin..." @@ -209,6 +223,7 @@ BootstrapRpmCommon() { gcc \ dialog \ augeas-libs \ + openssl \ openssl-devel \ libffi-devel \ redhat-rpm-config \ From b03fcee4eeb6594f51b2cc91bf690480e99e4617 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jan 2016 16:32:24 -0800 Subject: [PATCH 113/201] Cache Python packages --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d9b4cb5ea..09e580c3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python +cache: + directories: + - $HOME/.cache/pip + services: - rabbitmq - mariadb From 2d4c21ad4f0a6faa3bfd68c179a4e819739abde6 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 12 Jan 2016 18:16:08 -0500 Subject: [PATCH 114/201] Bring built le-auto script up to date. --- letsencrypt-auto-source/letsencrypt-auto | 51 +++++++++++++++--------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7000d3027..d080d8800 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -123,28 +123,42 @@ BootstrapDebCommon() { virtualenv="$virtualenv python-virtualenv" fi - augeas_pkg=libaugeas0 + augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + + } + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then if lsb_release -a | grep -q wheezy ; then - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then - /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." - sleep 1s - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - - sudo sh -c 'echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list' - $SUDO apt-get update - fi - fi - $SUDO apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 - augeas_pkg= + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" echo "Let's Encrypt apache plugin..." @@ -209,6 +223,7 @@ BootstrapRpmCommon() { gcc \ dialog \ augeas-libs \ + openssl \ openssl-devel \ libffi-devel \ redhat-rpm-config \ From 435dfc0c52c0494cefb01f563686dba304ec8744 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 12 Jan 2016 14:30:33 -0800 Subject: [PATCH 115/201] Undelete the old letsencrypt-auto for now --- bootstrap/_arch_common.sh | 26 +++++ bootstrap/_deb_common.sh | 94 +++++++++++++++++ bootstrap/_gentoo_common.sh | 23 ++++ bootstrap/_rpm_common.sh | 55 ++++++++++ bootstrap/_suse_common.sh | 14 +++ bootstrap/archlinux.sh | 1 + bootstrap/centos.sh | 1 + bootstrap/debian.sh | 1 + bootstrap/fedora.sh | 1 + bootstrap/freebsd.sh | 7 ++ bootstrap/gentoo.sh | 1 + bootstrap/install-deps.sh | 46 ++++++++ bootstrap/mac.sh | 18 ++++ bootstrap/manjaro.sh | 1 + bootstrap/suse.sh | 1 + bootstrap/ubuntu.sh | 1 + letsencrypt-auto | 204 ++++++++++++++++++++++++++++++++++++ 17 files changed, 495 insertions(+) create mode 100755 bootstrap/_arch_common.sh create mode 100755 bootstrap/_deb_common.sh create mode 100755 bootstrap/_gentoo_common.sh create mode 100755 bootstrap/_rpm_common.sh create mode 100755 bootstrap/_suse_common.sh create mode 120000 bootstrap/archlinux.sh create mode 120000 bootstrap/centos.sh create mode 120000 bootstrap/debian.sh create mode 120000 bootstrap/fedora.sh create mode 100755 bootstrap/freebsd.sh create mode 120000 bootstrap/gentoo.sh create mode 100755 bootstrap/install-deps.sh create mode 100755 bootstrap/mac.sh create mode 120000 bootstrap/manjaro.sh create mode 120000 bootstrap/suse.sh create mode 120000 bootstrap/ubuntu.sh create mode 100755 letsencrypt-auto diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh new file mode 100755 index 000000000..2b512792f --- /dev/null +++ b/bootstrap/_arch_common.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Tested with: +# - ArchLinux (x86_64) +# +# "python-virtualenv" is Python3, but "python2-virtualenv" provides +# only "virtualenv2" binary, not "virtualenv" necessary in +# ./bootstrap/dev/_common_venv.sh + +deps=" + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config +" + +missing=$(pacman -T $deps) + +if [ "$missing" ]; then + pacman -S --needed $missing +fi diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh new file mode 100755 index 000000000..c2f58db75 --- /dev/null +++ b/bootstrap/_deb_common.sh @@ -0,0 +1,94 @@ +#!/bin/sh + +# Current version tested with: +# +# - Ubuntu +# - 14.04 (x64) +# - 15.04 (x64) +# - Debian +# - 7.9 "wheezy" (x64) +# - sid (2015-10-21) (x64) + +# Past versions tested with: +# +# - Debian 8.0 "jessie" (x64) +# - Raspbian 7.8 (armhf) + +# Believed not to work: +# +# - Debian 6.0.10 "squeeze" (x64) + +apt-get update + +# virtualenv binary can be found in different packages depending on +# distro version (#346) + +virtualenv= +if apt-cache show virtualenv > /dev/null 2>&1; then + virtualenv="virtualenv" +fi + +if apt-cache show python-virtualenv > /dev/null 2>&1; then + virtualenv="$virtualenv python-virtualenv" +fi + +augeas_pkg="libaugeas0 augeas-lenses" +AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + +AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list + apt-get update + fi + fi + apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + +} + + +if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs +fi + +apt-get install -y --no-install-recommends \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + $augeas_pkg \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + + +if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 +fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh new file mode 100755 index 000000000..f49dc00f0 --- /dev/null +++ b/bootstrap/_gentoo_common.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +PACKAGES=" + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + +case "$PACKAGE_MANAGER" in + (paludis) + cave resolve --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + pmerge --noreplace $PACKAGES + ;; + (portage|*) + emerge --noreplace $PACKAGES + ;; +esac diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh new file mode 100755 index 000000000..db1665268 --- /dev/null +++ b/bootstrap/_rpm_common.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +# Tested with: +# - Fedora 22, 23 (x64) +# - Centos 7 (x64: on DigitalOcean droplet) + +if type dnf 2>/dev/null +then + tool=dnf +elif type yum 2>/dev/null +then + tool=yum + +else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 +fi + +# Some distros and older versions of current distros use a "python27" +# instead of "python" naming convention. Try both conventions. +if ! $tool install -y \ + python \ + python-devel \ + python-virtualenv +then + if ! $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi +fi + +if ! $tool install -y \ + gcc \ + dialog \ + augeas-libs \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates +then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 +fi + + +if $tool list installed "httpd" >/dev/null 2>&1; then + if ! $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi +fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh new file mode 100755 index 000000000..efeebe4f8 --- /dev/null +++ b/bootstrap/_suse_common.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# SLE12 don't have python-virtualenv + +zypper -nq in -l \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates \ diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh new file mode 120000 index 000000000..c5c9479f7 --- /dev/null +++ b/bootstrap/archlinux.sh @@ -0,0 +1 @@ +_arch_common.sh \ No newline at end of file diff --git a/bootstrap/centos.sh b/bootstrap/centos.sh new file mode 120000 index 000000000..a0db46d70 --- /dev/null +++ b/bootstrap/centos.sh @@ -0,0 +1 @@ +_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/debian.sh b/bootstrap/debian.sh new file mode 120000 index 000000000..068a039cb --- /dev/null +++ b/bootstrap/debian.sh @@ -0,0 +1 @@ +_deb_common.sh \ No newline at end of file diff --git a/bootstrap/fedora.sh b/bootstrap/fedora.sh new file mode 120000 index 000000000..a0db46d70 --- /dev/null +++ b/bootstrap/fedora.sh @@ -0,0 +1 @@ +_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh new file mode 100755 index 000000000..4482c35cd --- /dev/null +++ b/bootstrap/freebsd.sh @@ -0,0 +1,7 @@ +#!/bin/sh -xe + +pkg install -Ay \ + python \ + py27-virtualenv \ + augeas \ + libffi \ diff --git a/bootstrap/gentoo.sh b/bootstrap/gentoo.sh new file mode 120000 index 000000000..125d6a592 --- /dev/null +++ b/bootstrap/gentoo.sh @@ -0,0 +1 @@ +_gentoo_common.sh \ No newline at end of file diff --git a/bootstrap/install-deps.sh b/bootstrap/install-deps.sh new file mode 100755 index 000000000..e907e7035 --- /dev/null +++ b/bootstrap/install-deps.sh @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# Install OS dependencies. In the glorious future, letsencrypt-auto will +# source this... + +if test "`id -u`" -ne "0" ; then + SUDO=sudo +else + SUDO= +fi + +BOOTSTRAP=`dirname $0` +if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 +fi +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO $BOOTSTRAP/_deb_common.sh +elif [ -f /etc/arch-release ] ; then + echo "Bootstrapping dependencies for Archlinux..." + $SUDO $BOOTSTRAP/archlinux.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO $BOOTSTRAP/_rpm_common.sh +elif [ -f /etc/gentoo-release ] ; then + echo "Bootstrapping dependencies for Gentoo-based OSes..." + $SUDO $BOOTSTRAP/_gentoo_common.sh +elif uname | grep -iq FreeBSD ; then + echo "Bootstrapping dependencies for FreeBSD..." + $SUDO $BOOTSTRAP/freebsd.sh +elif `grep -qs openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE.." + $SUDO $BOOTSTRAP/suse.sh +elif uname | grep -iq Darwin ; then + echo "Bootstrapping dependencies for Mac OS X..." + echo "WARNING: Mac support is very experimental at present..." + $BOOTSTRAP/mac.sh +else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info" + exit 1 +fi diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh new file mode 100755 index 000000000..4d1fb8208 --- /dev/null +++ b/bootstrap/mac.sh @@ -0,0 +1,18 @@ +#!/bin/sh -e +if ! hash brew 2>/dev/null; then + echo "Homebrew Not Installed\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + +brew install augeas +brew install dialog + +if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python +fi + +if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv +fi diff --git a/bootstrap/manjaro.sh b/bootstrap/manjaro.sh new file mode 120000 index 000000000..c5c9479f7 --- /dev/null +++ b/bootstrap/manjaro.sh @@ -0,0 +1 @@ +_arch_common.sh \ No newline at end of file diff --git a/bootstrap/suse.sh b/bootstrap/suse.sh new file mode 120000 index 000000000..fc4c1dee4 --- /dev/null +++ b/bootstrap/suse.sh @@ -0,0 +1 @@ +_suse_common.sh \ No newline at end of file diff --git a/bootstrap/ubuntu.sh b/bootstrap/ubuntu.sh new file mode 120000 index 000000000..068a039cb --- /dev/null +++ b/bootstrap/ubuntu.sh @@ -0,0 +1 @@ +_deb_common.sh \ No newline at end of file diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 100755 index 000000000..20465dbb1 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1,204 @@ +#!/bin/sh -e +# +# A script to run the latest release version of the Let's Encrypt in a +# virtual environment +# +# Installs and updates the letencrypt virtualenv, and runs letsencrypt +# using that virtual environment. This allows the client to function decently +# without requiring specific versions of its dependencies from the operating +# system. + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN=${VENV_PATH}/bin +# The path to the letsencrypt-auto script. Everything that uses these might +# at some point be inlined... +LEA_PATH=`dirname "$0"` +BOOTSTRAP=${LEA_PATH}/bootstrap + +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then + VERBOSE=1 + elif [ "$arg" = "--debug" ] ; then + DEBUG=1 + fi +done + +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +ExperimentalBootstrap() { + # Arguments: Platform name, boostrap script name, SUDO command (iff needed) + if [ "$DEBUG" = 1 ] ; then + if [ "$2" != "" ] ; then + echo "Bootstrapping dependencies for $1..." + if [ "$3" != "" ] ; then + "$3" "$BOOTSTRAP/$2" + else + "$BOOTSTRAP/$2" + fi + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + if command -v python2.7 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2.7} + elif command -v python27 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python27} + elif command -v python2 > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python2} + elif command -v python > /dev/null ; then + export LE_PYTHON=${LE_PYTHON:-python} + else + echo "Cannot find any Pythons... please install one!" + exit 1 + fi + + PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ $PYVER -eq 26 ] ; then + ExperimentalBootstrap "Python 2.6" + elif [ $PYVER -lt 26 ] ; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -d $VENV_PATH ] +then + if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 + fi + + if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO $BOOTSTRAP/_deb_common.sh + elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO $BOOTSTRAP/_rpm_common.sh + elif `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + $SUDO $BOOTSTRAP/_suse_common.sh + elif [ -f /etc/arch-release ] ; then + if [ "$DEBUG" = 1 ] ; then + echo "Bootstrapping dependencies for Archlinux..." + $SUDO $BOOTSTRAP/archlinux.sh + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ] ; then + ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" + elif [ -f /etc/gentoo-release ] ; then + ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info" + fi + + DeterminePythonVersion + echo "Creating virtual environment..." + if [ "$VERBOSE" = 1 ] ; then + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH + else + virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null + fi +else + DeterminePythonVersion +fi + + +printf "Updating letsencrypt and virtual environment dependencies..." +if [ "$VERBOSE" = 1 ] ; then + echo + $VENV_BIN/pip install -U setuptools + $VENV_BIN/pip install -U pip + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache + # nginx is buggy / disabled for now, but upgrade it if the user has + # installed it manually + if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then + $VENV_BIN/pip install -U letsencrypt letsencrypt-nginx + fi +else + $VENV_BIN/pip install -U setuptools > /dev/null + printf . + $VENV_BIN/pip install -U pip > /dev/null + printf . + # nginx is buggy / disabled for now... + $VENV_BIN/pip install -U letsencrypt > /dev/null + printf . + $VENV_BIN/pip install -U letsencrypt-apache > /dev/null + if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then + printf . + $VENV_BIN/pip install -U letsencrypt-nginx > /dev/null + fi + echo +fi + +# Explain what's about to happen, for the benefit of those getting sudo +# password prompts... +echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" +$SUDO $VENV_BIN/letsencrypt "$@" From e192cce1fce3eab3ab37e35a55c713fd6b368b7e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jan 2016 18:43:34 -0800 Subject: [PATCH 116/201] Fix fake letsencrypt --- letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py index 370f70c51..9d811fab5 100755 --- a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py +++ b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py @@ -4,5 +4,5 @@ from sys import argv, stderr def main(): """Act like letsencrypt --version insofar as printing the version number to stderr.""" - if len(argv) >= 2 and argv[1] == '--version': + if '--version' in argv: stderr.write('letsencrypt 99.9.9\n') From 7945db7a2d1fab129e32be91a5f3c795fa32a9c3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jan 2016 18:44:02 -0800 Subject: [PATCH 117/201] Rebuild sdist --- .../dist/letsencrypt-99.9.9.tar.gz | Bin 899 -> 876 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz index fb6d62477154ac7cb693a42b211fdcbe967879f3..5f9a48a34a22ca70c28ac318336ba986ea32b60e 100644 GIT binary patch delta 838 zcmV-M1G)Tz2kZt1ABzYG<++uS2O)o3PunmQfcxxUVet}>NJ*NTsMH6vyVzPZ#P-0d zC^Sy-NbJaQ%c^PreNIB#Gy@8>WScl&h0@r$7>>V_6B84e5jGODRO*L^ZS^^5Z5u=3 zIZnPcJ;$hS=d{d@Yde-@d$u~xwA_x-!cMCJCYcNb#;uFw`rhN#_0NBi`SyQ5A^tP| zm8Nz&Yasqs$6ChU$m8!iw$;K0#Q)ie1W#}_%P>t+E+h`bc)E`>84@87LLrH85>U3g z*F{*dv$JzFk~pSU1Xtz>*7Ye78RZO9mhnjaMS~2df-*_j7|Tn7=lyJwTo8dJFRoYK zsI4wr?9vEpi%V(hTE%$ETUUP$zZO)IU9CERa_)X(+8$~F8~L9RIZ3x;{!$Isw(s=c zwif;yp6e<9)wd4#-<#3kqPA`KKR%Ku`!TlmLWLg)moQ`|dfpqgkrwxmG(t81ohvY{)Z$bEL6=iKc?&| zdrTTw>%U>v{I@*I2LFGX0Hg>1AL9S%;Oz6OH>Xc{`mgujGVA`Ej@p3#&BkB7@}v2? zKe}}ioE*Q?KlI-Y(7Gvt9{dcs{mAP3+}%fQQj-FY^S>UEt=9i;r~dpmU0DBVg!9@G z$q?PYK7ycNc#;0kQ~y)ff9`$$J5J4i!^_uy!2jpx6Olm}$UuMpQZBf5vDr3IKS)To z>ONX@my-mYPLd=Lvo3!6+Aey}6QZZVSl#NZf7~B@?W3cKyyT+3uM?Ge;$`bcIwCBq zoc4OT+PrK$i31sNkt|z4_lNkuiR#<&-?Y8_|Nr1W_z(U!?tl3XIEuzi`EPhu&40(X zP4K^w`d?LeeqJHr4Ur7S{>6mGp(^`}9hDb|`fe0E*_8Q$Ole_x-Uv1UlVJlElW+qE Q4Ezm#16P1_umDg108rDr&Hw-a delta 881 zcmV-%1CIRc27?C&ABzYGQ7@E{2O)oJZ<{a_hWYGYVf72C)TBUwsF9dd?XGTVwW`~P zEJ8Gxlof1boNUpw|2_k0LRghsQ`W8Li6Y15W;lNB_k@Uul+YocCZc(A(vqKJW#4F$ zj%5|4U6h((7G;5=8@6d_M#nOAMbk~wYAe`MYG9lSk7KM5&Qh{sKe<~Fths;5>_+a& ze}w4I_?MbmWHf4qG+@WI#^2Dg0^fM!6P|{Q zcCp^lP|u4=x2%0M-&_n5bUBV=k59Y!6vu3Mizmf)%!2Y^SloC5f0NG`Cp=p^a45gC zebGA>lV;+LEBHMde#3+1E7KaYmR>_nWB`4agzO5xd@=rhB-?y8nOM)?)oP z^|qqv9aC$A{_7y%EXIH7R2uU*VO(I3k0wVr6+Yn{A>@+)XK8=qu#2$d;NakNC~y?s zk|m$tY~CdRAx&8zpQx7>mOrE;nFWw}fR_h+jC^ zz#jGUKK1nhb{wqEFQgV7DxKsz4+Uwcy~AK{vrQ| z{Qqz0zsLcH!Kg0%Yn^sQ|F$#&p#M7P%M8!WGQ0s2-pIWkhmnsRoLA($K;&;NucQ;| zaxw{Xm1m8hrvF<1dvX4o>ECS6^>131k?CKzA^)$@|7Q^wj79S_Aa9ZWKQ9}()27lB zIAwRO9_8(`)!boGi{UeC^%pC$qwekzW@k&LZp{2~L{g`})P+H@z5IWn^K+gUsFjzS zY8NjZe$V?&b&wzZ3qi&J00000000000000000000000000000000000z!T*+!vj@6 H08jt`rHsBB From ab0762050490cc0243582d37bd9c0d49452886db Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 12 Jan 2016 18:50:52 -0800 Subject: [PATCH 118/201] Fixed fake letsencrypt hash --- letsencrypt-auto-source/tests/auto_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 4fa4b4e27..ae86fdcbc 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -267,7 +267,7 @@ class AutoTests(TestCase): """ NEW_LE_AUTO = build_le_auto( version='99.9.9', - requirements='# sha256: 7NpInQZj4v2dvdCBUYtcBHqVlBfnUmlsKF_oSOzU9zY\n' + requirements='# sha256: HMFNYatCTN7kRvUeUPESP4SC7HQFh_54YmyTO7ooc6A\n' 'letsencrypt==99.9.9') NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) From 86266f5fe1d25e1ab0dd883126fd0c7ec82f295e Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 13 Jan 2016 11:36:49 -0500 Subject: [PATCH 119/201] Remove backported Python 2.7 assertion helpers. I didn't backport their imports, so they had NameErrors in the failure case anyway. And, because of the docker image, these tests currently are run under only 2.7 at the moment. --- letsencrypt-auto-source/tests/auto_test.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 4fa4b4e27..92098a8ef 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -235,21 +235,6 @@ class AutoTests(TestCase): test suites. """ - # Remove these helpers when we no longer need to support Python 2.6: - def assertIn(self, member, container, msg=None): - """Just like self.assertTrue(a in b), but with a nicer default message.""" - if member not in container: - standardMsg = '%s not found in %s' % (safe_repr(member), - safe_repr(container)) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertNotIn(self, member, container, msg=None): - """Just like self.assertTrue(a not in b), but with a nicer default message.""" - if member in container: - standardMsg = '%s unexpectedly found in %s' % (safe_repr(member), - safe_repr(container)) - self.fail(self._formatMessage(msg, standardMsg)) - def test_successes(self): """Exercise most branches of letsencrypt-auto. From 587e2e76f3034d8fefce6acd54bee45dc46284dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jan 2016 13:25:29 -0800 Subject: [PATCH 120/201] Revert "Get all tests, even le_auto, working on Travis." This reverts commit 7ee23b723af6b7cb62f23f2bfdc2e472d80f89ec. --- .travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ab81f20b2..d9b4cb5ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,13 @@ language: python services: - rabbitmq + - mariadb # apacheconftest #- apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq mysql-server-5.6 mysql-client-5.6 mysql-client-core-5.6 - 'dpkg -s libaugeas0' - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5.1)"' @@ -25,7 +24,6 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=py26-oldest BOULDER_INTEGRATION=1 - TOXENV=py27-oldest BOULDER_INTEGRATION=1 - - TOXENV=le_auto - TOXENV=py33 - TOXENV=py34 - TOXENV=lint @@ -47,13 +45,14 @@ branches: - master - /^test-.*$/ -sudo: required -dist: trusty +# container-based infrastructure +sudo: false addons: # make sure simplehttp simple verification works (custom /etc/hosts) hosts: - le.wtf + mariadb: "10.0" apt: sources: - augeas From a1f6678d6108420d50895318cd2777bab87899b4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jan 2016 13:26:16 -0800 Subject: [PATCH 121/201] Revert changes to Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 67043edd3..da0110604 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto -RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ +COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh +RUN /opt/letsencrypt/src/ubuntu.sh && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ From a287b504a5e4de3ab9f09209c3090c67bab27b68 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jan 2016 13:26:36 -0800 Subject: [PATCH 122/201] Fix Vagrantfile path --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 4a603c2ce..c1e05d215 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,7 +7,7 @@ VAGRANTFILE_API_VERSION = "2" # Setup instructions from docs/contributing.rst $ubuntu_setup_script = < Date: Wed, 13 Jan 2016 14:05:22 -0500 Subject: [PATCH 123/201] enable config_test in configurator prepare --- .../letsencrypt_nginx/configurator.py | 31 ++++++--------- .../tests/configurator_test.py | 15 ++++--- .../letsencrypt_nginx/tests/util.py | 39 ++++++++++--------- letsencrypt/tests/cli_test.py | 3 +- 4 files changed, 42 insertions(+), 46 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 4a5a3ddcd..efa7e08b4 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -5,7 +5,6 @@ import re import shutil import socket import subprocess -import sys import time import OpenSSL @@ -106,11 +105,18 @@ class NginxConfigurator(common.Plugin): # This is called in determine_authenticator and determine_installer def prepare(self): - """Prepare the authenticator/installer.""" + """Prepare the authenticator/installer. + + :raises .errors.NoInstallationError: If Nginx ctl cannot be found + :raises .errors.MisconfigurationError: If Nginx is misconfigured + """ # Verify Nginx is installed if not le_util.exe_exists(self.conf('ctl')): raise errors.NoInstallationError + # Make sure configuration is valid + self.config_test() + self.parser = parser.NginxParser( self.conf('server-root'), self.mod_ssl_conf) @@ -409,26 +415,13 @@ class NginxConfigurator(common.Plugin): def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If config_test fails """ try: - proc = subprocess.Popen( - [self.conf('ctl'), "-c", self.nginx_conf, "-t"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - except (OSError, ValueError): - logger.fatal("Unable to run nginx config test") - sys.exit(1) - - if proc.returncode != 0: - # Enter recovery routine... - logger.error("Config test failed\n%s\n%s", stdout, stderr) - return False - - return True + le_util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"]) + except errors.SubprocessError as err: + raise errors.MisconfigurationError(str(err)) def _verify_setup(self): """Verify the setup to ensure safe operating environment. diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index f9af5183a..4fce33079 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -54,6 +54,7 @@ class NginxConfiguratorTest(util.NginxTest): mock_exe_exists.return_value = True self.config.version = None + self.config.config_test = mock.Mock() self.config.prepare() self.assertEquals((1, 6, 2), self.config.version) @@ -361,12 +362,14 @@ class NginxConfiguratorTest(util.NginxTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") - def test_config_test(self, mock_popen): - mocked = mock_popen() - mocked.communicate.return_value = ('', '') - mocked.returncode = 0 - self.assertTrue(self.config.config_test()) + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test(self, _): + self.config.config_test() + + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test_bad_process(self, mock_run_script): + mock_run_script.side_effect = errors.SubprocessError + self.assertRaises(errors.MisconfigurationError, self.config.config_test) def test_get_snakeoil_paths(self): # pylint: disable=protected-access diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index 3d70f7ac7..7a16e3738 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -49,25 +49,26 @@ def get_nginx_configurator( backups = os.path.join(work_dir, "backups") - with mock.patch("letsencrypt_nginx.configurator.le_util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - - config = configurator.NginxConfigurator( - config=mock.MagicMock( - nginx_server_root=config_path, - le_vhost_ext="-le-ssl.conf", - config_dir=config_dir, - work_dir=work_dir, - backup_dir=backups, - temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), - in_progress_dir=os.path.join(backups, "IN_PROGRESS"), - server="https://acme-server.org:443/new", - tls_sni_01_port=5001, - ), - name="nginx", - version=version) - config.prepare() + with mock.patch("letsencrypt_nginx.configurator.NginxConfigurator." + "config_test"): + with mock.patch("letsencrypt_nginx.configurator.le_util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + config = configurator.NginxConfigurator( + config=mock.MagicMock( + nginx_server_root=config_path, + le_vhost_ext="-le-ssl.conf", + config_dir=config_dir, + work_dir=work_dir, + backup_dir=backups, + temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), + in_progress_dir=os.path.join(backups, "IN_PROGRESS"), + server="https://acme-server.org:443/new", + tls_sni_01_port=5001, + ), + name="nginx", + version=version) + config.prepare() # Provide general config utility. nsconfig = configuration.NamespaceConfig(config.config) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 39c09dede..16ef5c093 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -202,8 +202,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods # (we can only do that if letsencrypt-nginx is actually present) ret, _, _, _ = self._call(args) self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("Could not find configuration root" in ret) - self.assertTrue("NoInstallationError" in ret) + self.assertTrue("MisconfigurationError" in ret) args = ["certonly", "--webroot"] ret, _, _, _ = self._call(args) From 99c575f04366d76cb195d7cd912714f3a0be7ede Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 13 Jan 2016 23:56:22 +0200 Subject: [PATCH 124/201] Check augeas version, and raise error if not recent enough --- .../letsencrypt_apache/configurator.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 4066d6264..00b93bb6e 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -158,6 +158,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) + if not self._check_aug_version(): + raise errors.NotSupportedError( + "Your libaugeas0 is outdated, upgrade it from backports " + " or re-bootstrap letsencrypt") + self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version) @@ -169,6 +174,17 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): install_ssl_options_conf(self.mod_ssl_conf) + def _check_aug_version(self): + """ Checks that we have recent enough version of libaugeas. + If augeas version is recent enough, it will support case insensitive + regexp matching""" + + self.aug.set("/test/path/testing/arg", "aRgUMeNT") + matches = self.aug.match( + "/test//*[self::arg=~regexp('argument', 'i')]") + self.aug.remove("/test/path") + return matches + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. From bccb2124bc23063f7a5a8688b8f9a1f3f747c7d3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jan 2016 14:05:19 -0800 Subject: [PATCH 125/201] Fix paths in contributing.rst --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 99fe0dad5..dc1580041 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -22,7 +22,7 @@ once: git clone https://github.com/letsencrypt/letsencrypt cd letsencrypt - ./letsencrypt-auto/letsencrypt-auto --os-packages-only + ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only ./bootstrap/dev/venv.sh Then in each shell where you're working on the client, do: @@ -369,7 +369,7 @@ OS-level dependencies can be installed like so: .. code-block:: shell - letsencrypt-auto/letsencrypt-auto --os-packages-only + letsencrypt-auto-source/letsencrypt-auto --os-packages-only In general... From 30ad7dce9fb74f1fd8a401d16641fe8b49b1e06d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 00:06:52 +0200 Subject: [PATCH 126/201] Pick up the augeas RuntimeError and pass the correct one --- letsencrypt-apache/letsencrypt_apache/configurator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 00b93bb6e..0c4d65c28 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -180,8 +180,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): regexp matching""" self.aug.set("/test/path/testing/arg", "aRgUMeNT") - matches = self.aug.match( - "/test//*[self::arg=~regexp('argument', 'i')]") + try: + matches = self.aug.match( + "/test//*[self::arg=~regexp('argument', 'i')]") + except RuntimeError: + return None self.aug.remove("/test/path") return matches From c3ea4bdc9b8a6014188d4a5d7559562e5ffd1083 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 13 Jan 2016 17:22:59 -0500 Subject: [PATCH 127/201] Roll back change to acme's pylintrc, which was needed to get lint to pass on Travis's Trusty beta (sudo) infra. We're stepping off that infra briefly, to keep it the same as boulder's. When we retire the old le-auto, we'll step back on and change boulder to use it as well. --- acme/.pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/.pylintrc b/acme/.pylintrc index dcb8af713..33650310d 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -60,7 +60,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=fixme,locally-disabled,abstract-class-not-used,too-many-instance-attributes +disable=fixme,locally-disabled,abstract-class-not-used # bstract-class-not-used cannot be disabled locally (at least in # pylint 1.4.1/2) From 7d51480c4dd8610c40d8e4aa4e78024e12e46e63 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 00:23:45 +0200 Subject: [PATCH 128/201] Remove the test path from augeas even if failing --- letsencrypt-apache/letsencrypt_apache/configurator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 0c4d65c28..72db9c853 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -184,6 +184,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): matches = self.aug.match( "/test//*[self::arg=~regexp('argument', 'i')]") except RuntimeError: + self.aug.remove("/test/path") return None self.aug.remove("/test/path") return matches From 25e428ce4b71f4ceba47b01629b0297172209a38 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 13 Jan 2016 17:27:47 -0500 Subject: [PATCH 129/201] Bring built le-auto up to date again. --- letsencrypt-auto-source/letsencrypt-auto | 45 +++++++++++++++--------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d080d8800..dac8f3ef9 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -2,8 +2,14 @@ # # Download and run the latest release version of the Let's Encrypt client. # -# WARNING: "letsencrypt-auto" IS A GENERATED FILE. EDIT -# letsencrypt-auto.template INSTEAD. +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* set -e # Work even if somebody does "sh thisscript.sh". @@ -15,6 +21,24 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin LE_AUTO_VERSION="0.2.0.dev0" +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then + VERBOSE=1 + elif [ "$arg" = "--no-self-upgrade" ] ; then + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1 + elif [ "$arg" = "--os-packages-only" ] ; then + OS_PACKAGES_ONLY=1 + elif [ "$arg" = "--debug" ]; then + DEBUG=1 + fi +done + # letsencrypt-auto needs root access to bootstrap OS dependencies, and # letsencrypt itself needs root access for almost all modes of operation # The "normal" case is that sudo is used for the steps that need root, but @@ -383,22 +407,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X } -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) -for arg in "$@" ; do - # This first clause is redundant with the third, but hedging on portability - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then - VERBOSE=1 - elif [ "$arg" = "--debug" ]; then - DEBUG=1 - fi -done -if [ "$1" = "--no-self-upgrade" ]; then + +if [ "$NO_SELF_UPGRADE" = 1 ]; then # Phase 2: Create venv, install LE, and run. - shift 1 # the --no-self-upgrade arg if [ -f "$VENV_BIN/letsencrypt" ]; then INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) else @@ -1630,7 +1643,7 @@ else # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi - if [ "$1" = "--os-packages-only" ]; then + if [ "$OS_PACKAGES_ONLY" = 1 ]; then echo "OS packages installed." exit 0 fi From ddbfb440412187447d4d51f0f7ac03025954aebf Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 00:50:34 +0200 Subject: [PATCH 130/201] Add tests --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- .../letsencrypt_apache/tests/configurator_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 72db9c853..ad407a3bc 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -185,7 +185,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "/test//*[self::arg=~regexp('argument', 'i')]") except RuntimeError: self.aug.remove("/test/path") - return None + return False self.aug.remove("/test/path") return matches diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 113b3b2b2..ce3d20297 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -978,6 +978,13 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:443", exclude=False)) + def test_aug_version(self): + mock_match = mock.Mock(return_value=["something"]) + self.config.aug.match = mock_match + self.assertEquals(self.config._check_aug_version(), ["something"]) + self.config.aug.match.side_effect = RuntimeError + self.assertFalse(self.config._check_aug_version()) + if __name__ == "__main__": unittest.main() # pragma: no cover From d0832f741462cfa94046d7ed7a81835a8836e1e8 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 01:09:28 +0200 Subject: [PATCH 131/201] Added the missing test --- .../letsencrypt_apache/tests/configurator_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index ce3d20297..fe7071f14 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -65,6 +65,15 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises( errors.NotSupportedError, self.config.prepare) + @mock.patch("letsencrypt_apache.parser.ApacheParser") + @mock.patch("letsencrypt_apache.configurator.le_util.exe_exists") + def test_prepare_old_aug(self, mock_exe_exists, _): + mock_exe_exists.return_value = True + self.config._check_aug_version = mock.Mock(return_value=False) + self.assertRaises( + errors.NotSupportedError, self.config.prepare) + + def test_add_parser_arguments(self): # pylint: disable=no-self-use from letsencrypt_apache.configurator import ApacheConfigurator # Weak test.. From 8f6ef8db5301ce4087a1de86eeaaaeda805f93f9 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 01:10:50 +0200 Subject: [PATCH 132/201] Modified error message --- letsencrypt-apache/letsencrypt_apache/configurator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index ad407a3bc..47d238140 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -160,8 +160,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self._check_aug_version(): raise errors.NotSupportedError( - "Your libaugeas0 is outdated, upgrade it from backports " - " or re-bootstrap letsencrypt") + "Apache plugin support requires libaugeas0 and augeas-lenses " + "version 1.2.0 or higher, please make sure you have you have " + "those installed.") self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.conf("vhost-root"), From 8c110e31d75a746ec6bb284bba61179140d445a9 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 01:30:34 +0200 Subject: [PATCH 133/201] Fixed tests --- .../letsencrypt_apache/tests/configurator_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index fe7071f14..5052da893 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -69,11 +69,11 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.configurator.le_util.exe_exists") def test_prepare_old_aug(self, mock_exe_exists, _): mock_exe_exists.return_value = True + self.config.config_test = mock.Mock() self.config._check_aug_version = mock.Mock(return_value=False) self.assertRaises( errors.NotSupportedError, self.config.prepare) - def test_add_parser_arguments(self): # pylint: disable=no-self-use from letsencrypt_apache.configurator import ApacheConfigurator # Weak test.. From fe8a0dcef26c15480ce0ae86f8f24450836e14a6 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 01:52:35 +0200 Subject: [PATCH 134/201] Make linter happy --- .../letsencrypt_apache/tests/configurator_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 5052da893..2a32b04be 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -70,7 +70,7 @@ class TwoVhost80Test(util.ApacheTest): def test_prepare_old_aug(self, mock_exe_exists, _): mock_exe_exists.return_value = True self.config.config_test = mock.Mock() - self.config._check_aug_version = mock.Mock(return_value=False) + self.config._check_aug_version = mock.Mock(return_value=False) # pylint: disable=protected-access self.assertRaises( errors.NotSupportedError, self.config.prepare) @@ -990,9 +990,9 @@ class TwoVhost80Test(util.ApacheTest): def test_aug_version(self): mock_match = mock.Mock(return_value=["something"]) self.config.aug.match = mock_match - self.assertEquals(self.config._check_aug_version(), ["something"]) + self.assertEquals(self.config._check_aug_version(), ["something"]) # pylint: disable=protected-access self.config.aug.match.side_effect = RuntimeError - self.assertFalse(self.config._check_aug_version()) + self.assertFalse(self.config._check_aug_version()) # pylint: disable=protected-access if __name__ == "__main__": From 8989dfc1ff7533fb961c237eadbee76cae70c049 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 13 Jan 2016 16:17:26 -0800 Subject: [PATCH 135/201] Disable Apache 2.2 support --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 4066d6264..2ce9d008b 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -154,7 +154,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: self.version = self.get_version() - if self.version < (2, 2): + if self.version < (2, 4): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) From a7b878b825e73b0d05abd0495d40eded7ae2d608 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 13 Jan 2016 16:51:13 -0800 Subject: [PATCH 136/201] Ensure that all pip upload version #s are reflect in git --- tools/release.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/release.sh b/tools/release.sh index 172f6fea1..a945e3970 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -85,9 +85,12 @@ SetVersion() { sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py + + # interactive user input + git add -p letsencrypt $SUBPKGS letsencrypt-compatibility-test - git add -p letsencrypt $SUBPKGS # interactive user input } + SetVersion "$version" git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" git tag --local-user "$RELEASE_GPG_KEY" \ From 4762ede4ea901abcc6ff0240fc84d7ba80763987 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 13 Jan 2016 17:09:45 -0800 Subject: [PATCH 137/201] Also *set* the letsencrypt-compatibility-test version number --- tools/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/release.sh b/tools/release.sh index a945e3970..43c7556de 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -80,7 +80,7 @@ git checkout "$RELEASE_BRANCH" SetVersion() { ver="$1" - for pkg_dir in $SUBPKGS + for pkg_dir in $SUBPKGS letsencrypt-compatibility-test do sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done From be1d1321b65527841eb85353aeb350a07101c369 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 13:25:15 +0200 Subject: [PATCH 138/201] Pep8 love --- .../letsencrypt_apache/augeas_configurator.py | 3 +- .../letsencrypt_apache/configurator.py | 97 +++++++----- .../letsencrypt_apache/constants.py | 9 +- .../letsencrypt_apache/display_ops.py | 5 +- .../letsencrypt_apache/parser.py | 18 +-- .../tests/complex_parsing_test.py | 6 +- .../tests/configurator_test.py | 142 ++++++++++-------- .../letsencrypt_apache/tests/obj_test.py | 3 +- .../letsencrypt_apache/tests/parser_test.py | 3 +- .../tests/tls_sni_01_test.py | 13 +- .../letsencrypt_apache/tests/util.py | 9 +- .../letsencrypt_apache/tls_sni_01.py | 4 +- 12 files changed, 182 insertions(+), 130 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 9e0948f12..9b51c32a9 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -120,7 +120,8 @@ class AugeasConfigurator(common.Plugin): self.reverter.add_to_temp_checkpoint( save_files, self.save_notes) else: - self.reverter.add_to_checkpoint(save_files, self.save_notes) + self.reverter.add_to_checkpoint(save_files, + self.save_notes) except errors.ReverterError as err: raise errors.PluginError(str(err)) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index bfc6a566c..2d822b3a1 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -133,7 +133,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" - return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST) + return os.path.join(self.config.config_dir, + constants.MOD_SSL_CONF_DEST) def prepare(self): """Prepare the authenticator/installer. @@ -191,15 +192,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return matches def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): # pylint: disable=unused-argument + chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. Currently tries to find the last directives to deploy the cert in the VHost associated with the given domain. If it can't find the - directives, it searches the "included" confs. The function verifies that - it has located the three directives and finally modifies them to point - to the correct destination. After the certificate is installed, the - VirtualHost is enabled if it isn't already. + directives, it searches the "included" confs. The function verifies + that it has located the three directives and finally modifies them + to point to the correct destination. After the certificate is + installed, the VirtualHost is enabled if it isn't already. .. todo:: Might be nice to remove chain directive if none exists This shouldn't happen within letsencrypt though @@ -215,8 +216,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # cert_key... can all be parsed appropriately self.prepare_server_https("443") - path = {"cert_path": self.parser.find_dir("SSLCertificateFile", None, vhost.path), - "cert_key": self.parser.find_dir("SSLCertificateKeyFile", None, vhost.path)} + path = {"cert_path": self.parser.find_dir("SSLCertificateFile", + None, vhost.path), + "cert_key": self.parser.find_dir("SSLCertificateKeyFile", + None, vhost.path)} # Only include if a certificate chain is specified if chain_path is not None: @@ -246,7 +249,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir(vhost.path, "SSLCertificateChainFile", chain_path) else: - raise errors.PluginError("--chain-path is required for your version of Apache") + raise errors.PluginError("--chain-path is required for your " + "version of Apache") else: if not fullchain_path: raise errors.PluginError("Please provide the --fullchain-path\ @@ -320,7 +324,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): elif not vhost.ssl: addrs = self._get_proposed_addrs(vhost, "443") # TODO: Conflicts is too conservative - if not any(vhost.enabled and vhost.conflicts(addrs) for vhost in self.vhosts): + if not any(vhost.enabled and vhost.conflicts(addrs) for + vhost in self.vhosts): vhost = self.make_vhost_ssl(vhost) else: logger.error( @@ -588,15 +593,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo - listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] + listens = [self.parser.get_arg(x).split()[0] for + x in self.parser.find_dir("Listen")] # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] if port in listens: return for listen in listens: - # For any listen statement, check if the machine also listens on Port 443. - # If not, add such a listen statement. + # For any listen statement, check if the machine also listens on + # Port 443. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces if port not in listens: @@ -624,8 +630,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir_to_ifmodssl( parser.get_aug_path( self.parser.loc["listen"]), "Listen", args) - self.save_notes += "Added Listen %s:%s directive to %s\n" % ( - ip, port, self.parser.loc["listen"]) + self.save_notes += ("Added Listen %s:%s directive to " + "%s\n") % (ip, port, + self.parser.loc["listen"]) listens.append("%s:%s" % (ip, port)) def prepare_https_modules(self, temp): @@ -824,20 +831,25 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _clean_vhost(self, vhost): # remove duplicated or conflicting ssl directives self._deduplicate_directives(vhost.path, - ["SSLCertificateFile", "SSLCertificateKeyFile"]) + ["SSLCertificateFile", + "SSLCertificateKeyFile"]) # remove all problematic directives self._remove_directives(vhost.path, ["SSLCertificateChainFile"]) def _deduplicate_directives(self, vh_path, directives): for directive in directives: - while len(self.parser.find_dir(directive, None, vh_path, False)) > 1: - directive_path = self.parser.find_dir(directive, None, vh_path, False) + while len(self.parser.find_dir(directive, None, + vh_path, False)) > 1: + directive_path = self.parser.find_dir(directive, None, + vh_path, False) self.aug.remove(re.sub(r"/\w*$", "", directive_path[0])) def _remove_directives(self, vh_path, directives): for directive in directives: - while len(self.parser.find_dir(directive, None, vh_path, False)) > 0: - directive_path = self.parser.find_dir(directive, None, vh_path, False) + while len(self.parser.find_dir(directive, None, + vh_path, False)) > 0: + directive_path = self.parser.find_dir(directive, None, + vh_path, False) self.aug.remove(re.sub(r"/\w*$", "", directive_path[0])) def _add_dummy_ssl_directives(self, vh_path): @@ -864,7 +876,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for addr in vhost.addrs: for test_vh in self.vhosts: if (vhost.filep != test_vh.filep and - any(test_addr == addr for test_addr in test_vh.addrs) and + any(test_addr == addr for + test_addr in test_vh.addrs) and not self.is_name_vhost(addr)): self.add_name_vhost(addr) logger.info("Enabling NameVirtualHosts on %s", addr) @@ -873,9 +886,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if need_to_save: self.save() - ############################################################################ + ###################################################################### # Enhancements - ############################################################################ + ###################################################################### def supported_enhancements(self): # pylint: disable=no-self-use """Returns currently supported enhancements.""" return ["redirect", "ensure-http-header"] @@ -936,14 +949,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add directives to server self.parser.add_dir(ssl_vhost.path, "Header", - constants.HEADER_ARGS[header_substring]) + constants.HEADER_ARGS[header_substring]) self.save_notes += ("Adding %s header to ssl vhost in %s\n" % - (header_substring, ssl_vhost.filep)) + (header_substring, ssl_vhost.filep)) self.save() logger.info("Adding %s header to ssl vhost in %s", header_substring, - ssl_vhost.filep) + ssl_vhost.filep) def _verify_no_matching_http_header(self, ssl_vhost, header_substring): """Checks to see if an there is an existing Header directive that @@ -963,14 +976,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): header_substring exists """ - header_path = self.parser.find_dir("Header", None, start=ssl_vhost.path) + header_path = self.parser.find_dir("Header", None, + start=ssl_vhost.path) if header_path: # "Existing Header directive for virtualhost" pat = '(?:[ "]|^)(%s)(?:[ "]|$)' % (header_substring.lower()) for match in header_path: if re.search(pat, self.aug.get(match).lower()): raise errors.PluginEnhancementAlreadyPresent( - "Existing %s header" % (header_substring)) + "Existing %s header" % (header_substring)) def _enable_redirect(self, ssl_vhost, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -1019,7 +1033,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check if LetsEncrypt redirection already exists self._verify_no_letsencrypt_redirect(general_vh) - # Note: if code flow gets here it means we didn't find the exact # letsencrypt RewriteRule config for redirection. Finding # another RewriteRule is likely to be fine in most or all cases, @@ -1038,10 +1051,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if self.get_version() >= (2, 3, 9): self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS_WITH_END) + constants.REWRITE_HTTPS_ARGS_WITH_END) else: self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS) + constants.REWRITE_HTTPS_ARGS) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % (general_vh.filep, ssl_vhost.filep)) @@ -1063,7 +1076,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): letsencrypt redirection WriteRule exists in virtual host. """ rewrite_path = self.parser.find_dir( - "RewriteRule", None, start=vhost.path) + "RewriteRule", None, start=vhost.path) # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value @@ -1078,12 +1091,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if rewrite_args_dict: redirect_args = [constants.REWRITE_HTTPS_ARGS, - constants.REWRITE_HTTPS_ARGS_WITH_END] + constants.REWRITE_HTTPS_ARGS_WITH_END] for matches in rewrite_args_dict.values(): if [self.aug.get(x) for x in matches] in redirect_args: raise errors.PluginEnhancementAlreadyPresent( - "Let's Encrypt has already enabled redirection") + "Let's Encrypt has already enabled redirection") def _is_rewrite_exists(self, vhost): """Checks if there exists a RewriteRule directive in vhost @@ -1096,7 +1109,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ rewrite_path = self.parser.find_dir( - "RewriteRule", None, start=vhost.path) + "RewriteRule", None, start=vhost.path) return bool(rewrite_path) def _is_rewrite_engine_on(self, vhost): @@ -1107,7 +1120,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ rewrite_engine_path = self.parser.find_dir("RewriteEngine", "on", - start=vhost.path) + start=vhost.path) if rewrite_engine_path: return self.parser.get_arg(rewrite_engine_path[0]) return False @@ -1153,7 +1166,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): else: rewrite_rule_args = constants.REWRITE_HTTPS_ARGS - return ("\n" "%s \n" "%s \n" @@ -1165,7 +1177,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ErrorLog /var/log/apache2/redirect.error.log\n" "LogLevel warn\n" "\n" - % (" ".join(str(addr) for addr in self._get_proposed_addrs(ssl_vhost)), + % (" ".join(str(addr) for + addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args))) @@ -1179,7 +1192,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.conf("vhost-root"), redirect_filename) + redirect_filepath = os.path.join(self.conf("vhost-root"), + redirect_filename) # Register the new file that will be created # Note: always register the creation before writing to ensure file will @@ -1207,7 +1221,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return None - def _get_proposed_addrs(self, vhost, port="80"): # pylint: disable=no-self-use + def _get_proposed_addrs(self, vhost, port="80"): """Return all addrs of vhost with the port replaced with the specified. :param obj.VirtualHost ssl_vhost: Original Vhost @@ -1287,7 +1301,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): .. note:: Does not make sure that the site correctly works or that all modules are enabled appropriately. - .. todo:: This function should number subdomains before the domain vhost + .. todo:: This function should number subdomains before the domain + vhost .. todo:: Make sure link is not broken... diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 8ac88b197..fe5ef3335 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -73,7 +73,8 @@ AUGEAS_LENS_DIR = pkg_resources.resource_filename( REWRITE_HTTPS_ARGS = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] -"""Apache version<2.3.9 rewrite rule arguments used for redirections to https vhost""" +"""Apache version<2.3.9 rewrite rule arguments used for redirections to +https vhost""" REWRITE_HTTPS_ARGS_WITH_END = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"] @@ -81,14 +82,14 @@ REWRITE_HTTPS_ARGS_WITH_END = [ https vhost""" HSTS_ARGS = ["always", "set", "Strict-Transport-Security", - "\"max-age=31536000\""] + "\"max-age=31536000\""] """Apache header arguments for HSTS""" UIR_ARGS = ["always", "set", "Content-Security-Policy", - "upgrade-insecure-requests"] + "upgrade-insecure-requests"] HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, - "Upgrade-Insecure-Requests": UIR_ARGS} + "Upgrade-Insecure-Requests": UIR_ARGS} def os_constant(key): diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index 45c55f49a..d05a995ba 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -79,8 +79,9 @@ def _vhost_menu(domain, vhosts): ) code, tag = zope.component.getUtility(interfaces.IDisplay).menu( - "We were unable to find a vhost with a ServerName or Address of {0}.{1}" - "Which virtual host would you like to choose?".format( + "We were unable to find a vhost with a ServerName " + "or Address of {0}.{1}Which virtual host would you " + "like to choose?".format( domain, os.linesep), choices, help_label="More Info", ok_label="Select") diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 82effad2b..cc7f2ec42 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -96,11 +96,12 @@ class ApacheParser(object): def update_runtime_variables(self): """" - .. note:: Compile time variables (apache2ctl -V) are not used within the - dynamic configuration files. These should not be parsed or + .. note:: Compile time variables (apache2ctl -V) are not used within + the dynamic configuration files. These should not be parsed or interpreted. - .. todo:: Create separate compile time variables... simply for arg_get() + .. todo:: Create separate compile time variables... + simply for arg_get() """ stdout = self._get_runtime_cfg() @@ -177,7 +178,8 @@ class ApacheParser(object): # Make sure we don't cause an IndexError (end of list) # Check to make sure arg + 1 doesn't exist if (i == (len(matches) - 1) or - not matches[i + 1].endswith("/arg[%d]" % (args + 1))): + not matches[i + 1].endswith("/arg[%d]" % + (args + 1))): filtered.append(matches[i][:-len("/arg[%d]" % args)]) return filtered @@ -311,8 +313,6 @@ class ApacheParser(object): for match in matches: dir_ = self.aug.get(match).lower() if dir_ == "include" or dir_ == "includeoptional": - # start[6:] to strip off /files - #print self._get_include_path(self.get_arg(match +"/arg")), directive, arg ordered_matches.extend(self.find_dir( directive, arg, self._get_include_path(self.get_arg(match + "/arg")), @@ -331,8 +331,8 @@ class ApacheParser(object): """ value = self.aug.get(match) - # No need to strip quotes for variables, as apache2ctl already does this - # but we do need to strip quotes for all normal arguments. + # No need to strip quotes for variables, as apache2ctl already does + # this, but we do need to strip quotes for all normal arguments. # Note: normal argument may be a quoted variable # e.g. strip now, not later @@ -454,7 +454,7 @@ class ApacheParser(object): https://apr.apache.org/docs/apr/2.0/apr__fnmatch_8h_source.html http://apache2.sourcearchive.com/documentation/2.2.16-6/apr__fnmatch_8h_source.html - :param str clean_fn_match: Apache style filename match, similar to globs + :param str clean_fn_match: Apache style filename match, like globs :returns: regex suitable for augeas :rtype: str diff --git a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py index 7099c388f..1fc5281c1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py @@ -96,7 +96,8 @@ class ComplexParserTest(util.ParserTest): else: self.assertFalse(self.parser.find_dir("FNMATCH_DIRECTIVE")) - # NOTE: Only run one test per function otherwise you will have inf recursion + # NOTE: Only run one test per function otherwise you will have + # inf recursion def test_include(self): self.verify_fnmatch("test_fnmatch.?onf") @@ -104,7 +105,8 @@ class ComplexParserTest(util.ParserTest): self.verify_fnmatch("../complex_parsing/[te][te]st_*.?onf") def test_include_fullpath(self): - self.verify_fnmatch(os.path.join(self.config_path, "test_fnmatch.conf")) + self.verify_fnmatch(os.path.join(self.config_path, + "test_fnmatch.conf")) def test_include_fullpath_trailing_slash(self): self.verify_fnmatch(self.config_path + "//") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 2a32b04be..00a98e33a 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -35,10 +35,10 @@ class TwoVhost80Test(util.ApacheTest): def mock_deploy_cert(self, config): """A test for a mock deploy cert""" self.config.real_deploy_cert = self.config.deploy_cert + def mocked_deploy_cert(*args, **kwargs): """a helper to mock a deployed cert""" - with mock.patch( - "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): config.real_deploy_cert(*args, **kwargs) self.config.deploy_cert = mocked_deploy_cert return self.config @@ -70,7 +70,8 @@ class TwoVhost80Test(util.ApacheTest): def test_prepare_old_aug(self, mock_exe_exists, _): mock_exe_exists.return_value = True self.config.config_test = mock.Mock() - self.config._check_aug_version = mock.Mock(return_value=False) # pylint: disable=protected-access + # pylint: disable=protected-access + self.config._check_aug_version = mock.Mock(return_value=False) self.assertRaises( errors.NotSupportedError, self.config.prepare) @@ -110,8 +111,8 @@ class TwoVhost80Test(util.ApacheTest): def test_add_servernames_alias(self): self.config.parser.add_dir( self.vh_truth[2].path, "ServerAlias", ["*.le.co"]) - self.config._add_servernames(self.vh_truth[2]) # pylint: disable=protected-access - + # pylint: disable=protected-access + self.config._add_servernames(self.vh_truth[2]) self.assertEqual( self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) @@ -177,7 +178,8 @@ class TwoVhost80Test(util.ApacheTest): def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[3] conflicting_vhost = obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:443")]), True, True) + "path", "aug_path", set([obj.Addr.fromstring("*:443")]), + True, True) self.config.vhosts.append(conflicting_vhost) self.assertRaises( @@ -196,7 +198,8 @@ class TwoVhost80Test(util.ApacheTest): def test_find_best_vhost_variety(self): # pylint: disable=protected-access ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))]), True, False) self.config.vhosts.append(ssl_vh) self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) @@ -277,7 +280,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -295,7 +299,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("ssl_module" in self.config.parser.modules) loc_cert = self.config.parser.find_dir( - "sslcertificatefile", "example/fullchain.pem", self.vh_truth[1].path) + "sslcertificatefile", "example/fullchain.pem", + self.vh_truth[1].path) loc_key = self.config.parser.find_dir( "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path) @@ -310,7 +315,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl_no_fullchain(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") @@ -320,11 +326,13 @@ class TwoVhost80Test(util.ApacheTest): self.config.assoc["random.demo"] = self.vh_truth[1] self.assertRaises(errors.PluginError, lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", "example/key.pem")) + "random.demo", "example/cert.pem", + "example/key.pem")) def test_deploy_cert_old_apache_no_chain(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 7)) self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") @@ -334,7 +342,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.assoc["random.demo"] = self.vh_truth[1] self.assertRaises(errors.PluginError, lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", "example/key.pem")) + "random.demo", "example/cert.pem", + "example/key.pem")) def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") @@ -442,7 +451,8 @@ class TwoVhost80Test(util.ApacheTest): # Test Listen statements with specific ip listeed self.config.prepare_server_https("443") - # Should only be 2 here, as the third interface already listens to the correct port + # Should only be 2 here, as the third interface + # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 2) # Check argument to new Listen statements @@ -456,9 +466,12 @@ class TwoVhost80Test(util.ApacheTest): # Test self.config.prepare_server_https("8080", temp=True) self.assertEqual(mock_add_dir.call_count, 3) - self.assertEqual(mock_add_dir.call_args_list[0][0][2], ["1.2.3.4:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[0][0][2], + ["1.2.3.4:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[1][0][2], + ["[::1]:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[2][0][2], + ["1.1.1.1:8080", "https"]) def test_prepare_server_https_mixed_listen(self): @@ -476,7 +489,8 @@ class TwoVhost80Test(util.ApacheTest): # Test Listen statements with specific ip listeed self.config.prepare_server_https("443") - # Should only be 2 here, as the third interface already listens to the correct port + # Should only be 2 here, as the third interface + # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 0) def test_make_vhost_ssl(self): @@ -510,7 +524,8 @@ class TwoVhost80Test(util.ApacheTest): for directive in ["SSLCertificateFile", "SSLCertificateKeyFile", "SSLCertificateChainFile", "SSLCACertificatePath"]: for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, directive, ["bogus"]) + self.config.parser.add_dir(self.vh_truth[1].path, + directive, ["bogus"]) self.config.save() self.config._clean_vhost(self.vh_truth[1]) @@ -536,23 +551,24 @@ class TwoVhost80Test(util.ApacheTest): # pylint: disable=protected-access DIRECTIVE = "Foo" for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, DIRECTIVE, ["bar"]) + self.config.parser.add_dir(self.vh_truth[1].path, + DIRECTIVE, ["bar"]) self.config.save() self.config._deduplicate_directives(self.vh_truth[1].path, [DIRECTIVE]) self.config.save() self.assertEqual( - len(self.config.parser.find_dir( - DIRECTIVE, None, self.vh_truth[1].path, False)), - 1) + len(self.config.parser.find_dir( + DIRECTIVE, None, self.vh_truth[1].path, False)), 1) def test_remove_directives(self): # pylint: disable=protected-access DIRECTIVES = ["Foo", "Bar"] for directive in DIRECTIVES: for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, directive, ["baz"]) + self.config.parser.add_dir(self.vh_truth[1].path, + directive, ["baz"]) self.config.save() self.config._remove_directives(self.vh_truth[1].path, DIRECTIVES) @@ -560,9 +576,8 @@ class TwoVhost80Test(util.ApacheTest): for directive in DIRECTIVES: self.assertEqual( - len(self.config.parser.find_dir( - directive, None, self.vh_truth[1].path, False)), - 0) + len(self.config.parser.find_dir( + directive, None, self.vh_truth[1].path, False)), 0) def test_make_vhost_ssl_extra_vhs(self): self.config.aug.match = mock.Mock(return_value=["p1", "p2"]) @@ -651,7 +666,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises(errors.PluginError, self.config.get_version) mock_script.return_value = ( - "Server Version: Apache/2.3{0} Apache/2.4.7".format(os.linesep), "") + "Server Version: Apache/2.3{0} Apache/2.4.7".format( + os.linesep), "") self.assertRaises(errors.PluginError, self.config.get_version) mock_script.side_effect = errors.SubprocessError("Can't find program") @@ -675,7 +691,8 @@ class TwoVhost80Test(util.ApacheTest): def test_config_test_bad_process(self, mock_run_script): mock_run_script.side_effect = errors.SubprocessError - self.assertRaises(errors.MisconfigurationError, self.config.config_test) + self.assertRaises(errors.MisconfigurationError, + self.config.config_test) def test_get_all_certs_keys(self): c_k = self.config.get_all_certs_keys() @@ -687,7 +704,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("default-ssl" in path) def test_get_all_certs_keys_malformed_conf(self): - self.config.parser.find_dir = mock.Mock(side_effect=[["path"], [], ["path"], []]) + self.config.parser.find_dir = mock.Mock( + side_effect=[["path"], [], ["path"], []]) c_k = self.config.get_all_certs_keys() self.assertFalse(c_k) @@ -708,13 +726,13 @@ class TwoVhost80Test(util.ApacheTest): def test_supported_enhancements(self): self.assertTrue(isinstance(self.config.supported_enhancements(), list)) - @mock.patch("letsencrypt.le_util.exe_exists") def test_enhance_unknown_vhost(self, mock_exe): self.config.parser.modules.add("rewrite_module") mock_exe.return_value = True ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("satoshi.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("satoshi.com",))]), True, False) self.config.vhosts.append(ssl_vh) self.assertRaises( @@ -735,7 +753,7 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "ensure-http-header", - "Strict-Transport-Security") + "Strict-Transport-Security") self.assertTrue("headers_module" in self.config.parser.modules) @@ -745,7 +763,7 @@ class TwoVhost80Test(util.ApacheTest): # These are not immediately available in find_dir even with save() and # load(). They must be found in sites-available hsts_header = self.config.parser.find_dir( - "Header", None, ssl_vhost.path) + "Header", None, ssl_vhost.path) # four args to HSTS header self.assertEqual(len(hsts_header), 4) @@ -757,12 +775,12 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("encryption-example.demo", "ensure-http-header", - "Strict-Transport-Security") + "Strict-Transport-Security") self.assertRaises( errors.PluginEnhancementAlreadyPresent, - self.config.enhance, "encryption-example.demo", "ensure-http-header", - "Strict-Transport-Security") + self.config.enhance, "encryption-example.demo", + "ensure-http-header", "Strict-Transport-Security") @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -773,7 +791,7 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") + "Upgrade-Insecure-Requests") self.assertTrue("headers_module" in self.config.parser.modules) @@ -783,7 +801,7 @@ class TwoVhost80Test(util.ApacheTest): # These are not immediately available in find_dir even with save() and # load(). They must be found in sites-available uir_header = self.config.parser.find_dir( - "Header", None, ssl_vhost.path) + "Header", None, ssl_vhost.path) # four args to HSTS header self.assertEqual(len(uir_header), 4) @@ -795,14 +813,12 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("encryption-example.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") + "Upgrade-Insecure-Requests") self.assertRaises( errors.PluginEnhancementAlreadyPresent, - self.config.enhance, "encryption-example.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") - - + self.config.enhance, "encryption-example.demo", + "ensure-http-header", "Upgrade-Insecure-Requests") @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -836,7 +852,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["Unknown"]) - self.assertTrue(self.config._is_rewrite_exists(self.vh_truth[3])) # pylint: disable=protected-access + # pylint: disable=protected-access + self.assertTrue(self.config._is_rewrite_exists(self.vh_truth[3])) def test_rewrite_engine_exists(self): # Skip the enable mod @@ -844,8 +861,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteEngine", "on") - self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3])) # pylint: disable=protected-access - + # pylint: disable=protected-access + self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3])) @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -857,7 +874,7 @@ class TwoVhost80Test(util.ApacheTest): # Create a preexisting rewrite rule self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["UnknownPattern", - "UnknownTarget"]) + "UnknownTarget"]) self.config.save() # This will create an ssl vhost for letsencrypt.demo @@ -879,11 +896,11 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("rewrite_module" in self.config.parser.modules) - def test_redirect_with_conflict(self): self.config.parser.modules.add("rewrite_module") ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))]), True, False) # No names ^ this guy should conflict. @@ -908,7 +925,8 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[1].name = "default.com" self.vh_truth[1].aliases = set(["yes.default.com"]) - self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access + # pylint: disable=protected-access + self.config._enable_redirect(self.vh_truth[1], "") self.assertEqual(len(self.config.vhosts), 7) def test_create_own_redirect_for_old_apache_version(self): @@ -918,7 +936,8 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[1].name = "default.com" self.vh_truth[1].aliases = set(["yes.default.com"]) - self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access + # pylint: disable=protected-access + self.config._enable_redirect(self.vh_truth[1], "") self.assertEqual(len(self.config.vhosts), 7) def test_sift_line(self): @@ -942,10 +961,10 @@ class TwoVhost80Test(util.ApacheTest): http_vhost.path, "RewriteEngine", "on") self.config.parser.add_dir( - http_vhost.path, "RewriteRule", - ["^", - "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,QSA,R=permanent]"]) + http_vhost.path, "RewriteRule", + ["^", + "https://%{SERVER_NAME}%{REQUEST_URI}", + "[L,QSA,R=permanent]"]) self.config.save() ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -954,8 +973,9 @@ class TwoVhost80Test(util.ApacheTest): "RewriteEngine", "on", ssl_vhost.path, False)) conf_text = open(ssl_vhost.filep).read() - commented_rewrite_rule = \ - "# RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent]" + commented_rewrite_rule = ("# RewriteRule ^ " + "https://%{SERVER_NAME}%{REQUEST_URI} " + "[L,QSA,R=permanent]") self.assertTrue(commented_rewrite_rule in conf_text) mock_get_utility().add_message.assert_called_once_with(mock.ANY, mock.ANY) @@ -990,9 +1010,11 @@ class TwoVhost80Test(util.ApacheTest): def test_aug_version(self): mock_match = mock.Mock(return_value=["something"]) self.config.aug.match = mock_match - self.assertEquals(self.config._check_aug_version(), ["something"]) # pylint: disable=protected-access + # pylint: disable=protected-access + self.assertEquals(self.config._check_aug_version(), + ["something"]) self.config.aug.match.side_effect = RuntimeError - self.assertFalse(self.config._check_aug_version()) # pylint: disable=protected-access + self.assertFalse(self.config._check_aug_version()) if __name__ == "__main__": diff --git a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py index 13eddaddf..a469702f1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py @@ -47,7 +47,8 @@ class VirtualHostTest(unittest.TestCase): self.assertTrue(self.vhost1.conflicts([self.addr2])) self.assertFalse(self.vhost1.conflicts([self.addr_default])) - self.assertFalse(self.vhost2.conflicts([self.addr1, self.addr_default])) + self.assertFalse(self.vhost2.conflicts([self.addr1, + self.addr_default])) def test_same_server(self): from letsencrypt_apache.obj import VirtualHost diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 9b78bf6d6..e976bc9f6 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -118,7 +118,8 @@ class BasicParserTest(util.ParserTest): # pylint: disable=protected-access path = os.path.join(self.parser.root, "httpd.conf") open(path, 'w').close() - self.parser.add_dir(self.parser.loc["default"], "Include", "httpd.conf") + self.parser.add_dir(self.parser.loc["default"], "Include", + "httpd.conf") self.assertEqual( path, self.parser._set_user_config_file()) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py index b69406932..a2d3cacc4 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py @@ -9,6 +9,8 @@ from letsencrypt.plugins import common_test from letsencrypt_apache import obj from letsencrypt_apache.tests import util +from six.moves import xrange # pylint: disable=redefined-builtin + class TlsSniPerformTest(util.ApacheTest): """Test the ApacheTlsSni01 challenge.""" @@ -58,7 +60,7 @@ class TlsSniPerformTest(util.ApacheTest): mock_setup_cert.assert_called_once_with(achall) - # Check to make sure challenge config path is included in apache config. + # Check to make sure challenge config path is included in apache config self.assertEqual( len(self.sni.configurator.parser.find_dir( "Include", self.sni.challenge_conf)), 1) @@ -78,8 +80,7 @@ class TlsSniPerformTest(util.ApacheTest): # pylint: disable=protected-access self.sni._setup_challenge_cert = mock_setup_cert - with mock.patch( - "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): sni_responses = self.sni.perform() self.assertEqual(mock_setup_cert.call_count, 2) @@ -126,13 +127,15 @@ class TlsSniPerformTest(util.ApacheTest): def test_get_addrs_default(self): self.sni.configurator.choose_vhost = mock.Mock( return_value=obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("_default_:443")]), + "path", "aug_path", + set([obj.Addr.fromstring("_default_:443")]), False, False) ) + # pylint: disable=protected-access self.assertEqual( set([obj.Addr.fromstring("*:443")]), - self.sni._get_addrs(self.achalls[0])) # pylint: disable=protected-access + self.sni._get_addrs(self.achalls[0])) if __name__ == "__main__": diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 798d4814b..569f9e150 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -62,7 +62,8 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods def get_apache_configurator( - config_path, vhost_path, config_dir, work_dir, version=(2, 4, 7), conf=None): + config_path, vhost_path, + config_dir, work_dir, version=(2, 4, 7), conf=None): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -129,10 +130,12 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, "mod_macro-example.conf/Macro/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, modmacro=True), + set([obj.Addr.fromstring("*:80")]), False, True, + modmacro=True), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), - os.path.join(aug_pre, "default-ssl-port-only.conf/IfModule/VirtualHost"), + os.path.join(aug_pre, ("default-ssl-port-only.conf/" + "IfModule/VirtualHost")), set([obj.Addr.fromstring("_default_:443")]), True, False), ] return vh_truth diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index ca7985f35..971072311 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -10,6 +10,7 @@ from letsencrypt_apache import parser logger = logging.getLogger(__name__) + class ApacheTlsSni01(common.TLSSNI01): """Class that performs TLS-SNI-01 challenges within the Apache configurator @@ -125,7 +126,8 @@ class ApacheTlsSni01(common.TLSSNI01): addrs.add(default_addr) else: addrs.add( - addr.get_sni_addr(self.configurator.config.tls_sni_01_port)) + addr.get_sni_addr( + self.configurator.config.tls_sni_01_port)) return addrs From 5357a556eb8a2ae5308c175b944c9797549464b1 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Thu, 14 Jan 2016 14:18:36 +0200 Subject: [PATCH 139/201] PyLint doesn't play well with six --- letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py index a2d3cacc4..9681bf9fc 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py @@ -9,7 +9,7 @@ from letsencrypt.plugins import common_test from letsencrypt_apache import obj from letsencrypt_apache.tests import util -from six.moves import xrange # pylint: disable=redefined-builtin +from six.moves import xrange # pylint: disable=redefined-builtin, import-error class TlsSniPerformTest(util.ApacheTest): From f909d22a3dd906b4230930440ec6ee0feea6bff1 Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Thu, 14 Jan 2016 22:10:00 +0700 Subject: [PATCH 140/201] Test with missing certificate when calling RenewableCert.names() --- letsencrypt/tests/renewer_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 61a9a6e75..a103f5dbf 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -381,6 +381,10 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.names(12), ["example.com", "www.example.com"]) + # Trying missing cert + os.unlink(self.test_rc.cert) + self.assertRaises(errors.CertStorageError, self.test_rc.names) + @mock.patch("letsencrypt.storage.datetime") def test_time_interval_judgments(self, mock_datetime): """Test should_autodeploy() and should_autorenew() on the basis From 54207f9ef3458bb1b22c4481e699476211cbb7c6 Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Thu, 14 Jan 2016 23:15:22 +0700 Subject: [PATCH 141/201] Test view_config_changes with for_logging=True --- letsencrypt/tests/reverter_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/letsencrypt/tests/reverter_test.py b/letsencrypt/tests/reverter_test.py index d31b6f2cc..7699c96f5 100644 --- a/letsencrypt/tests/reverter_test.py +++ b/letsencrypt/tests/reverter_test.py @@ -385,6 +385,15 @@ class TestFullCheckpointsReverter(unittest.TestCase): self.assertRaises( errors.ReverterError, self.reverter.view_config_changes) + def test_view_config_changes_for_logging(self): + self._setup_three_checkpoints() + + config_changes = self.reverter.view_config_changes(for_logging=True) + + self.assertIn("First Checkpoint", config_changes) + self.assertIn("Second Checkpoint", config_changes) + self.assertIn("Third Checkpoint", config_changes) + def _setup_three_checkpoints(self): """Generate some finalized checkpoints.""" # Checkpoint1 - config1 From 84a0fba774ce8f968d3f090362e3e745d67e250d Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Thu, 14 Jan 2016 23:16:26 +0700 Subject: [PATCH 142/201] Properly patch display_util --- letsencrypt/tests/display/ops_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/letsencrypt/tests/display/ops_test.py b/letsencrypt/tests/display/ops_test.py index 31db47cce..d98afe180 100644 --- a/letsencrypt/tests/display/ops_test.py +++ b/letsencrypt/tests/display/ops_test.py @@ -407,10 +407,11 @@ class ChooseNamesTest(unittest.TestCase): "uniçodé.com") self.assertEqual(_choose_names_manually(), []) # IDN exception with previous mocks - with mock.patch("letsencrypt.display.util") as mock_sl: - uerror = UnicodeEncodeError('mock', u'', - 0, 1, 'mock') - mock_sl.separate_list_input.side_effect = uerror + with mock.patch( + "letsencrypt.display.ops.display_util.separate_list_input" + ) as mock_sli: + unicode_error = UnicodeEncodeError('mock', u'', 0, 1, 'mock') + mock_sli.side_effect = unicode_error self.assertEqual(_choose_names_manually(), []) # Punycode and no retry mock_util().input.return_value = (display_util.OK, From 2e034e6c6c6992eea55e74fa6bb5cbd66347ee3c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Jan 2016 11:42:10 -0800 Subject: [PATCH 143/201] Revert changes to acme's setup.py --- acme/setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 372c05b13..3d7b882d5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -11,6 +11,8 @@ install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', + 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) + 'pyasn1', # urllib3 InsecurePlatformWarning (#304) # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13', 'pyrfc3339', @@ -31,11 +33,6 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') -if sys.version_info < (2, 7, 9): - # For secure SSL connection with Python 2.7 (InsecurePlatformWarning) - install_requires.append('ndg-httpsclient') - install_requires.append('pyasn1') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', From 5d93678303cf3d2cdc69aaa16d3297739a72d3e6 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 14 Jan 2016 16:40:47 -0500 Subject: [PATCH 144/201] Make ConfigArgParse dependencies unconditional as well. None of this is ideal, since we're making the dependencies tighter than they theoretically need to be, but the behavior of the old le-auto makes this necessary to make it succeed in practice (when using LE wheels). Once we move to the new le-auto (which pins everything and makes setup.py dependencies irrelevant for auto installs), we should redo this using env markers as in https://github.com/letsencrypt/letsencrypt/pull/2177. We're too afraid to do it now. Similarly, we're too afraid to change how we handle argparse right now, despite that it should be required directly by us under 2.6. In practice, ConfigArgParse pulls it in for us, so we're okay as long as it continues to do that. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ad7fb6909..4d9cc7850 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ version = meta['version'] # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), + 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', @@ -52,12 +53,10 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', - 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 'mock<1.1.0', ]) else: install_requires.extend([ - 'ConfigArgParse', 'mock', ]) From b7b3f24da085765f8963c95826a6b54b401c58cd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Jan 2016 16:35:20 -0800 Subject: [PATCH 145/201] Convert sites-enabled files to symlinks --- .../letsencrypt_apache/tests/util.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 798d4814b..12237b209 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -42,6 +42,20 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + # Make sure all vhosts in sites-enabled are symlinks (Python packaging + # does not preserve symlinks) + sites_enabled = os.path.join(self.config_path, "sites-enabled") + if not os.path.exists(sites_enabled): + return + + for vhost_basename in os.listdir(sites_enabled): + vhost = os.path.join(sites_enabled, vhost_basename) + if not os.path.islink(vhost): + os.remove(vhost) + target = os.path.join( + os.path.pardir, "sites-available", vhost_basename) + os.symlink(target, vhost) + class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods From 2939b62f242f05e24c5ede23c8dfeed2f1a0535c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Jan 2016 16:59:29 -0800 Subject: [PATCH 146/201] Stop cover from failing --- letsencrypt-apache/letsencrypt_apache/tests/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 12237b209..782ed6f44 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -50,7 +50,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods for vhost_basename in os.listdir(sites_enabled): vhost = os.path.join(sites_enabled, vhost_basename) - if not os.path.islink(vhost): + if not os.path.islink(vhost): # pragma: no cover os.remove(vhost) target = os.path.join( os.path.pardir, "sites-available", vhost_basename) From e59fcf7ddd58ce85a94435b166209c61460e26d1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Jan 2016 17:39:18 -0800 Subject: [PATCH 147/201] Release 0.2.0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 3d7b882d5..05f3812fe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.2.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index a5c5e8a7a..5155db8e1 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.2.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 1ff9e7649..b28f9790c 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.2.0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index bfb3c3758..94d1ec122 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.2.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 1c7815f78..f099a39eb 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.0.dev0' +__version__ = '0.2.0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index d487e556d..3e0128ccb 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.2.0' install_requires = [ 'setuptools', # pkg_resources From 2a6d3bedb61800cadca3e7264bce73b11b3261e3 Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Fri, 15 Jan 2016 11:37:20 +0700 Subject: [PATCH 148/201] Fix missing --webroot-path handling in webroot plugin --- letsencrypt/plugins/webroot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index 0679bc349..f8176417c 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -98,8 +98,8 @@ to serve all files under specified web root ({0}).""" def _path_for_achall(self, achall): try: path = self.full_roots[achall.domain] - except IndexError: - raise errors.PluginError("Missing --webroot-path for domain: {1}" + except KeyError: + raise errors.PluginError("Missing --webroot-path for domain: {0}" .format(achall.domain)) if not os.path.exists(path): raise errors.PluginError("Mysteriously missing path {0} for domain: {1}" From b89d383ff49b39425fd70d708b17a6bb5e251fff Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Fri, 15 Jan 2016 11:38:21 +0700 Subject: [PATCH 149/201] Add tests for missing paths in webroot plugin --- letsencrypt/plugins/webroot_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/letsencrypt/plugins/webroot_test.py b/letsencrypt/plugins/webroot_test.py index defe9396b..e3f926c7f 100644 --- a/letsencrypt/plugins/webroot_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -111,6 +111,18 @@ class AuthenticatorTest(unittest.TestCase): self.assertEqual(os.stat(self.validation_path).st_gid, parent_gid) self.assertEqual(os.stat(self.validation_path).st_uid, parent_uid) + def test_perform_missing_path(self): + self.auth.prepare() + + missing_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.HTTP01_P, domain="thing2.com", account_key=KEY) + self.assertRaises( + errors.PluginError, self.auth.perform, [missing_achall]) + + self.auth.full_roots[self.achall.domain] = 'null' + self.assertRaises( + errors.PluginError, self.auth.perform, [self.achall]) + def test_perform_cleanup(self): self.auth.prepare() responses = self.auth.perform([self.achall]) From 4366a8988345f4a0fe37afc858112468b384dc0d Mon Sep 17 00:00:00 2001 From: Filip Ochnik Date: Fri, 15 Jan 2016 12:55:24 +0700 Subject: [PATCH 150/201] Don't use assertIn --- letsencrypt/tests/reverter_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/tests/reverter_test.py b/letsencrypt/tests/reverter_test.py index 7699c96f5..aafd3b041 100644 --- a/letsencrypt/tests/reverter_test.py +++ b/letsencrypt/tests/reverter_test.py @@ -390,9 +390,9 @@ class TestFullCheckpointsReverter(unittest.TestCase): config_changes = self.reverter.view_config_changes(for_logging=True) - self.assertIn("First Checkpoint", config_changes) - self.assertIn("Second Checkpoint", config_changes) - self.assertIn("Third Checkpoint", config_changes) + self.assertTrue("First Checkpoint" in config_changes) + self.assertTrue("Second Checkpoint" in config_changes) + self.assertTrue("Third Checkpoint" in config_changes) def _setup_three_checkpoints(self): """Generate some finalized checkpoints.""" From 91d958aa59efec0d45ce29108c7b04d82537ee45 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 15 Jan 2016 15:03:53 -0800 Subject: [PATCH 151/201] Bump version to 0.2.1.dev0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 05f3812fe..71e50c196 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0' +version = '0.2.1.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 5155db8e1..4959867f3 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0' +version = '0.2.1.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index b28f9790c..4131fdcad 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0' +version = '0.2.1.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 94d1ec122..e6df4c636 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0' +version = '0.2.1.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index f099a39eb..00dbe4320 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.0' +__version__ = '0.2.1.dev0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 3e0128ccb..7024b55ab 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0' +version = '0.2.1.dev0' install_requires = [ 'setuptools', # pkg_resources From 17066198867e63b8c7fad8102b90a40a078a02fd Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 15 Jan 2016 18:09:27 -0500 Subject: [PATCH 152/201] Update known-good-set, and make deps unconditional. Bring everything to the latest versions. Make dependencies unconditional: argparse, ndg-httpsclient, and pyasn1 get in all the time, to match the state of master as of 0.2.0. --- letsencrypt-auto-source/letsencrypt-auto | 185 ++++++++---------- .../letsencrypt-auto.template | 6 - .../pieces/conditional_requirements.py | 29 --- .../pieces/letsencrypt-auto-requirements.txt | 150 +++++++------- 4 files changed, 166 insertions(+), 204 deletions(-) delete mode 100644 letsencrypt-auto-source/pieces/conditional_requirements.py diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index dac8f3ef9..722d06d89 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -433,63 +433,63 @@ if [ "$NO_SELF_UPGRADE" = 1 ]; then # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, -# `pip freeze`, and then gather the hashes. +# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, +# and then gather the hashes. -# sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 -# sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo -# sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 -# sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc -# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 -# sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY -# sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc -# sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg -# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM -# sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U -# sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis -# sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 -# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U -# sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU -# sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M -# sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 -# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y -# sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA -# sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs -# sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA -cffi==1.3.1 +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: U8HJ3bMEMVE-t_PN7wo-BrDxJSGIqqd0SvD1pM1F268 +# sha256: pWj0nfyhKo2fNwGHJX78WKOBCeHu5xTZKFYdegGKZPg +# sha256: gJxsqM-8ruv71DK0V2ABtA04_yRjdzy1dXfXXhoCC8M +# sha256: hs3KLNnLpBQiIwOQ3xff6qnzRKkR45dci-naV7NVSOk +# sha256: JLE9uErsOFyiPHuN7YPvi7QXe8GB0UdY-fl1vl0CDYY +# sha256: lprv_XwOCX9r4e_WgsFWriJlkaB5OpS2wtXkKT9MjU4 +# sha256: AA81jUsPokn-qrnBzn1bL-fgLnvfaAbCZBhQX8aF4mg +# sha256: qdhvRgu9g1ii1ROtd54_P8h447k6ALUAL66_YW_-a5w +# sha256: MSezqzPrI8ysBx-aCAJ0jlz3xcvNAkgrsGPjW0HbsLA +# sha256: 4rLUIjZGmkAiTTnntsYFdfOIsvQj81TH7pClt_WMgGU +# sha256: jC3Mr-6JsbQksL7GrS3ZYiyUnSAk6Sn12h7YAerHXx0 +# sha256: pN56TRGu1Ii6tPsU9JiFh6gpvs5aIEM_eA1uM7CAg8s +# sha256: XKj-MEJSZaSSdOSwITobyY9LE0Sa5elvmEdx5dg-WME +# sha256: pP04gC9Z5xTrqBoCT2LbcQsn2-J6fqEukRU3MnqoTTA +# sha256: hs1pErvIPpQF1Kc81_S07oNTZS0tvHyCAQbtW00bqzo +# sha256: jx0XfTZOo1kAQVriTKPkcb49UzTtBBkpQGjEn0WROZg +cffi==1.4.2 # sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc -ConfigArgParse == 0.10.0 +ConfigArgParse==0.10.0 # sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI configobj==5.0.6 -# sha256: rvaUNVR6WdmmY0V7hb2CtQXOOC24gvhAeWskGV6QjUM -# sha256: F-Cyopbq5TqKIyBxLA2fXjJkUjnlCuLAxwiToL88ZnI -# sha256: agSFbNkcDV2-0d-J8qsKBo4kGMbChiqXjTOaPpaXEsM -# sha256: Gvlc-QCXZIRSkVyi1i3echM_hL5HRbjGF7iE64Ozmho -# sha256: AL5UcOcPhyZBjf18qhTaS0oNH-eAk0UEzyqkfEkLbrQ -# sha256: jg2Kw83Grq8C5ZK_lDNJQ3OHzpNlEve64O4rPv6guYk -# sha256: 2WqBQR437XCoZdk2lyq5guhPVklKDUmC8sGLCA7E16c -# sha256: PWl5CSvtvm3ZEMr1jnbiMTEWAKDcCoKYprdaUIHC7-w -# sha256: e5vcpKkOmI7IYbvzsXB5LstBnzVUq1PWy5v0Tn6ojfc -# sha256: 1vc40Cac149QSCQ9qPFNN9YIR5gZ6tb_tZ5rZMdSZyI -# sha256: 7Tuo9Y-M71rHKXB4W61mQ4OyQ7yhraOhAB8mceCEv2A -# sha256: LC2rxmUOLqj82E-FqbbTcCZMWiO2tL1aAHZv7ASzMaQ -# sha256: gMNj9i7awTcvu1HUqNRR8-BmuvEGRdjBmdCnFlAxPtA -# sha256: B88JF-T8_-mH_DZxErK9gYy6l1YmOfsTv_moOgOKClQ -# sha256: VYSnU5WgJE790QPYe5jnq68Q8C4h_V21yOf52N-g2bE -# sha256: tz2PKTF-1lK0s2Dvw-IPLM5X5EYTk_STfMIx0IOtLyw -# sha256: J4eqAyfnD8bQbQsDvgp97hkUMLE9j1hHoCQMbmjMpeE -# sha256: WiBGvL2G_GOQ2vlSJ1rLSITuPQ2lTH-Baq2la745U8U -# sha256: lkC04TkXiWgAUtJZFcrDsPdNC8l8tj_26atSc9IwFmA -# sha256: vjY4IvCRJqUvHyI81AesY9bcPjXH4oPeipLqJiXN_64 -# sha256: KRKSOvdFX7LSQ5oDckJQfBLK7OfdZlnWL6gqYe2yuuA -cryptography==1.1.1 +# sha256: 1U_hszrB4J8cEj4vl0948z6V1h1PSALdISIKXD6MEX0 +# sha256: B1X2aE4RhSAFs2MTdh7ctbqEOmTNAizhrC3L1JqTYG0 +# sha256: zjhNo4lZlluh90VKJfVp737yqxRd8ueiml4pS3TgRnc +# sha256: GvQDkV3LmWHDB2iuZRr6tpKC0dpaut-mN1IhrBGHdQM +# sha256: ag08d91PH-W8ZfJ--3fsjQSjiNpesl66DiBAwJgZ30o +# sha256: KdelgcO6_wTh--IAaltHjZ7cfPmib8ijWUkkf09lA3k +# sha256: IPAWEKpAh_bVadjMIMR4uB8DhIYnWqqx3Dx12VAsZ-A +# sha256: l9hGUIulDVomml82OK4cFmWbNTFaH0B_oVF2cH2j0Jc +# sha256: djfqRMLL1NsvLKccsmtmPRczORqnafi8g2xZVilbd5g +# sha256: gR-eqJVbPquzLgQGU0XDB4Ui5rPuPZLz0n08fNcWpjM +# sha256: DXCMjYz97Qm4fCoLqHY856ZjWG4EPmrEL9eDHpKQHLY +# sha256: Efnq11YqPgATWGytM5o_em9Yg8zhw7S5jhrGnft3p_Y +# sha256: dNhnm55-0ePs-wq1NNyTUruxz3PTYsmQkJTAlyivqJY +# sha256: z1Hd-123eBaiB1OKZgEUuC4w4IAD_uhJmwILi4SA2sU +# sha256: 47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU +# sha256: dITvgYGUFB3_eUdf-74vd6-FHiw7v-Lk1ZEjEi-KTjM +# sha256: 7gLB6J7l7pUBV6VK1YTXN8Ec83putMCFPozz8n6WLcA +# sha256: pfGPaxhQpVVKV9v2YsrSUSpGBW5paHJqmFjngN1bnQo +# sha256: 26GA8xrb5xi6qdbPirY0hJSwlLK4GAL_8zvVDSfRPnM +# sha256: 5RinlLjzjoOC9_B3kUGBPOtIE6z9MRVBwNsOGJ69eN4 +# sha256: f1FFn4TWcERCdeYVg59FQsk1R6Euk4oKSQba_l994VM +cryptography==1.1.2 -# sha256: nUqSIOTrq9f_YNhT5pw92J3rrV3euaxedor4EezncI4 -# sha256: TTNFnDMlThutz9tHo0CoAc2ARm5G--NdJdMIvxSNrXA -enum34==1.1.1 +# sha256: JHXX_N31lR6S_1RpcnWIAt5SYL9Akxmp8ZNOa7yLHcc +# sha256: NZB977D5krdat3iPZf7cHPIP-iJojg5vbxKvwGs-pQE +enum34==1.1.2 # sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 # sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM @@ -499,9 +499,12 @@ funcsigs==0.4 # sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs idna==2.0 -# sha256: WJWfvsOuQ49axxC7YcQrYQ2Ju05bzDJHswd-I0hzTa0 -# sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY -ipaddress==1.0.15 +# sha256: k1cSgAzkdgcB2JrWd2Zs1SaR_S9vCzQMi0I5o8F5iKU +# sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA +ipaddress==1.0.16 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs @@ -534,6 +537,19 @@ pbr==1.8.1 # sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus psutil==3.3.0 +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + # sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M pycparser==2.14 @@ -567,17 +583,17 @@ python2-pythondialog==3.3.0 # sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM pytz==2015.7 -# sha256: ifGx8l3Ne2j1FOjTQaWy60ZvlgrnVoIuqrSAo8GoHCg -# sha256: hP6NW_Tc3MSQAkRsR6FG0XrBD6zwDZCGZZBkrEO2wls -requests==2.8.1 +# sha256: ET-7pVManjSUW302szoIToul0GZLcDyBp8Vy2RkZpbg +# sha256: xXeBXdAPE5QgP8ROuXlySwmPiCZKnviY7kW45enPWH8 +requests==2.9.1 # sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE # sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo six==1.10.0 -# sha256: tqk-kJsx4-FGWVhP9zKYZCdZBd3BuGU4Yb3-HllyUPQ -# sha256: 60-YmUtAqOLtziiegRyaOIgK5T65_28DHQ4kOmmw_L8 -Werkzeug==0.11.2 +# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms +# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI +Werkzeug==0.11.3 # sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo zope.component==4.2.2 @@ -604,54 +620,19 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: xlKw86fsP3VtbdljTTTboXgIw1QFHdCR1p8ff_zKnQ0 -# sha256: HCCDzax5ts-yAZDWhpMiQDUQS0bSXcnFLRerqZB_YPs -acme==0.1.1 +# sha256: fYwCUXn3Wd_tKYHuPfufzDQZuDNng0HZb_th3xepC7U +# sha256: dKkCf9CZnKaDTFOof-KoazoewjKTSAVxZUJmnj_3i_U +acme==0.2.0 -# sha256: F6dWBwnljkI2IASGO4VwCV05Nc3t0w8U0kWZsllpC8s -# sha256: jmeEAx7qchLhKOF75RqPuOSH2Wk6XficiL5TYyYfKs4 -letsencrypt==0.1.1 +# sha256: 4x7K5lzKwm_GjYMojvUh053qL4EfIC5hGFmW370-7jI +# sha256: kcm3VmxXIGNS7ShcKFnYdA9AfXnqcbV_otMsADr1p2A +letsencrypt==0.2.0 -# sha256: 06zn8Fstsyfw58RmLaaODObDzR2jfZVCor0Hc65LYus -# sha256: 8ZO1CwBNE8eunQXAoXKVoGiaeAmj2BkRTMy5tXrBuYY -letsencrypt-apache==0.1.1 +# sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 +# sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs +letsencrypt-apache==0.2.0 UNLIKELY_EOF - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/conditional_requirements.py" -"""Spit out additional pinned requirements depending on the Python version.""" -from sys import version_info - - -if __name__ == '__main__': - if version_info < (2, 7, 9): - print """ -# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ -ndg-httpsclient==0.4.0 - -# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 -# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 -# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A -# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U -# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU -# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg -# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg -# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 -# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 -# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik -# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 -pyasn1==0.1.9 -""" - if version_info < (2, 7): - print """ -# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ -# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ -argparse==1.4.0 -""" - -UNLIKELY_EOF - # ------------------------------------------------------------------------- - "$VENV_BIN/python" "$TEMP_DIR/conditional_requirements.py" >> "$TEMP_DIR/letsencrypt-auto-requirements.txt" # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" #!/usr/bin/env python diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 9e25119a6..8118a5f69 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -195,12 +195,6 @@ if [ "$NO_SELF_UPGRADE" = 1 ]; then cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" {{ letsencrypt-auto-requirements.txt }} UNLIKELY_EOF - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/conditional_requirements.py" -{{ conditional_requirements.py }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - "$VENV_BIN/python" "$TEMP_DIR/conditional_requirements.py" >> "$TEMP_DIR/letsencrypt-auto-requirements.txt" # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" {{ peep.py }} diff --git a/letsencrypt-auto-source/pieces/conditional_requirements.py b/letsencrypt-auto-source/pieces/conditional_requirements.py deleted file mode 100644 index d81f03c6a..000000000 --- a/letsencrypt-auto-source/pieces/conditional_requirements.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Spit out additional pinned requirements depending on the Python version.""" -from sys import version_info - - -if __name__ == '__main__': - if version_info < (2, 7, 9): - print """ -# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ -ndg-httpsclient==0.4.0 - -# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 -# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 -# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A -# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U -# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU -# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg -# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg -# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 -# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 -# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik -# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 -pyasn1==0.1.9 -""" - if version_info < (2, 7): - print """ -# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ -# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ -argparse==1.4.0 -""" diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index 5b800b8be..abdcd9d8d 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -1,61 +1,61 @@ # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r py26reqs.txt -e acme -e . -e letsencrypt-apache`, -# `pip freeze`, and then gather the hashes. +# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, +# and then gather the hashes. -# sha256: x40PeQZP0GQZT-XH5aO2svbbtIWDdrtSAqRIjGkfYS4 -# sha256: jLFOL0T7ke2Q8qcfFRdXalLSQdOB7RWXIyAMi4Fy6Bo -# sha256: D9Uqk2RK-TMBJjLTLYxn1oDWosvR_TBbnG3OL3tDw48 -# sha256: 9RMU8_KwLhPmO34BO_wynw4YaVYrDGrc6IIIF4pOCIc -# sha256: jktALLnaWPJ1ItB4F8Tksb7nY1evq4X8NfNeHn8JUe4 -# sha256: yybABKlI6OhwZTFmC1UBSROe25wy3kSR1tqAuJdTCBY -# sha256: a0wfaa1zEfNpDeK2EUUa2jpnTqDJYQFgP6v0ZKJBdQc -# sha256: PSbhDGMPaT71HHWqgD6Si282CSMBaNhcxzq98ovPSWg -# sha256: mHEajOPtqhXj2lmKvbK0ef5068ITOyd8UrtDIlbjyNM -# sha256: 0HgsaYx0ACAtteAygp5_qkFrHm7GBdmqLH1BSPEyp3U -# sha256: q19L-hSCKK6I7SNB1VsFkzAk3tr_jueszKPO80fvFis -# sha256: sP3W0k-DpDKq58mbqZnLnaJiSZbfYFb4vnwgYe8kt44 -# sha256: XLKdOXHKwRrzNIBMfAVwbAiSiklH-CB-jE69gH5ET2U -# sha256: n-ixA0rx6WBEEUSNoYmYqzzHQRbQThlajkuqlr0UsFU -# sha256: rSxLkZrDYgPZStjNCpk-BGtypxZUXfSxxKpmYAeWv2M -# sha256: COvlZqBLYsYxc4Qxt4_a6_sCSRzDYUWOIL3CjiRsUw4 -# sha256: VH9kAYaHxBEIVrBlhanWiQLrmV8Zj4sTepXY8CU8N3Y -# sha256: 2KyY9M2WQ-vDSTG9vchm0CZn6NjCWBJy-HwTtoVYwmA -# sha256: 0PsSOUbFAt9mg454QbOffkNsSZGwHR2vSu1m4kEcKMs -# sha256: 1F3TmncLSvtZHIJVX2qLvBrH6wGe2ptiHu4aCnIgEiA -cffi==1.3.1 +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: U8HJ3bMEMVE-t_PN7wo-BrDxJSGIqqd0SvD1pM1F268 +# sha256: pWj0nfyhKo2fNwGHJX78WKOBCeHu5xTZKFYdegGKZPg +# sha256: gJxsqM-8ruv71DK0V2ABtA04_yRjdzy1dXfXXhoCC8M +# sha256: hs3KLNnLpBQiIwOQ3xff6qnzRKkR45dci-naV7NVSOk +# sha256: JLE9uErsOFyiPHuN7YPvi7QXe8GB0UdY-fl1vl0CDYY +# sha256: lprv_XwOCX9r4e_WgsFWriJlkaB5OpS2wtXkKT9MjU4 +# sha256: AA81jUsPokn-qrnBzn1bL-fgLnvfaAbCZBhQX8aF4mg +# sha256: qdhvRgu9g1ii1ROtd54_P8h447k6ALUAL66_YW_-a5w +# sha256: MSezqzPrI8ysBx-aCAJ0jlz3xcvNAkgrsGPjW0HbsLA +# sha256: 4rLUIjZGmkAiTTnntsYFdfOIsvQj81TH7pClt_WMgGU +# sha256: jC3Mr-6JsbQksL7GrS3ZYiyUnSAk6Sn12h7YAerHXx0 +# sha256: pN56TRGu1Ii6tPsU9JiFh6gpvs5aIEM_eA1uM7CAg8s +# sha256: XKj-MEJSZaSSdOSwITobyY9LE0Sa5elvmEdx5dg-WME +# sha256: pP04gC9Z5xTrqBoCT2LbcQsn2-J6fqEukRU3MnqoTTA +# sha256: hs1pErvIPpQF1Kc81_S07oNTZS0tvHyCAQbtW00bqzo +# sha256: jx0XfTZOo1kAQVriTKPkcb49UzTtBBkpQGjEn0WROZg +cffi==1.4.2 # sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc -ConfigArgParse == 0.10.0 +ConfigArgParse==0.10.0 # sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI configobj==5.0.6 -# sha256: rvaUNVR6WdmmY0V7hb2CtQXOOC24gvhAeWskGV6QjUM -# sha256: F-Cyopbq5TqKIyBxLA2fXjJkUjnlCuLAxwiToL88ZnI -# sha256: agSFbNkcDV2-0d-J8qsKBo4kGMbChiqXjTOaPpaXEsM -# sha256: Gvlc-QCXZIRSkVyi1i3echM_hL5HRbjGF7iE64Ozmho -# sha256: AL5UcOcPhyZBjf18qhTaS0oNH-eAk0UEzyqkfEkLbrQ -# sha256: jg2Kw83Grq8C5ZK_lDNJQ3OHzpNlEve64O4rPv6guYk -# sha256: 2WqBQR437XCoZdk2lyq5guhPVklKDUmC8sGLCA7E16c -# sha256: PWl5CSvtvm3ZEMr1jnbiMTEWAKDcCoKYprdaUIHC7-w -# sha256: e5vcpKkOmI7IYbvzsXB5LstBnzVUq1PWy5v0Tn6ojfc -# sha256: 1vc40Cac149QSCQ9qPFNN9YIR5gZ6tb_tZ5rZMdSZyI -# sha256: 7Tuo9Y-M71rHKXB4W61mQ4OyQ7yhraOhAB8mceCEv2A -# sha256: LC2rxmUOLqj82E-FqbbTcCZMWiO2tL1aAHZv7ASzMaQ -# sha256: gMNj9i7awTcvu1HUqNRR8-BmuvEGRdjBmdCnFlAxPtA -# sha256: B88JF-T8_-mH_DZxErK9gYy6l1YmOfsTv_moOgOKClQ -# sha256: VYSnU5WgJE790QPYe5jnq68Q8C4h_V21yOf52N-g2bE -# sha256: tz2PKTF-1lK0s2Dvw-IPLM5X5EYTk_STfMIx0IOtLyw -# sha256: J4eqAyfnD8bQbQsDvgp97hkUMLE9j1hHoCQMbmjMpeE -# sha256: WiBGvL2G_GOQ2vlSJ1rLSITuPQ2lTH-Baq2la745U8U -# sha256: lkC04TkXiWgAUtJZFcrDsPdNC8l8tj_26atSc9IwFmA -# sha256: vjY4IvCRJqUvHyI81AesY9bcPjXH4oPeipLqJiXN_64 -# sha256: KRKSOvdFX7LSQ5oDckJQfBLK7OfdZlnWL6gqYe2yuuA -cryptography==1.1.1 +# sha256: 1U_hszrB4J8cEj4vl0948z6V1h1PSALdISIKXD6MEX0 +# sha256: B1X2aE4RhSAFs2MTdh7ctbqEOmTNAizhrC3L1JqTYG0 +# sha256: zjhNo4lZlluh90VKJfVp737yqxRd8ueiml4pS3TgRnc +# sha256: GvQDkV3LmWHDB2iuZRr6tpKC0dpaut-mN1IhrBGHdQM +# sha256: ag08d91PH-W8ZfJ--3fsjQSjiNpesl66DiBAwJgZ30o +# sha256: KdelgcO6_wTh--IAaltHjZ7cfPmib8ijWUkkf09lA3k +# sha256: IPAWEKpAh_bVadjMIMR4uB8DhIYnWqqx3Dx12VAsZ-A +# sha256: l9hGUIulDVomml82OK4cFmWbNTFaH0B_oVF2cH2j0Jc +# sha256: djfqRMLL1NsvLKccsmtmPRczORqnafi8g2xZVilbd5g +# sha256: gR-eqJVbPquzLgQGU0XDB4Ui5rPuPZLz0n08fNcWpjM +# sha256: DXCMjYz97Qm4fCoLqHY856ZjWG4EPmrEL9eDHpKQHLY +# sha256: Efnq11YqPgATWGytM5o_em9Yg8zhw7S5jhrGnft3p_Y +# sha256: dNhnm55-0ePs-wq1NNyTUruxz3PTYsmQkJTAlyivqJY +# sha256: z1Hd-123eBaiB1OKZgEUuC4w4IAD_uhJmwILi4SA2sU +# sha256: 47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU +# sha256: dITvgYGUFB3_eUdf-74vd6-FHiw7v-Lk1ZEjEi-KTjM +# sha256: 7gLB6J7l7pUBV6VK1YTXN8Ec83putMCFPozz8n6WLcA +# sha256: pfGPaxhQpVVKV9v2YsrSUSpGBW5paHJqmFjngN1bnQo +# sha256: 26GA8xrb5xi6qdbPirY0hJSwlLK4GAL_8zvVDSfRPnM +# sha256: 5RinlLjzjoOC9_B3kUGBPOtIE6z9MRVBwNsOGJ69eN4 +# sha256: f1FFn4TWcERCdeYVg59FQsk1R6Euk4oKSQba_l994VM +cryptography==1.1.2 -# sha256: nUqSIOTrq9f_YNhT5pw92J3rrV3euaxedor4EezncI4 -# sha256: TTNFnDMlThutz9tHo0CoAc2ARm5G--NdJdMIvxSNrXA -enum34==1.1.1 +# sha256: JHXX_N31lR6S_1RpcnWIAt5SYL9Akxmp8ZNOa7yLHcc +# sha256: NZB977D5krdat3iPZf7cHPIP-iJojg5vbxKvwGs-pQE +enum34==1.1.2 # sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 # sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM @@ -65,9 +65,12 @@ funcsigs==0.4 # sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs idna==2.0 -# sha256: WJWfvsOuQ49axxC7YcQrYQ2Ju05bzDJHswd-I0hzTa0 -# sha256: r2yFz8nNsSuGFlXmufL1lhi_MIjL3oWHJ7LAqY6fZjY -ipaddress==1.0.15 +# sha256: k1cSgAzkdgcB2JrWd2Zs1SaR_S9vCzQMi0I5o8F5iKU +# sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA +ipaddress==1.0.16 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs @@ -100,6 +103,19 @@ pbr==1.8.1 # sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus psutil==3.3.0 +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + # sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M pycparser==2.14 @@ -133,17 +149,17 @@ python2-pythondialog==3.3.0 # sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM pytz==2015.7 -# sha256: ifGx8l3Ne2j1FOjTQaWy60ZvlgrnVoIuqrSAo8GoHCg -# sha256: hP6NW_Tc3MSQAkRsR6FG0XrBD6zwDZCGZZBkrEO2wls -requests==2.8.1 +# sha256: ET-7pVManjSUW302szoIToul0GZLcDyBp8Vy2RkZpbg +# sha256: xXeBXdAPE5QgP8ROuXlySwmPiCZKnviY7kW45enPWH8 +requests==2.9.1 # sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE # sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo six==1.10.0 -# sha256: tqk-kJsx4-FGWVhP9zKYZCdZBd3BuGU4Yb3-HllyUPQ -# sha256: 60-YmUtAqOLtziiegRyaOIgK5T65_28DHQ4kOmmw_L8 -Werkzeug==0.11.2 +# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms +# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI +Werkzeug==0.11.3 # sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo zope.component==4.2.2 @@ -170,14 +186,14 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: xlKw86fsP3VtbdljTTTboXgIw1QFHdCR1p8ff_zKnQ0 -# sha256: HCCDzax5ts-yAZDWhpMiQDUQS0bSXcnFLRerqZB_YPs -acme==0.1.1 +# sha256: fYwCUXn3Wd_tKYHuPfufzDQZuDNng0HZb_th3xepC7U +# sha256: dKkCf9CZnKaDTFOof-KoazoewjKTSAVxZUJmnj_3i_U +acme==0.2.0 -# sha256: F6dWBwnljkI2IASGO4VwCV05Nc3t0w8U0kWZsllpC8s -# sha256: jmeEAx7qchLhKOF75RqPuOSH2Wk6XficiL5TYyYfKs4 -letsencrypt==0.1.1 +# sha256: 4x7K5lzKwm_GjYMojvUh053qL4EfIC5hGFmW370-7jI +# sha256: kcm3VmxXIGNS7ShcKFnYdA9AfXnqcbV_otMsADr1p2A +letsencrypt==0.2.0 -# sha256: 06zn8Fstsyfw58RmLaaODObDzR2jfZVCor0Hc65LYus -# sha256: 8ZO1CwBNE8eunQXAoXKVoGiaeAmj2BkRTMy5tXrBuYY -letsencrypt-apache==0.1.1 +# sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 +# sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs +letsencrypt-apache==0.2.0 From e1bd1645b6340e4a6ad7cb655af4aa6bb9d19945 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 15 Jan 2016 18:25:26 -0500 Subject: [PATCH 153/201] Revert moving mock to test_requires. We'll take this up later, but I don't want to hold up the new le-auto on this debate. --- acme/setup.py | 4 +++- setup.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 36a5d5d0c..53f906629 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -29,7 +29,10 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', + 'mock<1.1.0', ]) +else: + install_requires.append('mock') docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags @@ -70,7 +73,6 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, - tests_require='mock<1.1.0' if sys.version_info < (2, 7) else 'mock', extras_require={ 'docs': docs_extras, 'testing': testing_extras, diff --git a/setup.py b/setup.py index fd44bd5b2..647bfb8d2 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,10 @@ if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', + 'mock<1.1.0', ]) +else: + install_requires.append('mock') dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 @@ -111,7 +114,6 @@ setup( include_package_data=True, install_requires=install_requires, - tests_require='mock<1.1.0' if sys.version_info < (2, 7) else 'mock', extras_require={ 'dev': dev_extras, 'docs': docs_extras, From e9239018ec48baff71b52143fa26cf331f2dded9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 15 Jan 2016 18:41:15 -0500 Subject: [PATCH 154/201] Add mock==1.0.1, the Python 2.6 compatible version, to le-auto reqs. This should ward off the runtime crashes described in https://github.com/erikrose/letsencrypt/commit/6c05197a43fffe3dcd2c10f41954f8a61aec2134. --- letsencrypt-auto-source/letsencrypt-auto | 4 ++++ .../pieces/letsencrypt-auto-requirements.txt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 722d06d89..d0cccc08e 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -632,6 +632,10 @@ letsencrypt==0.2.0 # sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs letsencrypt-apache==0.2.0 +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +mock==1.0.1 + UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index abdcd9d8d..bbda9f0b2 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -197,3 +197,7 @@ letsencrypt==0.2.0 # sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 # sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs letsencrypt-apache==0.2.0 + +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +mock==1.0.1 From 432d250672acdbfae8df9c1b0cd0d76688407e7c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 15 Jan 2016 15:49:15 -0800 Subject: [PATCH 155/201] Revert "Temporarily disable Apache 2.2 support" --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 2d822b3a1..8818923f4 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -155,7 +155,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: self.version = self.get_version() - if self.version < (2, 4): + if self.version < (2, 2): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) From 89f05379b74464eff88317e9fdd304590c44c7fd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 11:19:34 -0800 Subject: [PATCH 156/201] Document that --csr only works in certonly mode --- letsencrypt/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 89606089f..450860c74 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1086,7 +1086,8 @@ def _create_subparsers(helpful): "--csr", type=read_file, help="Path to a Certificate Signing Request (CSR) in DER" " format; note that the .csr file *must* contain a Subject" - " Alternative Name field for each domain you want certified.") + " Alternative Name field for each domain you want certified." + " Currently --csr only works with the 'certonly' subcommand'") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), From 6c0b1e46d7ef18fa00defb83b14005686b27e147 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 11:42:52 -0800 Subject: [PATCH 157/201] Import latest upstream augeas lens * Handles perl sripts embedded in Apache conf files * Fixes: #2079 --- .../letsencrypt_apache/augeas_lens/httpd.aug | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index 0f2cb7b45..edaca3fef 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -106,11 +106,17 @@ let section (body:lens) = let inner = (sep_spc . argv arg_sec)? . sep_osp . dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . indent . dels "[ \t\n\r]*/ ">\n" ] +let perl_section = [ indent . label "Perl" . del //i "" + . store /[^<]*/ + . del /<\/perl>/i "" . eol ] + + let rec content = section (content|directive) + | perl_section let lns = (content|directive|comment|empty)* @@ -121,6 +127,7 @@ let filter = (incl "/etc/apache2/apache2.conf") . (incl "/etc/apache2/conf-available/*.conf") . (incl "/etc/apache2/mods-available/*") . (incl "/etc/apache2/sites-available/*") . + (incl "/etc/apache2/vhosts.d/*.conf") . (incl "/etc/httpd/conf.d/*.conf") . (incl "/etc/httpd/httpd.conf") . (incl "/etc/httpd/conf/httpd.conf") . From e87de72662e3ca0f78fc3ef8ddf550f457aa50bc Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:13:51 -0800 Subject: [PATCH 158/201] Revert "Fix "global" max_attempt bug (#1719)" --- acme/acme/client.py | 30 ++++++++++-------------------- acme/acme/client_test.py | 5 +---- acme/acme/errors.py | 17 +++++++++-------- acme/acme/errors_test.py | 7 ++++--- 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 478536ecc..49c6bcb21 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,5 +1,4 @@ """ACME client API.""" -import collections import datetime import heapq import logging @@ -335,9 +334,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts (per - authorization) before `PollError` with non-empty ``waiting`` - is raised. + :param int max_attempts: Maximum number of attempts before + `PollError` with non-empty ``waiting`` is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -351,11 +349,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ - # pylint: disable=too-many-locals - assert max_attempts > 0 - attempts = collections.defaultdict(int) - exhausted = set() - # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -363,7 +356,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting: + while waiting and max_attempts: + max_attempts -= 1 # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -377,20 +371,16 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr - attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - if attempts[authzr] < max_attempts: - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) - else: - exhausted.add(authzr) + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) - if exhausted or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(exhausted, updated) + if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(waiting, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 9abc69c7c..449bd695e 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -319,10 +319,7 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime, - # make sure that max_attempts is per-authorization, rather - # than global - max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) + csr, authzrs, mintime=mintime) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 77d47c522..0385667c7 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,25 +56,26 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`exhausted` will be non-empty) + This might be caused by either timeout (`waiting` will be non-empty) or by some authorization being invalid. - :ivar exhausted: Set of `.AuthorizationResource` that didn't finish - within max allowed attempts. + :ivar waiting: Priority queue with `datetime.datatime` (based on + ``Retry-After``) as key, and original `.AuthorizationResource` + as value. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, exhausted, updated): - self.exhausted = exhausted + def __init__(self, waiting, updated): + self.waiting = waiting self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.exhausted) + return bool(self.waiting) def __repr__(self): - return '{0}(exhausted={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.exhausted, self.updated) + return '{0}(waiting={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.waiting, self.updated) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 966be8f1e..45b269a0b 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,4 +1,5 @@ """Tests for acme.errors.""" +import datetime import unittest import mock @@ -35,9 +36,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - exhausted=set([mock.sentinel.AR]), + waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], updated={}) - self.invalid = PollError(exhausted=set(), updated={ + self.invalid = PollError(waiting=[], updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -45,7 +46,7 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' + self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' 'sentinel.AR2})', repr(self.invalid)) From c05fa8934c3f3a56e94ab5d7db18c2aaa78b9e95 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:34:19 -0800 Subject: [PATCH 159/201] Revert "Cache Python packages" --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 09e580c3c..d9b4cb5ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python -cache: - directories: - - $HOME/.cache/pip - services: - rabbitmq - mariadb From 5535c0675b9f0602b26c23690ad6e2d246501008 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 18 Jan 2016 12:46:10 -0800 Subject: [PATCH 160/201] Revert "Revert "Fix "global" max_attempt bug (#1719)"" --- acme/acme/client.py | 30 ++++++++++++++++++++---------- acme/acme/client_test.py | 5 ++++- acme/acme/errors.py | 17 ++++++++--------- acme/acme/errors_test.py | 7 +++---- 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 49c6bcb21..478536ecc 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,4 +1,5 @@ """ACME client API.""" +import collections import datetime import heapq import logging @@ -334,8 +335,9 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts before - `PollError` with non-empty ``waiting`` is raised. + :param int max_attempts: Maximum number of attempts (per + authorization) before `PollError` with non-empty ``waiting`` + is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -349,6 +351,11 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ + # pylint: disable=too-many-locals + assert max_attempts > 0 + attempts = collections.defaultdict(int) + exhausted = set() + # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -356,8 +363,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting and max_attempts: - max_attempts -= 1 + while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -371,16 +377,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr + attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + if attempts[authzr] < max_attempts: + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) + else: + exhausted.add(authzr) - if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(waiting, updated) + if exhausted or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 449bd695e..9abc69c7c 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -319,7 +319,10 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime) + csr, authzrs, mintime=mintime, + # make sure that max_attempts is per-authorization, rather + # than global + max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 0385667c7..77d47c522 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,26 +56,25 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`waiting` will be non-empty) + This might be caused by either timeout (`exhausted` will be non-empty) or by some authorization being invalid. - :ivar waiting: Priority queue with `datetime.datatime` (based on - ``Retry-After``) as key, and original `.AuthorizationResource` - as value. + :ivar exhausted: Set of `.AuthorizationResource` that didn't finish + within max allowed attempts. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, waiting, updated): - self.waiting = waiting + def __init__(self, exhausted, updated): + self.exhausted = exhausted self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.waiting) + return bool(self.exhausted) def __repr__(self): - return '{0}(waiting={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.waiting, self.updated) + return '{0}(exhausted={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.exhausted, self.updated) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 45b269a0b..966be8f1e 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,5 +1,4 @@ """Tests for acme.errors.""" -import datetime import unittest import mock @@ -36,9 +35,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], + exhausted=set([mock.sentinel.AR]), updated={}) - self.invalid = PollError(waiting=[], updated={ + self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -46,7 +45,7 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' + self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' 'sentinel.AR2})', repr(self.invalid)) From 3a90b4c7c5e2e8656fd111dbf28d8d29344a32b3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 18 Jan 2016 21:39:25 +0000 Subject: [PATCH 161/201] acme: fix empty set repr py3 compat --- acme/acme/errors_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 966be8f1e..1e5f3d479 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -45,8 +45,8 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(exhausted=set([]), updated={sentinel.AR: ' - 'sentinel.AR2})', repr(self.invalid)) + self.assertEqual('PollError(exhausted=%s, updated={sentinel.AR: ' + 'sentinel.AR2})' % repr(set()), repr(self.invalid)) if __name__ == "__main__": From aefd5b25e12ad9ec31411e3896aa2a0fc074fd0f Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 19 Jan 2016 12:22:53 -0500 Subject: [PATCH 162/201] Revert switch to `python setup.py test` in tox.ini. This had more of a purpose when we were moving mock to test_requires. I'll reintroduce this in the separate PR for that. Also bring back the testing extra in tox for now. --- tox.ini | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tox.ini b/tox.ini index 54da87810..c54c9934c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,20 +8,23 @@ skipsdist = true envlist = py{26,27,33,34,35},py{26,27}-oldest,cover,lint +# nosetest -v => more verbose output, allows to detect busy waiting +# loops, especially on Travis + [testenv] # packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 commands = - pip install -e acme - python acme/setup.py test - pip install -e . - python setup.py test + pip install -e acme[testing] + nosetests -v acme + pip install -e .[testing] + nosetests -v letsencrypt pip install -e letsencrypt-apache - python letsencrypt-apache/setup.py test + nosetests -v letsencrypt_apache pip install -e letsencrypt-nginx - python letsencrypt-nginx/setup.py test + nosetests -v letsencrypt_nginx pip install -e letshelp-letsencrypt - python letshelp-letsencrypt/setup.py test + nosetests -v letshelp_letsencrypt setenv = PYTHONPATH = {toxinidir} @@ -37,18 +40,18 @@ deps = [testenv:py33] commands = - pip install -e acme - python acme/setup.py test + pip install -e acme[testing] + nosetests -v acme [testenv:py34] commands = - pip install -e acme - python acme/setup.py test + pip install -e acme[testing] + nosetests -v acme [testenv:py35] commands = - pip install -e acme - python acme/setup.py test + pip install -e acme[testing] + nosetests -v acme [testenv:cover] basepython = python2.7 From b20eab67cef3002f184d0e4732b590994cff94ba Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Tue, 19 Jan 2016 12:30:34 -0500 Subject: [PATCH 163/201] Remove errant DS_Store. Ick. --- .../tests/fake-letsencrypt/dist/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Tue, 19 Jan 2016 17:56:20 -0500 Subject: [PATCH 164/201] Remove nosetests -v option from setup.cfg, and add trailing newline. --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4c9007edb..ca4c1b1ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,5 +9,3 @@ nocapture=1 cover-package=letsencrypt,acme,letsencrypt_apache,letsencrypt_nginx cover-erase=1 cover-tests=1 -# More verbose output: allows to detect busy waiting loops, especially on Travis -verbosity=1 \ No newline at end of file From 9e00cd5c2c45b622e4f2b1d7848735e6e9a72be0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 Jan 2016 15:47:50 -0800 Subject: [PATCH 165/201] Retry enabling pip cache --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7f6a2f87c..67da27d00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python +cache: + directories: + - $HOME/.cache/pip + services: - rabbitmq - mariadb From 65fbeede6955f0863aa5c3f82c9bba53e340c5a7 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 20 Jan 2016 16:24:21 -0500 Subject: [PATCH 166/201] Downgrade declared ConfigArgParse requirement. Fix #2243. --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4d9cc7850..cc2bb449a 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,10 @@ version = meta['version'] # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), - 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 + # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but + # saying so here causes a runtime error against our temporary fork of 0.9.3 + # in which we added 2.6 support (see #2243), so we relax the requirement. + 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', From 15cc9e182696c2c7be971a1687ea0ce011b625b5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 14:17:24 -0800 Subject: [PATCH 167/201] [test_leauto_upgrades] actually use everything from v0.1.0 for setup --- tests/letstest/scripts/test_leauto_upgrades.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index b7849755a..100a7f6b6 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -8,11 +8,23 @@ cd letsencrypt SAVE="$PIP_EXTRA_INDEX_URL" unset PIP_EXTRA_INDEX_URL export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" -./letsencrypt-auto -v --debug --version + +#OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" + +if ! command -v git ; then + if ! ( sudo apt-get update || sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then + echo git installation failed! + exit 1 + fi +fi +BRANCH=`git rev-parse --abbrev-ref HEAD` +git checkout v0.1.0 +./letsencrypt-auto -v --debug --version unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" +git checkout -f "$BRANCH" if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then echo upgrade appeared to fail exit 1 From 122a36ebd23c7f66e2efffcf2e6d3ac7f54d192b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 14:22:28 -0800 Subject: [PATCH 168/201] the || isn't right for update --- tests/letstest/scripts/test_leauto_upgrades.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 100a7f6b6..f0560e025 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -11,8 +11,12 @@ export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" #OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" + if ! command -v git ; then - if ! ( sudo apt-get update || sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then + if [ "$OS_TYPE" = "ubuntu" ] ; then + sudo apt-get update + fi + if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then echo git installation failed! exit 1 fi From b8281fdd345158397c27ab39e6fad6345d14365d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 14:25:01 -0800 Subject: [PATCH 169/201] Retry in case of ephemeral errors --- tests/letstest/multitester.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index dee6968c3..19a6aad1a 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -139,7 +139,15 @@ def make_instance(instance_name, time.sleep(1.0) # give instance a name - new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + try: + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + except botocore.exceptions.ClientError, e: + if "InvalidInstanceID.NotFound" in str(e): + # This seems to be ephemeral... retry + time.sleep(1) + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + else: + raise return new_instance def terminate_and_clean(instances): From 45f32f9cdc568e7f379fa444f55785f30a8cc08c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 16:01:37 -0800 Subject: [PATCH 170/201] Do not say we've renewed a cert if it was reinstalled --- letsencrypt/cli.py | 8 ++++---- letsencrypt/display/ops.py | 8 +++++--- letsencrypt/tests/cli_test.py | 4 +++- letsencrypt/tests/display/ops_test.py | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index db6519af1..dfb2a0945 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -382,7 +382,7 @@ def _auth_from_domains(le_client, config, domains): if action == "reinstall": # The lineage already exists; allow the caller to try installing # it without getting a new certificate at all. - return lineage + return lineage, "reinstall" elif action == "renew": original_server = lineage.configuration["renewalparams"]["server"] _avoid_invalidating_lineage(config, lineage, original_server) @@ -407,7 +407,7 @@ def _auth_from_domains(le_client, config, domains): _report_new_cert(lineage.cert, lineage.fullchain) - return lineage + return lineage, action def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" @@ -556,7 +556,7 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo # TODO: Handle errors from _init_le_client? le_client = _init_le_client(args, config, authenticator, installer) - lineage = _auth_from_domains(le_client, config, domains) + lineage, action = _auth_from_domains(le_client, config, domains) le_client.deploy_certificate( domains, lineage.privkey, lineage.cert, @@ -567,7 +567,7 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo if len(lineage.available_versions("cert")) == 1: display_ops.success_installation(domains) else: - display_ops.success_renewal(domains) + display_ops.success_renewal(domains, action) _suggest_donate() diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 5ceb7fcfc..a82f84d6d 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -309,22 +309,24 @@ def success_installation(domains): pause=False) -def success_renewal(domains): +def success_renewal(domains, action): """Display a box confirming the renewal of an existing certificate. .. todo:: This should be centered on the screen :param list domains: domain names which were renewed + :param str action: can be "reinstall" or "renew" """ util(interfaces.IDisplay).notification( - "Your existing certificate has been successfully renewed, and the " + "Your existing certificate has been successfully {3}ed, and the " "new certificate has been installed.{1}{1}" "The new certificate covers the following domains: {0}{1}{1}" "You should test your configuration at:{1}{2}".format( _gen_https_names(domains), os.linesep, - os.linesep.join(_gen_ssl_lab_urls(domains))), + os.linesep.join(_gen_ssl_lab_urls(domains)), + action), height=(14 + len(domains)), pause=False) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 16ef5c093..e9058163e 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -134,12 +134,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods @mock.patch('letsencrypt.cli._determine_account') @mock.patch('letsencrypt.cli.client.Client.obtain_and_enroll_certificate') @mock.patch('letsencrypt.cli._auth_from_domains') - def test_user_agent(self, _afd, _obt, det, _client): + def test_user_agent(self, afd, _obt, det, _client): # Normally the client is totally mocked out, but here we need more # arguments to automate it... args = ["--standalone", "certonly", "-m", "none@none.com", "-d", "example.com", '--agree-tos'] + self.standard_args det.return_value = mock.MagicMock(), None + afd.return_value = mock.MagicMock(), "newcert" + with mock.patch('letsencrypt.cli.client.acme_client.ClientNetwork') as acme_net: self._call_no_clientmock(args) os_ver = " ".join(le_util.get_os_info()) diff --git a/letsencrypt/tests/display/ops_test.py b/letsencrypt/tests/display/ops_test.py index d98afe180..5f7a86785 100644 --- a/letsencrypt/tests/display/ops_test.py +++ b/letsencrypt/tests/display/ops_test.py @@ -465,7 +465,7 @@ class SuccessRenewalTest(unittest.TestCase): @classmethod def _call(cls, names): from letsencrypt.display.ops import success_renewal - success_renewal(names) + success_renewal(names, "renew") @mock.patch("letsencrypt.display.ops.util") def test_success_renewal(self, mock_util): From e112e2ce6126845afb06028abdc81008cbbde507 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 17:16:19 -0800 Subject: [PATCH 171/201] Remove pylint disable --- letsencrypt/display/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 2ea6b9bfe..602c246de 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -21,7 +21,7 @@ CANCEL = "cancel" HELP = "help" """Display exit code when for when the user requests more help.""" -def _wrap_lines(msg): # pylint: disable=no-self-use +def _wrap_lines(msg): """Format lines nicely to 80 chars. :param str msg: Original message From 22dccf0adb30369351a7c062f90934073de9451a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 17:16:32 -0800 Subject: [PATCH 172/201] Use sphinx backticks more consistently --- letsencrypt/display/util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 602c246de..47f54cd5f 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -77,9 +77,9 @@ class NcursesDisplay(object): :param str help_label: label of the help button :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, tag) where - code - int display exit code - tag - str corresponding to the item chosen + :returns: tuple of the form (`code`, `tag`) where + `code` - int display exit code + `tag` - str corresponding to the item chosen :rtype: tuple """ @@ -126,7 +126,7 @@ class NcursesDisplay(object): :param str message: Message to display that asks for input. :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, string) where + :returns: tuple of the form (`code`, `string`) where `code` - int display exit code `string` - input entered by the user @@ -166,7 +166,7 @@ class NcursesDisplay(object): :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, list_tags) where + :returns: tuple of the form (`code`, `list_tags`) where `code` - int display exit code `list_tags` - list of str tags selected by the user From 410bd227936b481e41a648ec1b1ce264c856f63b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 17:21:40 -0800 Subject: [PATCH 173/201] As previously implemented, iDisplay.menu() returns an index, not a tag --- letsencrypt/display/util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 47f54cd5f..dde00584f 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -77,9 +77,9 @@ class NcursesDisplay(object): :param str help_label: label of the help button :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (`code`, `tag`) where + :returns: tuple of the form (`code`, `index`) where `code` - int display exit code - `tag` - str corresponding to the item chosen + `int` - index of the selected item :rtype: tuple """ @@ -112,12 +112,12 @@ class NcursesDisplay(object): (str(i), choice) for i, choice in enumerate(choices, 1) ] # pylint: disable=star-args - code, tag = self.dialog.menu(message, **menu_options) + code, index = self.dialog.menu(message, **menu_options) if code == CANCEL: return code, -1 - return code, int(tag) - 1 + return code, int(index) - 1 def input(self, message, **_kwargs): From 8c9757a06255c4486608956fa5226d8812a1888d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 20 Jan 2016 17:55:45 -0800 Subject: [PATCH 174/201] Correct docstring --- letsencrypt/display/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index dde00584f..a91c5bab1 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -412,7 +412,7 @@ class FileDisplay(object): return OK, selection class NoninteractiveDisplay(object): - """File-based display.""" + """An iDisplay implementation that never asks for interactive user input""" zope.interface.implements(interfaces.IDisplay) From b8690cd47147dcc162dadf2c53951624ab8ed102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20HUBSCHER?= Date: Thu, 21 Jan 2016 10:11:23 +0100 Subject: [PATCH 175/201] Make wheel universal --- acme/setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 acme/setup.cfg diff --git a/acme/setup.cfg b/acme/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/acme/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From 4c8f5fff8ccaa3b79ccee58393f70c7d40148017 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 21 Jan 2016 11:37:32 -0500 Subject: [PATCH 176/201] Fixed formatting of code blocks --- docs/ciphers.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 49c0824a3..fb854f307 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -170,7 +170,7 @@ Changing your settings This will probably look something like -..code-block: shell +.. code-block:: shell letsencrypt --cipher-recommendations mozilla-secure letsencrypt --cipher-recommendations mozilla-intermediate @@ -179,14 +179,14 @@ This will probably look something like to track Mozilla's *Secure*, *Intermediate*, or *Old* recommendations, and -..code-block: shell +.. code-block:: shell letsencrypt --update-ciphers on to enable updating ciphers with each new Let's Encrypt client release, or -..code-block: shell +.. code-block:: shell letsencrypt --update-ciphers off From b75235b3dc80bd79a06a9800d956403a4517e823 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 21 Jan 2016 15:27:23 -0800 Subject: [PATCH 177/201] Close test coverage gaps (And fix a bug in one test...) --- letsencrypt/tests/display/util_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/display/util_test.py b/letsencrypt/tests/display/util_test.py index 8759a7faa..a16eb544e 100644 --- a/letsencrypt/tests/display/util_test.py +++ b/letsencrypt/tests/display/util_test.py @@ -291,6 +291,12 @@ class NoninteractiveDisplayTest(unittest.TestCase): self.mock_stdout = mock.MagicMock() self.displayer = display_util.NoninteractiveDisplay(self.mock_stdout) + def test_notification_no_pause(self): + self.displayer.notification("message", 10) + string = self.mock_stdout.write.call_args[0][0] + + self.assertTrue("message" in string) + def test_input(self): d = "an incomputable value" ret = self.displayer.input("message", default=d) @@ -310,7 +316,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): def test_checklist(self): d = [1, 3] - ret = self.displayer.menu("message", TAGS, default=d) + ret = self.displayer.checklist("message", TAGS, default=d) self.assertEqual(ret, (display_util.OK, d)) self.assertRaises(errors.MissingCommandlineFlag, self.displayer.checklist, "message", TAGS) From 467e8fdb049395f624c51cbb0b18c8d8935c562b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 21 Jan 2016 15:38:38 -0800 Subject: [PATCH 178/201] [interfaces.py] add missing :raises: for iDisplay.input --- letsencrypt/interfaces.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index ced84bc54..db5d2c5e8 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -399,6 +399,9 @@ class IDisplay(zope.interface.Interface): `input` - str of the user's input :rtype: tuple + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ def yesno(message, yes_label="Yes", no_label="No", default=None, From 81d9ed220c009126f7e2ad74f4838ab7dce74a25 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 21 Jan 2016 15:44:37 -0800 Subject: [PATCH 179/201] Less elaborate exception processing --- letsencrypt/cli.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4eaadb5f6..0eb9595ca 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -647,17 +647,14 @@ def install(args, config, plugins): except errors.PluginSelectionError, e: return e.message - try: - domains = _find_domains(args, installer) - le_client = _init_le_client( - args, config, authenticator=None, installer=installer) - assert args.cert_path is not None # required=True in the subparser - le_client.deploy_certificate( - domains, args.key_path, args.cert_path, args.chain_path, - args.fullchain_path) - le_client.enhance_config(domains, config) - except errors.MissingCommandlineFlag, e: - return e.message + domains = _find_domains(args, installer) + le_client = _init_le_client( + args, config, authenticator=None, installer=installer) + assert args.cert_path is not None # required=True in the subparser + le_client.deploy_certificate( + domains, args.key_path, args.cert_path, args.chain_path, + args.fullchain_path) + le_client.enhance_config(domains, config) def revoke(args, config, unused_plugins): # TODO: coop with renewal config From 9a1199ed24b3346689b0a3e7eec8815ae0f1e9c4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 21 Jan 2016 15:50:45 -0800 Subject: [PATCH 180/201] Wrangle a lot of **_kwargs --- letsencrypt/display/util.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index a91c5bab1..12c32ff05 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -64,7 +64,7 @@ class NcursesDisplay(object): self.dialog.msgbox(message, height, width=self.width) def menu(self, message, choices, ok_label="OK", cancel_label="Cancel", - help_label="", **_kwargs): + help_label="", **unused_kwargs): """Display a menu. :param str message: title of menu @@ -75,7 +75,7 @@ class NcursesDisplay(object): :param str ok_label: label of the OK button :param str help_label: label of the help button - :param dict _kwargs: absorbs default / cli_args + :param dict unused_kwargs: absorbs default / cli_args :returns: tuple of the form (`code`, `index`) where `code` - int display exit code @@ -120,7 +120,7 @@ class NcursesDisplay(object): return code, int(index) - 1 - def input(self, message, **_kwargs): + def input(self, message, **unused_kwargs): """Display an input box to the user. :param str message: Message to display that asks for input. @@ -138,7 +138,7 @@ class NcursesDisplay(object): return self.dialog.inputbox(message, width=self.width, height=height) - def yesno(self, message, yes_label="Yes", no_label="No", **_kwargs): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Display a Yes/No dialog box. Yes and No label must begin with different letters. @@ -156,7 +156,7 @@ class NcursesDisplay(object): message, self.height, self.width, yes_label=yes_label, no_label=no_label) - def checklist(self, message, tags, default_status=True, **_kwargs): + def checklist(self, message, tags, default_status=True, **unused_kwargs): """Displays a checklist. :param message: Message to display before choices @@ -204,7 +204,7 @@ class FileDisplay(object): raw_input("Press Enter to Continue") def menu(self, message, choices, ok_label="", cancel_label="", - help_label="", **_kwargs): + help_label="", **unused_kwargs): # pylint: disable=unused-argument """Display a menu. @@ -230,7 +230,7 @@ class FileDisplay(object): return code, selection - 1 - def input(self, message, **_kwargs): + def input(self, message, **unused_kwargs): # pylint: disable=no-self-use """Accept input from the user. @@ -251,7 +251,7 @@ class FileDisplay(object): else: return OK, ans - def yesno(self, message, yes_label="Yes", no_label="No", **_kwargs): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Query the user with a yes/no question. Yes and No label must begin with different letters, and must contain at @@ -287,7 +287,7 @@ class FileDisplay(object): ans.startswith(no_label[0].upper())): return False - def checklist(self, message, tags, default_status=True, **_kwargs): + def checklist(self, message, tags, default_status=True, **unused_kwargs): # pylint: disable=unused-argument """Display a checklist. @@ -445,8 +445,9 @@ class NoninteractiveDisplay(object): "{line}{frame}{line}{msg}{line}{frame}{line}".format( line=os.linesep, frame=side_frame, msg=message)) - def menu(self, message, choices, default=None, cli_flag=None, **kwargs): - # pylint: disable=unused-argument + def menu(self, message, choices, ok_label=None, cancel_label=None, + default=None, cli_flag=None): + # pylint: disable=unused-argument,too-many-arguments """Avoid displaying a menu. :param str message: title of menu From e6c608b0a69ecd87318f2eef13874a5be94a8fe1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 21 Jan 2016 15:55:44 -0800 Subject: [PATCH 181/201] Do not autoexpand --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0eb9595ca..ab04b906d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -323,7 +323,7 @@ def _handle_subset_cert_request(config, domains, cert): br=os.linesep) if config.expand or config.renew_by_default or zope.component.getUtility( interfaces.IDisplay).yesno(question, "Expand", "Cancel", - default=True): + cli_flag="--expand (or in some cases, --duplicate)"): return "renew", cert else: reporter_util = zope.component.getUtility(interfaces.IReporter) From 0802ade04eec84d7071e3052c3c05fc368741c7a Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 21 Jan 2016 15:59:30 -0800 Subject: [PATCH 182/201] fix apache 2.2 --- letsencrypt-apache/letsencrypt_apache/configurator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 8818923f4..546211696 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -557,8 +557,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # search for NameVirtualHost directive for ip_addr # note ip_addr can be FQDN although Apache does not recommend it - return (self.version >= (2, 4) or - self.parser.find_dir("NameVirtualHost", str(target_addr))) + if (self.version >= (2,4)): + return True + else: + self.save("don't lose config changes", True) + return (self.parser.find_dir("NameVirtualHost", str(target_addr))) def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. From 66dbd23f2b8e687d8e83758fd17c463a8b41d1be Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 22 Jan 2016 00:07:50 -0500 Subject: [PATCH 183/201] Upgrade peep to 3.0. This will avoid crashing when used with pip 8.x, which was released today and is already the 3rd most used client against PyPI. (7.1.2 and 1.5.4 take spots 1 and 2, respectively.) --- letsencrypt-auto-source/letsencrypt-auto | 12 +++++++----- letsencrypt-auto-source/pieces/peep.py | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d0cccc08e..3cdd49549 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.2.0.dev0" +LE_AUTO_VERSION="0.2.1.dev0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -745,7 +745,7 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar -__version__ = 2, 5, 0 +__version__ = 3, 0, 0 try: from pip.index import FormatControl # noqa @@ -1003,9 +1003,11 @@ def package_finder(argv): # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ - 'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified', - 'allow_all_external', ('allow_all_prereleases', 'pre'), - 'process_dependency_links'] + 'find_links', + FORMAT_CONTROL_ARG, + ('allow_all_prereleases', 'pre'), + 'process_dependency_links' + ] kwargs = {} for option in possible_options: kw, attr = option if isinstance(option, tuple) else (option, option) diff --git a/letsencrypt-auto-source/pieces/peep.py b/letsencrypt-auto-source/pieces/peep.py index 6b9393a5e..c4e51f483 100755 --- a/letsencrypt-auto-source/pieces/peep.py +++ b/letsencrypt-auto-source/pieces/peep.py @@ -104,7 +104,7 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar -__version__ = 2, 5, 0 +__version__ = 3, 0, 0 try: from pip.index import FormatControl # noqa @@ -362,9 +362,11 @@ def package_finder(argv): # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ - 'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified', - 'allow_all_external', ('allow_all_prereleases', 'pre'), - 'process_dependency_links'] + 'find_links', + FORMAT_CONTROL_ARG, + ('allow_all_prereleases', 'pre'), + 'process_dependency_links' + ] kwargs = {} for option in possible_options: kw, attr = option if isinstance(option, tuple) else (option, option) From b75b887a837add11055bfa0a0e18aadaa5ebb69f Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 22 Jan 2016 10:03:29 -0800 Subject: [PATCH 184/201] fixed linting issues --- letsencrypt-apache/letsencrypt_apache/configurator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 546211696..65e7a14a8 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -557,11 +557,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # search for NameVirtualHost directive for ip_addr # note ip_addr can be FQDN although Apache does not recommend it - if (self.version >= (2,4)): + if self.version >= (2, 4): return True else: self.save("don't lose config changes", True) - return (self.parser.find_dir("NameVirtualHost", str(target_addr))) + return self.parser.find_dir("NameVirtualHost", str(target_addr)) def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. From 5192fa36ab25fe7a5dbeb757b1d5308e20d7c754 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 22 Jan 2016 11:47:49 -0800 Subject: [PATCH 185/201] move save command up to tls_sni --- letsencrypt-apache/letsencrypt_apache/configurator.py | 7 ++----- letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 65e7a14a8..8818923f4 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -557,11 +557,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # search for NameVirtualHost directive for ip_addr # note ip_addr can be FQDN although Apache does not recommend it - if self.version >= (2, 4): - return True - else: - self.save("don't lose config changes", True) - return self.parser.find_dir("NameVirtualHost", str(target_addr)) + return (self.version >= (2, 4) or + self.parser.find_dir("NameVirtualHost", str(target_addr))) def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index 971072311..cc1d749a0 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -76,6 +76,7 @@ class ApacheTlsSni01(common.TLSSNI01): # Setup the configuration addrs = self._mod_config() + self.configurator.save("Don't lose mod_config changes", True) self.configurator.make_addrs_sni_ready(addrs) # Save reversible changes From 30db2372b5afcd8535142aac2960aa95e5cb7ef0 Mon Sep 17 00:00:00 2001 From: Sorvani Date: Mon, 7 Dec 2015 21:40:48 -0600 Subject: [PATCH 186/201] Update _rpm_common.sh fixes #1823 Add check for python-tools and python-pip --- bootstrap/_rpm_common.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh index db1665268..73890155e 100755 --- a/bootstrap/_rpm_common.sh +++ b/bootstrap/_rpm_common.sh @@ -3,6 +3,7 @@ # Tested with: # - Fedora 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) +# - CentOS 7 Minimal install in a Hyper-V VM if type dnf 2>/dev/null then @@ -21,12 +22,16 @@ fi if ! $tool install -y \ python \ python-devel \ - python-virtualenv + python-virtualenv \ + python-tools \ + python-pip then if ! $tool install -y \ python27 \ python27-devel \ - python27-virtualenv + python27-virtualenv \ + python27-tools \ + python27-pip then echo "Could not install Python dependencies. Aborting bootstrap!" exit 1 From 55dba783c00608c84db6b111c0bf56261787dee2 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 22 Jan 2016 15:18:33 -0500 Subject: [PATCH 187/201] Port bootstrapper fixes to the new le-auto's bootstrappers. --- letsencrypt-auto-source/letsencrypt-auto | 9 +++++++-- .../pieces/bootstrappers/rpm_common.sh | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 3cdd49549..9eebddc9d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -213,6 +213,7 @@ BootstrapRpmCommon() { # Tested with: # - Fedora 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM if type dnf 2>/dev/null then @@ -231,12 +232,16 @@ BootstrapRpmCommon() { if ! $SUDO $tool install -y \ python \ python-devel \ - python-virtualenv + python-virtualenv \ + python-tools \ + python-pip then if ! $SUDO $tool install -y \ python27 \ python27-devel \ - python27-virtualenv + python27-virtualenv \ + python27-tools \ + python27-pip then echo "Could not install Python dependencies. Aborting bootstrap!" exit 1 diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 8a8f1526a..68a11a531 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -2,6 +2,7 @@ BootstrapRpmCommon() { # Tested with: # - Fedora 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM if type dnf 2>/dev/null then @@ -20,12 +21,16 @@ BootstrapRpmCommon() { if ! $SUDO $tool install -y \ python \ python-devel \ - python-virtualenv + python-virtualenv \ + python-tools \ + python-pip then if ! $SUDO $tool install -y \ python27 \ python27-devel \ - python27-virtualenv + python27-virtualenv \ + python27-tools \ + python27-pip then echo "Could not install Python dependencies. Aborting bootstrap!" exit 1 From 32f703b6b2f28ef710180979a280506ada1b8009 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 22 Jan 2016 16:37:36 -0800 Subject: [PATCH 188/201] Ignore renewal configs that don't end in .conf Should fix #2254 --- letsencrypt/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 16e305cfd..3c343eae3 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -194,6 +194,8 @@ def _find_duplicative_certs(config, domains): le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) for renewal_file in os.listdir(configs_dir): + if not renewal_file.endswith(".conf"): + continue try: full_path = os.path.join(configs_dir, renewal_file) candidate_lineage = storage.RenewableCert(full_path, cli_config) From 904f44864e3b1721b52ac1ce0b0088fef0b13d19 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 22 Jan 2016 16:44:54 -0800 Subject: [PATCH 189/201] rm stray print --- letsencrypt/tests/cli_test.py | 1 - tests/letstest/scripts/test_leauto_upgrades.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 75c69b502..bc0fa6892 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -138,7 +138,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods try: with mock.patch('letsencrypt.cli.sys.stderr'): out = cli.main(self.standard_args + args[:]) # NOTE: parser can alter its args! - print out except errors.MissingCommandlineFlag, exc: self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index b7849755a..262839ab1 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -13,7 +13,7 @@ unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" -if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then +if ! ./letsencrypt-auto -v --debug --version | grep 0.2.0 ; then echo upgrade appeared to fail exit 1 fi From 260534d1c3b91ead22ea14c91f79f978410e97a5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 22 Jan 2016 17:23:13 -0800 Subject: [PATCH 190/201] Moderate warning label for cli.cli_command --- letsencrypt/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ab04b906d..f1bf00058 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -46,7 +46,11 @@ logger = logging.getLogger(__name__) # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: -# /home/user/.local/share/letsencrypt/bin/letsencrypt" +# "/home/user/.local/share/letsencrypt/bin/letsencrypt" +# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before running +# letsencrypt-auto (and sudo stops us from seing if they did), so it should only be used +# for purposes where inability to detect letsencrypt-auto fails safely + fragment = os.path.join(".local", "share", "letsencrypt") cli_command = "letsencrypt-auto" if fragment in sys.argv[0] else "letsencrypt" From 58b50ba0083e809cf4da762172b4b6792e281145 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 22 Jan 2016 17:27:45 -0800 Subject: [PATCH 191/201] fix typo --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f1bf00058..1a1d09274 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -48,7 +48,7 @@ logger = logging.getLogger(__name__) # When invoked from letsencrypt-auto, sys.argv[0] is something like: # "/home/user/.local/share/letsencrypt/bin/letsencrypt" # Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before running -# letsencrypt-auto (and sudo stops us from seing if they did), so it should only be used +# letsencrypt-auto (and sudo stops us from seeing if they did), so it should only be used # for purposes where inability to detect letsencrypt-auto fails safely fragment = os.path.join(".local", "share", "letsencrypt") From b5af4264bcb82120a97c57404d1dd643bd5250f1 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Tue, 19 Jan 2016 20:40:49 +0200 Subject: [PATCH 192/201] Adding instructions on how to install Go 1.5.3 on Linux --- docs/contributing.rst | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 5ec44470d..e83657386 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -96,11 +96,32 @@ Integration testing with the boulder CA Generally it is sufficient to open a pull request and let Github and Travis run integration tests for you. -Mac OS X users: Run `./tests/mac-bootstrap.sh` instead of `boulder-start.sh` to -install dependencies, configure the environment, and start boulder. +Mac OS X users: Run ``./tests/mac-bootstrap.sh`` instead of +``boulder-start.sh`` to install dependencies, configure the +environment, and start boulder. -Otherwise, install `Go`_ 1.5, libtool-ltdl, mariadb-server and -rabbitmq-server and then start Boulder_, an ACME CA server:: +Otherwise, install `Go`_ 1.5, ``libtool-ltdl``, ``mariadb-server`` and +``rabbitmq-server`` and then start Boulder_, an ACME CA server. + +If you can't get packages of Go 1.5 for your Linux system, +you can execute the following commands to install it: + +.. code-block:: shell + + wget https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz -P /tmp/ + sudo tar -C /usr/local -xzf /tmp/go1.5.3.linux-amd64.tar.gz + if ! grep -Fxq "export GOROOT=/usr/local/go" ~/.profile ; then echo "export GOROOT=/usr/local/go" >> ~/.profile; fi + if ! grep -Fxq "export PATH=\\$GOROOT/bin:\\$PATH" ~/.profile ; then echo "export PATH=\\$GOROOT/bin:\\$PATH" >> ~/.profile; fi + +These commands download `Go`_ 1.5.3 to ``/tmp/``, extracts to ``/usr/local``, +and then adds the export lines required to execute ``boulder-start.sh`` to +``~/.profile`` if they were not previously added + +Make sure you execute the following command after `Go`_ finishes installing:: + + if ! grep -Fxq "export GOPATH=\\$HOME/go" ~/.profile ; then echo "export GOPATH=\\$HOME/go" >> ~/.profile; fi + +Afterwards, you'd be able to start Boulder_ using the following command:: ./tests/boulder-start.sh From ca75532328b03c93b7585ced963ed241fca57300 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 25 Jan 2016 12:52:10 -0800 Subject: [PATCH 193/201] Same fix to renewer.py (don't read non-.conf) --- letsencrypt/renewer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 2d2675745..83c6106c0 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -172,6 +172,8 @@ def main(cli_args=sys.argv[1:]): constants.CONFIG_DIRS_MODE, uid) for renewal_file in os.listdir(cli_config.renewal_configs_dir): + if not renewal_file.endswith(".conf"): + continue print("Processing " + renewal_file) try: # TODO: Before trying to initialize the RenewableCert object, From 2d0f66ea32e573c80bed360ac6a56e2ad8c82d67 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 25 Jan 2016 12:52:27 -0800 Subject: [PATCH 194/201] Test now makes a non-.conf file to be ignored --- letsencrypt/tests/renewer_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index a103f5dbf..7030e4998 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -68,6 +68,13 @@ class BaseRenewableCertTest(unittest.TestCase): config.write() self.config = config + # We also create a file that isn't a renewal config in the same + # location to test that logic that reads in all-and-only renewal + # configs will ignore it and NOT attempt to parse it. + junk = open(os.path.join(self.tempdir, "renewal", "IGNORE.THIS"), "w") + junk.write("This file should be ignored!") + junk.close() + self.defaults = configobj.ConfigObj() self.test_rc = storage.RenewableCert(config.filename, self.cli_config) From b7c9ed1b996e4cb216bd37487e39efc10cc66be7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 25 Jan 2016 15:36:00 -0800 Subject: [PATCH 195/201] lintmonster --- letsencrypt/tests/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index bc0fa6892..be7b8acda 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -137,7 +137,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods exc = None try: with mock.patch('letsencrypt.cli.sys.stderr'): - out = cli.main(self.standard_args + args[:]) # NOTE: parser can alter its args! + cli.main(self.standard_args + args[:]) # NOTE: parser can alter its args! except errors.MissingCommandlineFlag, exc: self.assertTrue(message in str(exc)) self.assertTrue(exc is not None) From 558f29bf836d085fb4c6e69898405eefbd609075 Mon Sep 17 00:00:00 2001 From: Ola Bini Date: Wed, 27 Jan 2016 09:03:09 -0500 Subject: [PATCH 196/201] Update comment to clarify that the ranking takes into account SSL --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index efa7e08b4..691688969 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -239,6 +239,7 @@ class NginxConfigurator(common.Plugin): def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. + The ranking gives preference to SSL vhosts. :param str target_name: The name to match :returns: list of dicts containing the vhost, the matching name, and From 7c6678b87366a20ae343488383f991ed6e531525 Mon Sep 17 00:00:00 2001 From: bmw Date: Wed, 27 Jan 2016 10:43:56 -0800 Subject: [PATCH 197/201] Revert "Revert "Temporarily disable Apache 2.2 support"" --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 8818923f4..2d822b3a1 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -155,7 +155,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: self.version = self.get_version() - if self.version < (2, 2): + if self.version < (2, 4): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) From ac0a15d48cdb90878eed337677344c62330465a9 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 27 Jan 2016 15:35:06 -0500 Subject: [PATCH 198/201] Add ordereddict, a conditional dependency of ConfigArgParse under Python 2.6. Ref #2200. It doesn't hurt under 2.7. --- letsencrypt-auto-source/letsencrypt-auto | 5 ++++- .../pieces/letsencrypt-auto-requirements.txt | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9eebddc9d..89fa373d3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -438,7 +438,7 @@ if [ "$NO_SELF_UPGRADE" = 1 ]; then # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, +# this, do `pip install -e acme -e . -e letsencrypt-apache`, `pip freeze`, # and then gather the hashes. # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ @@ -511,6 +511,9 @@ ipaddress==1.0.16 # sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ ndg-httpsclient==0.4.0 +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index bbda9f0b2..6980f4f73 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -1,5 +1,5 @@ # This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install -r -e acme -e . -e letsencrypt-apache`, `pip freeze`, +# this, do `pip install -e acme -e . -e letsencrypt-apache`, `pip freeze`, # and then gather the hashes. # sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ @@ -72,6 +72,9 @@ ipaddress==1.0.16 # sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ ndg-httpsclient==0.4.0 +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + # sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs # sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs parsedatetime==1.5 From d1d23b118fb661c26c6a1b5c8f1cdec09fa44ad7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 27 Jan 2016 13:16:11 -0800 Subject: [PATCH 199/201] Did it --- tools/release.sh | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/tools/release.sh b/tools/release.sh index d7dc4b6c6..9d625191e 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -81,21 +81,6 @@ if [ "$RELEASE_BRANCH" != "candidate-$version" ] ; then fi git checkout "$RELEASE_BRANCH" -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# and that it's signed correctly -if ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; then - echo Failed letsencrypt-auto signature check on "$RELEASE_BRANCH" - echo please fix that and re-run - exit 1 -else - echo Signature check on letsencrypt-auto successful -fi - - SetVersion() { ver="$1" for pkg_dir in $SUBPKGS letsencrypt-compatibility-test @@ -110,9 +95,6 @@ SetVersion() { } SetVersion "$version" -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" \ - --sign --message "Release $version" "$tag" echo "Preparing sdists and wheels" for pkg_dir in . $SUBPKGS @@ -175,6 +157,21 @@ for module in letsencrypt $subpkgs_modules ; do done deactivate +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" +done + +git diff --cached +git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +git tag --local-user "$RELEASE_GPG_KEY" \ + --sign --message "Release $version" "$tag" + cd .. echo Now in $PWD name=${root_without_le%.*} From cf218dd7f15d03414e6b850729492884c4676a28 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 27 Jan 2016 15:02:14 -0800 Subject: [PATCH 200/201] Release 0.3.0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 20 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/letsencrypt-auto-requirements.txt | 18 ++++++++-------- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index d9875c169..d81c13423 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index cbac3e0b2..af83c9b60 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.1.dev0' +version = '0.3.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 89fa373d3..e0812501c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.2.1.dev0" +LE_AUTO_VERSION="0.3.0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -628,17 +628,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: fYwCUXn3Wd_tKYHuPfufzDQZuDNng0HZb_th3xepC7U -# sha256: dKkCf9CZnKaDTFOof-KoazoewjKTSAVxZUJmnj_3i_U -acme==0.2.0 +# sha256: QMIkIvGF3mcJhGLAKRX7n5EVIPjOrfLtklN6ePjbJes +# sha256: fNFWiij6VxfG5o7u3oNbtrYKQ4q9vhzOLATfxNlozvQ +acme==0.3.0 -# sha256: 4x7K5lzKwm_GjYMojvUh053qL4EfIC5hGFmW370-7jI -# sha256: kcm3VmxXIGNS7ShcKFnYdA9AfXnqcbV_otMsADr1p2A -letsencrypt==0.2.0 +# sha256: qdnzpoRf_44QXKoktNoAKs2RBAxUta2Sr6GS0t_tAKo +# sha256: ELWJaHNvBZIqVPJYkla8yXLtXIuamqAf6f_VAFv16Uk +letsencrypt==0.3.0 -# sha256: AKuIT6b7gXXD2Cs7Qoem8ZrxcqBjABz1IgxhHGxmwX0 -# sha256: Cak7i4RaDsZixQMXWWpW-blTHaak09l94aLi9v7lljs -letsencrypt-apache==0.2.0 +# sha256: EypLpEw3-Tr8unw4aSFsHXgRiU8ZYLrJKOJohP2tC9M +# sha256: HYvP13GzA-DDJYwlfOoaraJO0zuYO48TCSAyTUAGCqA +letsencrypt-apache==0.3.0 # sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 # sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7db9da58e067d7e48975769a81e467cb06234515..4bb5f97ea686dd282f99591ef1d8416dd002942d 100644 GIT binary patch literal 256 zcmV+b0ssEr`(`t5&`>Jt>x>2a9mQS9O6fDHCDL_8lB$tS0lnsR{$ToQQaX9Rhs?!? zX%XvgMxhK&1EB+aICOQ`&uqrPI6(_);_;FS#$BTKNzITO>|85R;}$Z6Pp{SA92rI= zvvxGgNXBR}fB(IGMM=B{V{!z6R{|+^0bk>ltYeHHi|`ZE>5vLZ>vHm!!hF=F#iJxo z0S`3=R_LJTAg+Y6PoDi1aaPX67Osp_7_RB6^A#jZ>bB)#GwK^?2cp8L>3XHP!UlI| zY)FPzE6XOJSbgU_0rnDP8Sw9TYg1Y?Q5$Bb+pxBcip}z$e>44+VYR5nu7$TZnAP{R GaMfnHo`A#v literal 256 zcmV+b0ssE6R`Nfv8?!-f+CE*@!S0ACLy{E|JU%tqM7iG<#6M@Em3w}HKKYC`bS}vU z?tho}W5otLnsyNAzcq4|hTYh=T4~PtnG(4zn1eAn*g16JvDb7epvvsQrug@k2A43S zy^l@bGyvyzkqA6eWW<{1OppQsGomAa!R!SuP2jT@@T+4;Q9;JX*VG(&_n`oA2v^l( zDnkV5lw?Gtc&~2`EE~cMe(|$75%?VS?Tr<-1V5HGOpA5rpuzv_&_s Date: Wed, 27 Jan 2016 15:05:37 -0800 Subject: [PATCH 201/201] Bump version to 0.4.0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index d81c13423..b5bec3476 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.3.0' +version = '0.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index af83c9b60..a6553d890 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.3.0' +version = '0.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 00181130a..b7f448e83 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.3.0' +version = '0.4.0.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 6741d1d52..c1ff85185 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.3.0' +version = '0.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 32b637bb6..1dd7d7eba 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.3.0' +__version__ = '0.4.0.dev0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 20bfb7158..000f86c31 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.3.0' +version = '0.4.0.dev0' install_requires = [ 'setuptools', # pkg_resources