From e7226d28041a03a8f2d6d61ac84e765e8235a180 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 16:43:24 -0800 Subject: [PATCH 001/127] Automate testing with the apache-conf-library --- tests/apache-conf-files/hackish-apache-test | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index c6663551e..8efe65e42 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -18,11 +18,25 @@ function CleanupExit() { exit 1 } +FAILS=0 trap CleanupExit INT for f in *.conf ; do - echo testing "$f" + echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - sudo "$LEROOT"/venv/bin/letsencrypt --apache certonly -t + RESULT=`echo c | sudo "$LEROOT"/venv/bin/letsencrypt --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then + echo passed + else + echo failed + echo $RESULT + echo + echo + FAILS=`expr $FAILS + 1` + fi sudo rm /etc/apache2/sites-{enabled,available}/"$f" done +if [ "$FAILS" -ne 0 ] ; then + return 1 +fi +return 0 From db712534e545fa05211568b738474b74b0c92217 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 Dec 2015 16:53:12 -0800 Subject: [PATCH 002/127] Make dump() public --- acme/acme/jose/util.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index ab3606efc..1d98aad4e 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -43,8 +43,17 @@ class ComparableX509(object): # pylint: disable=too-few-public-methods def __getattr__(self, name): return getattr(self._wrapped, name) - def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): - # pylint: disable=missing-docstring,protected-access + def dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): + """Dumps the object into a buffer with the specified encoding. + + :param int filetype: The desired encoding. Should be one of + OpenSSL.crypto.FILETYPE_ASN1, OpenSSL.crypto.FILETYPE_PEM, + or OpenSSL.crypto.FILETYPE_TEXT. + + :returns: Encoded X509 object. + :rtype: str + + """ if isinstance(self._wrapped, OpenSSL.crypto.X509): func = OpenSSL.crypto.dump_certificate else: # assert in __init__ makes sure this is X509Req @@ -54,10 +63,10 @@ class ComparableX509(object): # pylint: disable=too-few-public-methods def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented - return self._dump() == other._dump() # pylint: disable=protected-access + return self.dump() == other.dump() def __hash__(self): - return hash((self.__class__, self._dump())) + return hash((self.__class__, self.dump())) def __ne__(self, other): return not self == other From 52705107ff76f1e713b9b903837e7c38c230f411 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 17:01:36 -0800 Subject: [PATCH 003/127] let the environment determine how letsencrypt is run --- tests/apache-conf-files/hackish-apache-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 8efe65e42..e6f25b15f 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -9,6 +9,7 @@ export EA=/etc/apache2/ TESTDIR="`dirname $0`" LEROOT="`realpath \"$TESTDIR/../../\"`" cd $TESTDIR/passing +LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" function CleanupExit() { echo control c, exiting tests... @@ -24,7 +25,7 @@ for f in *.conf ; do echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - RESULT=`echo c | sudo "$LEROOT"/venv/bin/letsencrypt --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else From 3c6af7094c67b37e8b815309fe0a492e232fccbf Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 17:21:47 -0800 Subject: [PATCH 004/127] Bugfix, and use --staging --- tests/apache-conf-files/hackish-apache-test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index e6f25b15f..c4df319ed 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -25,7 +25,7 @@ for f in *.conf ; do echo -n testing "$f"... sudo cp "$f" "$EA"/sites-available/ sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" - RESULT=`echo c | sudo "$LETSENCRYPT" --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else @@ -38,6 +38,6 @@ for f in *.conf ; do sudo rm /etc/apache2/sites-{enabled,available}/"$f" done if [ "$FAILS" -ne 0 ] ; then - return 1 + exit 1 fi -return 0 +exit 0 From d21ca90560ef590526971eb3c753eb96a34cc041 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 Dec 2015 17:33:08 -0800 Subject: [PATCH 005/127] Use dump on ComparableX509 --- acme/acme/challenges_test.py | 3 +-- acme/acme/jose/json_util.py | 6 ++---- acme/acme/jose/jws.py | 3 +-- acme/acme/jose/jws_test.py | 8 ++------ letsencrypt/cli.py | 6 +++--- letsencrypt/client.py | 6 ++---- letsencrypt/renewer.py | 3 +-- letsencrypt/tests/cli_test.py | 4 ++-- 8 files changed, 14 insertions(+), 25 deletions(-) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index a4e78ebe9..c01511171 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -420,8 +420,7 @@ class ProofOfPossessionHintsTest(unittest.TestCase): self.jmsg_to = { 'jwk': jwk, 'certFingerprints': cert_fingerprints, - 'certs': (jose.encode_b64jose(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT)),), + 'certs': (jose.encode_b64jose(CERT.dump()),), 'subjectKeyIdentifiers': subject_key_identifiers, 'serialNumbers': serial_numbers, 'issuers': issuers, diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 7b95e3fce..66776172b 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -372,8 +372,7 @@ def encode_cert(cert): :rtype: unicode """ - return encode_b64jose(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert)) + return encode_b64jose(cert.dump()) def decode_cert(b64der): @@ -397,8 +396,7 @@ def encode_csr(csr): :rtype: unicode """ - return encode_b64jose(OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr)) + return encode_b64jose(csr.dump()) def decode_csr(b64der): diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py index 1a073e17d..939932d36 100644 --- a/acme/acme/jose/jws.py +++ b/acme/acme/jose/jws.py @@ -123,8 +123,7 @@ class Header(json_util.JSONObjectWithFields): @x5c.encoder def x5c(value): # pylint: disable=missing-docstring,no-self-argument - return [base64.b64encode(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert)) for cert in value] + return [base64.b64encode(cert.dump()) for cert in value] @x5c.decoder def x5c(value): # pylint: disable=missing-docstring,no-self-argument diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py index 69341f228..065243774 100644 --- a/acme/acme/jose/jws_test.py +++ b/acme/acme/jose/jws_test.py @@ -3,7 +3,6 @@ import base64 import unittest import mock -import OpenSSL from acme import test_util @@ -68,13 +67,10 @@ class HeaderTest(unittest.TestCase): from acme.jose.jws import Header header = Header(x5c=(CERT, CERT)) jobj = header.to_partial_json() - cert_b64 = base64.b64encode(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT)) + cert_b64 = base64.b64encode(CERT.dump()) self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) self.assertEqual(header, Header.from_json(jobj)) - jobj['x5c'][0] = base64.b64encode( - b'xxx' + OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT)) + jobj['x5c'][0] = base64.b64encode(b'xxx' + CERT.dump()) self.assertRaises(errors.DeserializationError, Header.from_json, jobj) def test_find_key(self): diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 29519d430..1a141f556 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -391,9 +391,9 @@ def _auth_from_domains(le_client, config, domains): new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) # TODO: Check whether it worked! <- or make sure errors are thrown (jdk) lineage.save_successor( - lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body), - new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) + lineage.latest_common_version(), + new_certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), new_key.pem, + crypto_util.dump_pyopenssl_chain(new_chain)) lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 080ee7991..59ac11a72 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -299,8 +299,7 @@ class Client(object): "by your operating system package manager") lineage = storage.RenewableCert.new_lineage( - domains[0], OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body), + domains[0], certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), key.pem, crypto_util.dump_pyopenssl_chain(chain), params, config, cli_config) return lineage @@ -329,8 +328,7 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body) + cert_pem = certr.body.dump(OpenSSL.crypto.FILETYPE_PEM) cert_file, act_cert_path = le_util.unique_file(cert_path, 0o644) try: cert_file.write(cert_pem) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 3996cfe67..6e2366d82 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -102,8 +102,7 @@ def renew(cert, old_version): # new_key if the old key is to be used (since save_successor # already understands this distinction!) return cert.save_successor( - old_version, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body), + old_version, new_certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) # TODO: Notify results else: diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index ccf16f5b5..39c09dede 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -417,11 +417,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) - mock_cert = mock.MagicMock(body='body') + mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') mock_renewal.return_value = ("renew", mock_lineage) mock_client = mock.MagicMock() - mock_client.obtain_certificate.return_value = (mock_cert, 'chain', + mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') mock_init.return_value = mock_client with mock.patch('letsencrypt.cli.OpenSSL'): From bf764e4852dcf061e586789455afcb9785f39e3f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 18:01:49 -0800 Subject: [PATCH 006/127] Support appending to non-Debianish Apache setups --- tests/apache-conf-files/hackish-apache-test | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index c4df319ed..99fa123f7 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -23,8 +23,14 @@ FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... - sudo cp "$f" "$EA"/sites-available/ - sudo ln -s "$EA/sites-available/$f" "$EA/sites-enabled/$f" + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo cp "$f" "$EA"/sites-available/ + sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + else + TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" + sudo cp -a "$APPEND_APACHECONF" "$TMP" + sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" + fi RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed @@ -35,7 +41,11 @@ for f in *.conf ; do echo FAILS=`expr $FAILS + 1` fi - sudo rm /etc/apache2/sites-{enabled,available}/"$f" + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo rm /etc/apache2/sites-{enabled,available}/"$f" + else + sudo mv "$TMP" "$APPEND_APACHECONF" + fi done if [ "$FAILS" -ne 0 ] ; then exit 1 From 9013fecc9cea8bfca8e29789520772c3bd96cd51 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:41:35 -0800 Subject: [PATCH 007/127] Prep for testfarming. --- tests/apache-conf-files/hackish-apache-test | 16 ++++++++++------ tests/apache-conf-files/passing/README.modules | 3 +-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 99fa123f7..3d4336579 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -14,11 +14,19 @@ LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" function CleanupExit() { echo control c, exiting tests... if [ "$f" != "" ] ; then - sudo rm /etc/apache2/sites-{enabled,available}/"$f" + Cleanup fi exit 1 } +function Cleanup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo rm /etc/apache2/sites-{enabled,available}/"$f" + else + sudo mv "$TMP" "$APPEND_APACHECONF" + fi +} + FAILS=0 trap CleanupExit INT for f in *.conf ; do @@ -41,11 +49,7 @@ for f in *.conf ; do echo FAILS=`expr $FAILS + 1` fi - if [ "$APPEND_APACHECONF" = "" ] ; then - sudo rm /etc/apache2/sites-{enabled,available}/"$f" - else - sudo mv "$TMP" "$APPEND_APACHECONF" - fi + Cleanup done if [ "$FAILS" -ne 0 ] ; then exit 1 diff --git a/tests/apache-conf-files/passing/README.modules b/tests/apache-conf-files/passing/README.modules index 7edbd3e84..32c3ef019 100644 --- a/tests/apache-conf-files/passing/README.modules +++ b/tests/apache-conf-files/passing/README.modules @@ -1,5 +1,4 @@ -Modules required to parse these conf files: - +# Modules required to parse these conf files: ssl rewrite macro From 20b3188c6527588f1f6ab552104bf3af39978e2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 16 Dec 2015 19:46:56 -0800 Subject: [PATCH 008/127] No kwargs plz --- letsencrypt/plugins/standalone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 4319e51f9..cde7041d8 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -150,7 +150,7 @@ class Authenticator(common.Plugin): # one self-signed key for all tls-sni-01 certificates self.key = OpenSSL.crypto.PKey() - self.key.generate_key(OpenSSL.crypto.TYPE_RSA, bits=2048) + self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) self.served = collections.defaultdict(set) From babb33991bffb51fd82acc75afa87ce4774168c4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:51:45 -0800 Subject: [PATCH 009/127] Neaten things with a Setup() function --- tests/apache-conf-files/hackish-apache-test | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index 3d4336579..b8caaadc0 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -19,6 +19,17 @@ function CleanupExit() { exit 1 } +function Setup() { + if [ "$APPEND_APACHECONF" = "" ] ; then + sudo cp "$f" "$EA"/sites-available/ + sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + else + TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" + sudo cp -a "$APPEND_APACHECONF" "$TMP" + sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" + fi +} + function Cleanup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo rm /etc/apache2/sites-{enabled,available}/"$f" @@ -31,14 +42,7 @@ FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... - if [ "$APPEND_APACHECONF" = "" ] ; then - sudo cp "$f" "$EA"/sites-available/ - sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" - else - TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" - sudo cp -a "$APPEND_APACHECONF" "$TMP" - sudo bash -c "cat \"$f\" >> \"$APPEND_APACHECONF\"" - fi + Setup RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed From 7efdac6c66720715e6aeb595aa5d6149967c89f2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 Dec 2015 17:28:36 -0800 Subject: [PATCH 010/127] Fixed SANs problem --- acme/acme/challenges_test.py | 4 +++- acme/acme/crypto_util.py | 30 ++++++++++-------------------- acme/acme/crypto_util_test.py | 16 ++++++++++------ acme/acme/jose/util_test.py | 3 +++ 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index c01511171..5c2b842ec 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -264,7 +264,9 @@ class TLSSNI01ResponseTest(unittest.TestCase): def test_verify_bad_cert(self): self.assertFalse(self.response.verify_cert( - test_util.load_cert('cert.pem'))) + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, + test_util.load_vector('cert.pem')))) def test_simple_verify_bad_key_authorization(self): key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 72a93141a..cde8b2a9a 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -4,11 +4,10 @@ import logging import socket import sys -from six.moves import range # pylint: disable=import-error,redefined-builtin - import OpenSSL from acme import errors +from acme import jose logger = logging.getLogger(__name__) @@ -161,31 +160,22 @@ def _pyopenssl_cert_or_req_san(cert_or_req): :rtype: `list` of `unicode` """ - # constants based on implementation of - # OpenSSL.crypto.X509Error._subjectAltNameString + # constants based on PyOpenSSL certificate/CSR text dump + label = "DNS" parts_separator = ", " part_separator = ":" - extension_short_name = b"subjectAltName" - - if hasattr(cert_or_req, 'get_extensions'): # X509Req - extensions = cert_or_req.get_extensions() - else: # X509 - extensions = [cert_or_req.get_extension(i) - for i in range(cert_or_req.get_extension_count())] - - # pylint: disable=protected-access,no-member - label = OpenSSL.crypto.X509Extension._prefixes[OpenSSL.crypto._lib.GEN_DNS] - assert parts_separator not in label prefix = label + part_separator + title = "X509v3 Subject Alternative Name:" - san_extensions = [ - ext._subjectAltNameString().split(parts_separator) - for ext in extensions if ext.get_short_name() == extension_short_name] + text = jose.ComparableX509(cert_or_req).dump(OpenSSL.crypto.FILETYPE_TEXT) + lines = iter(text.decode("utf-8").splitlines()) + sans = [next(lines).split(parts_separator) + for line in lines if title in line] # WARNING: this function assumes that no SAN can include # parts_separator, hence the split! - return [part.split(part_separator)[1] for parts in san_extensions - for part in parts if part.startswith(prefix)] + return [part.split(part_separator)[1] for parts in sans + for part in parts if part.lstrip().startswith(prefix)] def gen_ss_cert(key, domains, not_before=None, diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index bfd16388c..9e3062774 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -6,6 +6,8 @@ import unittest from six.moves import socketserver # pylint: disable=import-error +import OpenSSL + from acme import errors from acme import jose from acme import test_util @@ -64,16 +66,18 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" @classmethod - def _call(cls, loader, name): + def _call(cls, cert_or_req): # pylint: disable=protected-access from acme.crypto_util import _pyopenssl_cert_or_req_san - return _pyopenssl_cert_or_req_san(loader(name)) + return _pyopenssl_cert_or_req_san(cert_or_req) - def _call_cert(self, name): - return self._call(test_util.load_cert, name) + def _call_cert(self, name, filetype=OpenSSL.crypto.FILETYPE_PEM): + return self._call(OpenSSL.crypto.load_certificate( + filetype, test_util.load_vector(name))) - def _call_csr(self, name): - return self._call(test_util.load_csr, name) + def _call_csr(self, name, filetype=OpenSSL.crypto.FILETYPE_PEM): + return self._call(OpenSSL.crypto.load_certificate_request( + filetype, test_util.load_vector(name))) def test_cert_no_sans(self): self.assertEqual(self._call_cert('cert.pem'), []) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py index 4cdd9127f..5920ce11f 100644 --- a/acme/acme/jose/util_test.py +++ b/acme/acme/jose/util_test.py @@ -20,6 +20,9 @@ class ComparableX509Test(unittest.TestCase): self.cert2 = test_util.load_cert('cert.pem') self.cert_other = test_util.load_cert('cert-san.pem') + def test_getattr_proxy(self): + self.assertTrue(self.cert1.has_expired()) + def test_eq(self): self.assertEqual(self.req1, self.req2) self.assertEqual(self.cert1, self.cert2) From a28f8fe4427c12c2523b16903325d0362b53123e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 Dec 2015 17:47:15 -0800 Subject: [PATCH 011/127] Drop version dependency --- acme/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e35b40d6e..5fc1bd8d0 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -12,8 +12,8 @@ install_requires = [ 'cryptography>=0.8', 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) 'pyasn1', # urllib3 InsecurePlatformWarning (#304) - # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) - 'PyOpenSSL>=0.15', + # Connection.set_tlsext_host_name (>=0.13) + 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', 'requests', From 03fdd03a878ce1451c6e766f32421e604ceb4f72 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 19:52:35 -0800 Subject: [PATCH 012/127] Experimentally try travis with the hackish-apache-test --- .travis.yml | 1 + tox.ini | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8dde06ceb..2b37c1bef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover + - TOXENV=hackishapachetest # Only build pushes to the master branch, PRs, and branches beginning with diff --git a/tox.ini b/tox.ini index d1fafe20f..cffbc4e18 100644 --- a/tox.ini +++ b/tox.ini @@ -67,3 +67,10 @@ commands = pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt + +[testenv:hackishapachetest] +basepython = python2.7 +setenv = + LETSENCRYPT=letsencrypt +commands = + ./tests/apache-conf-files/hackish-apache-test From 9129dcbc8797f67d85d04fa29541b5cec446e2ce Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 16 Dec 2015 20:04:27 -0800 Subject: [PATCH 013/127] Make sure there's always a domain name to prompt a question --- tests/apache-conf-files/hackish-apache-test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/apache-conf-files/hackish-apache-test b/tests/apache-conf-files/hackish-apache-test index b8caaadc0..27ee0fdf0 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/tests/apache-conf-files/hackish-apache-test @@ -23,6 +23,13 @@ function Setup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo cp "$f" "$EA"/sites-available/ sudo ln -sf "$EA/sites-available/$f" "$EA/sites-enabled/$f" + sudo echo """ + + ServerName example.com + DocumentRoot /tmp/ + ErrorLog /tmp/error.log + CustomLog /tmp/requests.log combined +""" >> $EA/sites-available/throwaway-example.conf else TMP="/tmp/`basename \"$APPEND_APACHECONF\"`.$$" sudo cp -a "$APPEND_APACHECONF" "$TMP" @@ -33,6 +40,7 @@ function Setup() { function Cleanup() { if [ "$APPEND_APACHECONF" = "" ] ; then sudo rm /etc/apache2/sites-{enabled,available}/"$f" + sudo rm $EA/sites-available/throwaway-example.conf else sudo mv "$TMP" "$APPEND_APACHECONF" fi From d777e7fabad3f379c266382a36f970a13e85cf12 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 17 Dec 2015 15:45:47 -0800 Subject: [PATCH 014/127] This sort of works in tox; travis is unlikely due to sudo --- .travis.yml | 1 + .../letsencrypt_apache/tests}/apache-conf-files/NEEDED.txt | 0 .../tests}/apache-conf-files/failing/ipv6-1143.conf | 0 .../tests}/apache-conf-files/failing/ipv6-1143b.conf | 0 .../apache-conf-files/failing/missing-double-quote-1724.conf | 0 .../tests}/apache-conf-files/failing/multivhost-1093.conf | 0 .../tests}/apache-conf-files/failing/multivhost-1093b.conf | 0 .../tests}/apache-conf-files/hackish-apache-test | 2 +- .../tests}/apache-conf-files/passing/1626-1531.conf | 0 .../tests}/apache-conf-files/passing/README.modules | 0 .../tests}/apache-conf-files/passing/anarcat-1531.conf | 0 .../passing/drupal-errordocument-arg-1724.conf | 0 .../tests}/apache-conf-files/passing/drupal-htaccess-1531.conf | 0 .../tests}/apache-conf-files/passing/example-1755.conf | 0 .../tests}/apache-conf-files/passing/example-ssl.conf | 0 .../tests}/apache-conf-files/passing/example.conf | 0 .../apache-conf-files/passing/finalize-1243.apache2.conf.txt | 0 .../tests}/apache-conf-files/passing/finalize-1243.conf | 0 .../tests}/apache-conf-files/passing/missing-quote-1724.conf | 0 .../tests}/apache-conf-files/passing/modmacro-1385.conf | 0 .../tests}/apache-conf-files/passing/owncloud-1264.conf | 0 .../tests}/apache-conf-files/passing/roundcube-1222.conf | 0 .../tests}/apache-conf-files/passing/semacode-1598.conf | 0 .../apache-conf-files/passing/sslrequire-wordlist-1827.htaccess | 0 .../apache-conf-files/passing/two-blocks-one-line-1693.conf | 0 tox.ini | 2 +- 26 files changed, 3 insertions(+), 2 deletions(-) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/NEEDED.txt (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/ipv6-1143.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/ipv6-1143b.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/missing-double-quote-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/multivhost-1093.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/failing/multivhost-1093b.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/hackish-apache-test (97%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/1626-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/README.modules (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/anarcat-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/drupal-errordocument-arg-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/drupal-htaccess-1531.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example-1755.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example-ssl.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/example.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/finalize-1243.apache2.conf.txt (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/finalize-1243.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/missing-quote-1724.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/modmacro-1385.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/owncloud-1264.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/roundcube-1222.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/semacode-1598.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/two-blocks-one-line-1693.conf (100%) diff --git a/.travis.yml b/.travis.yml index 2b37c1bef..dfbc09e2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: python services: - rabbitmq - mariadb + - apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml diff --git a/tests/apache-conf-files/NEEDED.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt similarity index 100% rename from tests/apache-conf-files/NEEDED.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/NEEDED.txt diff --git a/tests/apache-conf-files/failing/ipv6-1143.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143.conf diff --git a/tests/apache-conf-files/failing/ipv6-1143b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf similarity index 100% rename from tests/apache-conf-files/failing/ipv6-1143b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/ipv6-1143b.conf diff --git a/tests/apache-conf-files/failing/missing-double-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/failing/missing-double-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/missing-double-quote-1724.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093.conf diff --git a/tests/apache-conf-files/failing/multivhost-1093b.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf similarity index 100% rename from tests/apache-conf-files/failing/multivhost-1093b.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/failing/multivhost-1093b.conf diff --git a/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test similarity index 97% rename from tests/apache-conf-files/hackish-apache-test rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 27ee0fdf0..9e828bf2d 100755 --- a/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -7,7 +7,7 @@ # assess, but it should be automated export EA=/etc/apache2/ TESTDIR="`dirname $0`" -LEROOT="`realpath \"$TESTDIR/../../\"`" +LEROOT="`realpath \"$TESTDIR/../../../../\"`" cd $TESTDIR/passing LETSENCRYPT="${LETSENCRYPT:-$LEROOT/venv/bin/letsencrypt}" diff --git a/tests/apache-conf-files/passing/1626-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/1626-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/1626-1531.conf diff --git a/tests/apache-conf-files/passing/README.modules b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules similarity index 100% rename from tests/apache-conf-files/passing/README.modules rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/README.modules diff --git a/tests/apache-conf-files/passing/anarcat-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/anarcat-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/anarcat-1531.conf diff --git a/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-errordocument-arg-1724.conf diff --git a/tests/apache-conf-files/passing/drupal-htaccess-1531.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf similarity index 100% rename from tests/apache-conf-files/passing/drupal-htaccess-1531.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/drupal-htaccess-1531.conf diff --git a/tests/apache-conf-files/passing/example-1755.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf similarity index 100% rename from tests/apache-conf-files/passing/example-1755.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-1755.conf diff --git a/tests/apache-conf-files/passing/example-ssl.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf similarity index 100% rename from tests/apache-conf-files/passing/example-ssl.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example-ssl.conf diff --git a/tests/apache-conf-files/passing/example.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf similarity index 100% rename from tests/apache-conf-files/passing/example.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/example.conf diff --git a/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.apache2.conf.txt diff --git a/tests/apache-conf-files/passing/finalize-1243.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf similarity index 100% rename from tests/apache-conf-files/passing/finalize-1243.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/finalize-1243.conf diff --git a/tests/apache-conf-files/passing/missing-quote-1724.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf similarity index 100% rename from tests/apache-conf-files/passing/missing-quote-1724.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/missing-quote-1724.conf diff --git a/tests/apache-conf-files/passing/modmacro-1385.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf similarity index 100% rename from tests/apache-conf-files/passing/modmacro-1385.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/modmacro-1385.conf diff --git a/tests/apache-conf-files/passing/owncloud-1264.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf similarity index 100% rename from tests/apache-conf-files/passing/owncloud-1264.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/owncloud-1264.conf diff --git a/tests/apache-conf-files/passing/roundcube-1222.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf similarity index 100% rename from tests/apache-conf-files/passing/roundcube-1222.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/roundcube-1222.conf diff --git a/tests/apache-conf-files/passing/semacode-1598.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf similarity index 100% rename from tests/apache-conf-files/passing/semacode-1598.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/semacode-1598.conf diff --git a/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess similarity index 100% rename from tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/sslrequire-wordlist-1827.htaccess diff --git a/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf similarity index 100% rename from tests/apache-conf-files/passing/two-blocks-one-line-1693.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/two-blocks-one-line-1693.conf diff --git a/tox.ini b/tox.ini index cffbc4e18..1d2f12e6c 100644 --- a/tox.ini +++ b/tox.ini @@ -73,4 +73,4 @@ basepython = python2.7 setenv = LETSENCRYPT=letsencrypt commands = - ./tests/apache-conf-files/hackish-apache-test + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test From a819ee146d6369ab08fe022ae3f7481e50d32ed9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 17 Dec 2015 18:03:34 -0800 Subject: [PATCH 015/127] Experimentally try sudo in travis This may decontainerise us, so we might not want to merge even if it works... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dfbc09e2e..a0ca6576f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ branches: - /^test-.*$/ # container-based infrastructure -sudo: false +sudo: true addons: # make sure simplehttp simple verification works (custom /etc/hosts) From 7a16e2e2489bceb6cb88e21e0fa0d75900572250 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 00:17:22 -0800 Subject: [PATCH 016/127] Wrangle things to actually run in travis --- .travis.yml | 11 +++++++---- .../tests/apache-conf-files/hackish-apache-test | 1 + tox.ini | 5 +++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0ca6576f..b1fe7f55d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ env: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH matrix: - - TOXENV=py26 BOULDER_INTEGRATION=1 - - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=lint - - TOXENV=cover - TOXENV=hackishapachetest +# - TOXENV=py26 BOULDER_INTEGRATION=1 +# - TOXENV=py27 BOULDER_INTEGRATION=1 +# - TOXENV=lint +# - TOXENV=cover # Only build pushes to the master branch, PRs, and branches beginning with @@ -60,6 +60,9 @@ addons: - openssl # For Boulder integration testing - rsyslog + # for hackishapachetest + - realpath + - apache2 install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 9e828bf2d..664423d7a 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -51,6 +51,7 @@ trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup + echo running from $PWD RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed diff --git a/tox.ini b/tox.ini index 1d2f12e6c..1a637777d 100644 --- a/tox.ini +++ b/tox.ini @@ -69,8 +69,9 @@ commands = pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt [testenv:hackishapachetest] -basepython = python2.7 +#basepython = python2.7 setenv = - LETSENCRYPT=letsencrypt + LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt commands = + pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test From 8d71b2d6c38aa062ef20a4cb10d7e9fb4cc345c3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 08:48:49 -0800 Subject: [PATCH 017/127] Install Apache modules in travis --- .travis.yml | 2 ++ .../tests/apache-conf-files/hackish-apache-test | 13 +++++++++++-- tox.ini | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1fe7f55d..6b0af69c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,8 @@ addons: # for hackishapachetest - realpath - apache2 + - libapache2-mod-wsgi + - sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index 664423d7a..cf06b48af 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -46,13 +46,22 @@ function Cleanup() { fi } +# if our environment asks us to enable modules, do our best! +if [ "$1" = --debian-modules ] ; then + sudo apt-get install -y libapache2-mod-wsgi + + for mod in ssl rewrite macro wsgi deflate ; do + sudo a2enmod $mod + done +fi + + FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup - echo running from $PWD - RESULT=`echo c | sudo "$LETSENCRYPT" --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | sudo "$LETSENCRYPT" -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then echo passed else diff --git a/tox.ini b/tox.ini index 1a637777d..abb934055 100644 --- a/tox.ini +++ b/tox.ini @@ -74,4 +74,4 @@ setenv = LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt - sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test --debian-modules From a0e902d405e8cd4bd3b584847ec8636affc818e0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 11:43:20 -0800 Subject: [PATCH 018/127] More module deps! --- .../tests/apache-conf-files/hackish-apache-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index cf06b48af..bfe71cc51 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -48,9 +48,9 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then - sudo apt-get install -y libapache2-mod-wsgi + sudo apt-get install -y libapache2-mod-{wsgi,macro} - for mod in ssl rewrite macro wsgi deflate ; do + for mod in ssl rewrite macro wsgi deflate userdir version ; do sudo a2enmod $mod done fi From 05e210d42ad5fe38a4b1b66a659e6dfe3d77e736 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 18 Dec 2015 11:44:52 -0800 Subject: [PATCH 019/127] Also add dependencies to travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6b0af69c3..ab1931be1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ addons: - realpath - apache2 - libapache2-mod-wsgi + - libapache2-mod-macro - sudo install: "travis_retry pip install tox coveralls" From 47f7e70b764c4b5c760256659260059afee14df2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 21 Dec 2015 13:49:46 -0800 Subject: [PATCH 020/127] This is a more correct test --- .../tests/apache-conf-files/hackish-apache-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test index bfe71cc51..c661e6435 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test @@ -62,7 +62,7 @@ for f in *.conf ; do echo -n testing "$f"... Setup RESULT=`echo c | sudo "$LETSENCRYPT" -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` - if echo $RESULT | grep -Eq \("Please specify --domains"\|"mod_macro is not yet"\) ; then + if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then echo passed else echo failed From e41ddd2cc7210563506bb5107841cfb76c27c6e2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 15:50:48 -0800 Subject: [PATCH 021/127] Rename hackishapachetest -> apacheconftest Reenable other travis tests as well as this one --- .travis.yml | 10 +++++----- .../{hackish-apache-test => apache-conf-test} | 0 tox.ini | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) rename letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/{hackish-apache-test => apache-conf-test} (100%) diff --git a/.travis.yml b/.travis.yml index ab1931be1..9efb95165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ env: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH matrix: - - TOXENV=hackishapachetest -# - TOXENV=py26 BOULDER_INTEGRATION=1 -# - TOXENV=py27 BOULDER_INTEGRATION=1 -# - TOXENV=lint -# - TOXENV=cover + - TOXENV=py26 BOULDER_INTEGRATION=1 + - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=lint + - TOXENV=cover + - TOXENV=apacheconftest # Only build pushes to the master branch, PRs, and branches beginning with diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test similarity index 100% rename from letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test diff --git a/tox.ini b/tox.ini index abb934055..aac7c15eb 100644 --- a/tox.ini +++ b/tox.ini @@ -68,10 +68,10 @@ commands = pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt -[testenv:hackishapachetest] +[testenv:apacheconftest] #basepython = python2.7 setenv = - LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/hackishapachetest/bin/letsencrypt + LETSENCRYPT=/home/travis/build/letsencrypt/letsencrypt/.tox/apacheconftest/bin/letsencrypt commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt - sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/hackish-apache-test --debian-modules + sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test --debian-modules From ef1973ae2888232a5b19d2c5dae4783506ccf83c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 15:58:09 -0800 Subject: [PATCH 022/127] Move recently included tests to the new apache-conf-test location --- .../tests}/apache-conf-files/passing/graphite-quote-1934.conf | 0 .../tests}/apache-conf-files/passing/rewrite-quote-1960.conf | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/graphite-quote-1934.conf (100%) rename {tests => letsencrypt-apache/letsencrypt_apache/tests}/apache-conf-files/passing/rewrite-quote-1960.conf (100%) diff --git a/tests/apache-conf-files/passing/graphite-quote-1934.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf similarity index 100% rename from tests/apache-conf-files/passing/graphite-quote-1934.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf diff --git a/tests/apache-conf-files/passing/rewrite-quote-1960.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf similarity index 100% rename from tests/apache-conf-files/passing/rewrite-quote-1960.conf rename to letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/rewrite-quote-1960.conf From d9ea151fbb2c54e5dc390b23590857de7f02cf73 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 16:04:30 -0800 Subject: [PATCH 023/127] Tweak the graphite config file so that apacheconftest passes again (At least on debian and other systems that have a www-data user...) --- .../tests/apache-conf-files/passing/graphite-quote-1934.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf index 2a8734b43..f257dd9a8 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/passing/graphite-quote-1934.conf @@ -1,6 +1,6 @@ - WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=_graphite group=_graphite + WSGIDaemonProcess _graphite processes=5 threads=5 display-name='%{GROUP}' inactivity-timeout=120 user=www-data group=www-data WSGIProcessGroup _graphite WSGIImportScript /usr/share/graphite-web/graphite.wsgi process-group=_graphite application-group=%{GLOBAL} WSGIScriptAlias / /usr/share/graphite-web/graphite.wsgi From 068504ddbf2d4943d4254e8bf0a8904cd347f7cd Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 22 Dec 2015 17:19:01 -0800 Subject: [PATCH 024/127] Correct comment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9efb95165..4e0849e3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ addons: - openssl # For Boulder integration testing - rsyslog - # for hackishapachetest + # for apacheconftest - realpath - apache2 - libapache2-mod-wsgi From 6db40626196c790af20e5834f5b85432dd358cb8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:45:08 -0800 Subject: [PATCH 025/127] Split module installation into substeps - This may prevent failures if one thing is uninstallable --- .../tests/apache-conf-files/apache-conf-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index c661e6435..38d5f05c7 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -48,7 +48,8 @@ function Cleanup() { # if our environment asks us to enable modules, do our best! if [ "$1" = --debian-modules ] ; then - sudo apt-get install -y libapache2-mod-{wsgi,macro} + sudo apt-get install -y libapache2-mod-wsgi + sudo apt-get install -y libapache2-mod-macro for mod in ssl rewrite macro wsgi deflate userdir version ; do sudo a2enmod $mod From 66a861ead1cc9188bc93133558567eaf41aa72b9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Dec 2015 13:48:52 -0500 Subject: [PATCH 026/127] Add test_comparable_{cert,csr} --- acme/acme/challenges_test.py | 2 +- acme/acme/crypto_util_test.py | 2 +- acme/acme/jose/json_util_test.py | 4 ++-- acme/acme/jose/jws_test.py | 2 +- acme/acme/jose/util_test.py | 14 +++++++------- acme/acme/messages_test.py | 4 ++-- acme/acme/standalone_test.py | 4 ++-- acme/acme/test_util.py | 16 ++++++++++++---- letsencrypt/tests/client_test.py | 2 +- letsencrypt/tests/renewer_test.py | 7 +++++-- letsencrypt/tests/test_util.py | 16 ++++++++++++---- 11 files changed, 46 insertions(+), 27 deletions(-) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 5c2b842ec..0b02102b2 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -13,7 +13,7 @@ from acme import other from acme import test_util -CERT = test_util.load_cert('cert.pem') +CERT = test_util.load_comparable_cert('cert.pem') KEY = jose.JWKRSA(key=test_util.load_rsa_private_key('rsa512_key.pem')) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 9e3062774..72530ec9d 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -17,7 +17,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" def setUp(self): - self.cert = test_util.load_cert('cert.pem') + self.cert = test_util.load_comparable_cert('cert.pem') key = test_util.load_pyopenssl_private_key('rsa512_key.pem') # pylint: disable=protected-access certs = {b'foo': (key, self.cert._wrapped)} diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py index a055f3bf7..25e36211e 100644 --- a/acme/acme/jose/json_util_test.py +++ b/acme/acme/jose/json_util_test.py @@ -12,8 +12,8 @@ from acme.jose import interfaces from acme.jose import util -CERT = test_util.load_cert('cert.pem') -CSR = test_util.load_csr('csr.pem') +CERT = test_util.load_comparable_cert('cert.pem') +CSR = test_util.load_comparable_csr('csr.pem') class FieldTest(unittest.TestCase): diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py index 065243774..3fd0fbb89 100644 --- a/acme/acme/jose/jws_test.py +++ b/acme/acme/jose/jws_test.py @@ -12,7 +12,7 @@ from acme.jose import jwa from acme.jose import jwk -CERT = test_util.load_cert('cert.pem') +CERT = test_util.load_comparable_cert('cert.pem') KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py index 5920ce11f..392f81777 100644 --- a/acme/acme/jose/util_test.py +++ b/acme/acme/jose/util_test.py @@ -11,14 +11,14 @@ class ComparableX509Test(unittest.TestCase): """Tests for acme.jose.util.ComparableX509.""" def setUp(self): - # test_util.load_{csr,cert} return ComparableX509 - self.req1 = test_util.load_csr('csr.pem') - self.req2 = test_util.load_csr('csr.pem') - self.req_other = test_util.load_csr('csr-san.pem') + # test_util.load_comparable_{csr,cert} return ComparableX509 + self.req1 = test_util.load_comparable_csr('csr.pem') + self.req2 = test_util.load_comparable_csr('csr.pem') + self.req_other = test_util.load_comparable_csr('csr-san.pem') - self.cert1 = test_util.load_cert('cert.pem') - self.cert2 = test_util.load_cert('cert.pem') - self.cert_other = test_util.load_cert('cert-san.pem') + self.cert1 = test_util.load_comparable_cert('cert.pem') + self.cert2 = test_util.load_comparable_cert('cert.pem') + self.cert_other = test_util.load_comparable_cert('cert-san.pem') def test_getattr_proxy(self): self.assertTrue(self.cert1.has_expired()) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index 5a7a71299..8e74826bf 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -8,8 +8,8 @@ from acme import jose from acme import test_util -CERT = test_util.load_cert('cert.der') -CSR = test_util.load_csr('csr.der') +CERT = test_util.load_comparable_cert('cert.der') +CSR = test_util.load_comparable_csr('csr.der') KEY = test_util.load_rsa_private_key('rsa512_key.pem') diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 02b1f69d3..2778635f5 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -35,7 +35,7 @@ class TLSSNI01ServerTest(unittest.TestCase): self.certs = { b'localhost': (test_util.load_pyopenssl_private_key('rsa512_key.pem'), # pylint: disable=protected-access - test_util.load_cert('cert.pem')._wrapped), + test_util.load_cert('cert.pem')), } from acme.standalone import TLSSNI01Server self.server = TLSSNI01Server(("", 0), certs=self.certs) @@ -146,7 +146,7 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): time.sleep(1) # wait until thread starts else: self.assertEqual(jose.ComparableX509(cert), - test_util.load_cert('cert.pem')) + test_util.load_comparable_cert('cert.pem')) break diff --git a/acme/acme/test_util.py b/acme/acme/test_util.py index 2b4c6e00c..24eceff5a 100644 --- a/acme/acme/test_util.py +++ b/acme/acme/test_util.py @@ -40,16 +40,24 @@ def load_cert(*names): """Load certificate.""" loader = _guess_loader( names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return jose.ComparableX509(OpenSSL.crypto.load_certificate( - loader, load_vector(*names))) + return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) + + +def load_comparable_cert(*names): + """Load ComparableX509 cert.""" + return jose.ComparableX509(load_cert(*names)) def load_csr(*names): """Load certificate request.""" loader = _guess_loader( names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - loader, load_vector(*names))) + return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names)) + + +def load_comparable_csr(*names): + """Load ComparableX509 certificate request.""" + return jose.ComparableX509(load_csr(*names)) def load_rsa_private_key(*names): diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 6b76f70c9..cf1b3b39f 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -141,7 +141,7 @@ class ClientTest(unittest.TestCase): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_cert(certs[0])) + certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) chain_cert = [test_util.load_cert(certs[1]), test_util.load_cert(certs[2])] candidate_cert_path = os.path.join(tmp_path, "certs", "cert.pem") diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index daec9678f..269a9193d 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -9,6 +9,8 @@ import unittest import configobj import mock +from acme import jose + from letsencrypt import configuration from letsencrypt import errors from letsencrypt.storage import ALL_FOUR @@ -702,9 +704,10 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.configfile["renewalparams"]["authenticator"] = "apache" mock_client = mock.MagicMock() # pylint: disable=star-args + comparable_cert = jose.ComparableX509(CERT) mock_client.obtain_certificate.return_value = ( - mock.MagicMock(body=CERT), [CERT], mock.Mock(pem="key"), - mock.sentinel.csr) + mock.MagicMock(body=comparable_cert), [comparable_cert], + mock.Mock(pem="key"), mock.sentinel.csr) mock_c.return_value = mock_client self.assertEqual(2, renewer.renew(self.test_rc, 1)) # TODO: We could also make several assertions about calls that should diff --git a/letsencrypt/tests/test_util.py b/letsencrypt/tests/test_util.py index 2b4c6e00c..24eceff5a 100644 --- a/letsencrypt/tests/test_util.py +++ b/letsencrypt/tests/test_util.py @@ -40,16 +40,24 @@ def load_cert(*names): """Load certificate.""" loader = _guess_loader( names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return jose.ComparableX509(OpenSSL.crypto.load_certificate( - loader, load_vector(*names))) + return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) + + +def load_comparable_cert(*names): + """Load ComparableX509 cert.""" + return jose.ComparableX509(load_cert(*names)) def load_csr(*names): """Load certificate request.""" loader = _guess_loader( names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1) - return jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - loader, load_vector(*names))) + return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names)) + + +def load_comparable_csr(*names): + """Load ComparableX509 certificate request.""" + return jose.ComparableX509(load_csr(*names)) def load_rsa_private_key(*names): From 494e6e77110c6af028df31ad270bfa70d1658ad2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:53:13 -0800 Subject: [PATCH 027/127] Remove TODO that has been done --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 2 -- 1 file changed, 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index 38d5f05c7..ec0041534 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -3,8 +3,6 @@ # A hackish script to see if the client is behaving as expected # with each of the "passing" conf files. -# TODO presently this requires interaction and human judgement to -# assess, but it should be automated export EA=/etc/apache2/ TESTDIR="`dirname $0`" LEROOT="`realpath \"$TESTDIR/../../../../\"`" From 3dc3df4b345ee45d3b5d28a71fb1379d2e064446 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 10:58:28 -0800 Subject: [PATCH 028/127] Document the inclusion of apacheconftest in tox --- docs/contributing.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..6c70830b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -65,8 +65,14 @@ Testing The following tools are there to help you: -- ``tox`` starts a full set of tests. Please make sure you run it - before submitting a new pull request. +- ``tox`` starts a full set of tests. Please note that it includes + apacheconftest, which uses the system's Apache install to test config file + parsing, so it should only be run on systems that have an + experimental, non-production Apache2 install on them. ``tox -e + apacheconftest`` can be used to run those specific Apache conf tests. + +- ``tox -e py27``, ``tox -e py26`` etc, run unit tests for specific Python + versions. - ``tox -e cover`` checks the test coverage only. Calling the ``./tox.cover.sh`` script directly (or even ``./tox.cover.sh $pkg1 From 980637a936d24199061f4e4b1182971812062ddf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Dec 2015 17:12:33 -0500 Subject: [PATCH 029/127] Audit calls to test_util.load_cert --- acme/acme/challenges.py | 9 ++++++++- acme/acme/crypto_util_test.py | 16 ++++++---------- letsencrypt/tests/client_test.py | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 1e456d325..a121b4639 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -391,7 +391,14 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): return crypto_util.probe_sni(**kwargs) def verify_cert(self, cert): - """Verify tls-sni-01 challenge certificate.""" + """Verify tls-sni-01 challenge certificate. + + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ # pylint: disable=protected-access sans = crypto_util._pyopenssl_cert_or_req_san(cert) logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 72530ec9d..25b0d2f67 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -6,8 +6,6 @@ import unittest from six.moves import socketserver # pylint: disable=import-error -import OpenSSL - from acme import errors from acme import jose from acme import test_util @@ -66,18 +64,16 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" @classmethod - def _call(cls, cert_or_req): + def _call(cls, loader, name): # pylint: disable=protected-access from acme.crypto_util import _pyopenssl_cert_or_req_san - return _pyopenssl_cert_or_req_san(cert_or_req) + return _pyopenssl_cert_or_req_san(loader(name)) - def _call_cert(self, name, filetype=OpenSSL.crypto.FILETYPE_PEM): - return self._call(OpenSSL.crypto.load_certificate( - filetype, test_util.load_vector(name))) + def _call_cert(self, name): + return self._call(test_util.load_cert, name) - def _call_csr(self, name, filetype=OpenSSL.crypto.FILETYPE_PEM): - return self._call(OpenSSL.crypto.load_certificate_request( - filetype, test_util.load_vector(name))) + def _call_csr(self, name): + return self._call(test_util.load_csr, name) def test_cert_no_sans(self): self.assertEqual(self._call_cert('cert.pem'), []) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index cf1b3b39f..2f117f80c 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -142,8 +142,8 @@ class ClientTest(unittest.TestCase): os.chmod(tmp_path, 0o755) # TODO: really?? certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_cert(certs[1]), - test_util.load_cert(certs[2])] + chain_cert = [test_util.load_comparable_cert(certs[1]), + test_util.load_comparable_cert(certs[2])] candidate_cert_path = os.path.join(tmp_path, "certs", "cert.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") From ce9e3c1f94213d1c2158664436505e22e732e216 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 23 Dec 2015 15:00:07 -0800 Subject: [PATCH 030/127] Some OSes don't enable the mime module by default? --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index ec0041534..4e0443bb7 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -49,7 +49,7 @@ if [ "$1" = --debian-modules ] ; then sudo apt-get install -y libapache2-mod-wsgi sudo apt-get install -y libapache2-mod-macro - for mod in ssl rewrite macro wsgi deflate userdir version ; do + for mod in ssl rewrite macro wsgi deflate userdir version mime ; do sudo a2enmod $mod done fi From 75b551762b461de933d83de231735380751c5044 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 23 Dec 2015 19:09:05 -0500 Subject: [PATCH 031/127] Expose wrapped, not dump --- acme/acme/challenges_test.py | 3 ++- acme/acme/crypto_util.py | 8 ++++++-- acme/acme/crypto_util_test.py | 2 +- acme/acme/jose/json_util.py | 6 ++++-- acme/acme/jose/jws.py | 3 ++- acme/acme/jose/jws_test.py | 7 +++++-- acme/acme/jose/util.py | 28 ++++++++++++++-------------- letsencrypt/cli.py | 6 +++--- letsencrypt/client.py | 6 ++++-- letsencrypt/crypto_util.py | 2 +- letsencrypt/renewer.py | 3 ++- 11 files changed, 44 insertions(+), 30 deletions(-) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 0b02102b2..6b277ac27 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -422,7 +422,8 @@ class ProofOfPossessionHintsTest(unittest.TestCase): self.jmsg_to = { 'jwk': jwk, 'certFingerprints': cert_fingerprints, - 'certs': (jose.encode_b64jose(CERT.dump()),), + 'certs': (jose.encode_b64jose(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped)),), 'subjectKeyIdentifiers': subject_key_identifiers, 'serialNumbers': serial_numbers, 'issuers': issuers, diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index cde8b2a9a..15890175f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -7,7 +7,6 @@ import sys import OpenSSL from acme import errors -from acme import jose logger = logging.getLogger(__name__) @@ -167,7 +166,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): prefix = label + part_separator title = "X509v3 Subject Alternative Name:" - text = jose.ComparableX509(cert_or_req).dump(OpenSSL.crypto.FILETYPE_TEXT) + if isinstance(cert_or_req, OpenSSL.crypto.X509): + func = OpenSSL.crypto.dump_certificate + else: + func = OpenSSL.crypto.dump_certificate_request + text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req) + lines = iter(text.decode("utf-8").splitlines()) sans = [next(lines).split(parts_separator) for line in lines if title in line] diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 25b0d2f67..b3c39c388 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -18,7 +18,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): self.cert = test_util.load_comparable_cert('cert.pem') key = test_util.load_pyopenssl_private_key('rsa512_key.pem') # pylint: disable=protected-access - certs = {b'foo': (key, self.cert._wrapped)} + certs = {b'foo': (key, self.cert.wrapped)} from acme.crypto_util import SSLSocket diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 66776172b..42fb389e6 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -372,7 +372,8 @@ def encode_cert(cert): :rtype: unicode """ - return encode_b64jose(cert.dump()) + return encode_b64jose(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) def decode_cert(b64der): @@ -396,7 +397,8 @@ def encode_csr(csr): :rtype: unicode """ - return encode_b64jose(csr.dump()) + return encode_b64jose(OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) def decode_csr(b64der): diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py index 939932d36..9c14cf729 100644 --- a/acme/acme/jose/jws.py +++ b/acme/acme/jose/jws.py @@ -123,7 +123,8 @@ class Header(json_util.JSONObjectWithFields): @x5c.encoder def x5c(value): # pylint: disable=missing-docstring,no-self-argument - return [base64.b64encode(cert.dump()) for cert in value] + return [base64.b64encode(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] @x5c.decoder def x5c(value): # pylint: disable=missing-docstring,no-self-argument diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py index 3fd0fbb89..ec91f6a1b 100644 --- a/acme/acme/jose/jws_test.py +++ b/acme/acme/jose/jws_test.py @@ -3,6 +3,7 @@ import base64 import unittest import mock +import OpenSSL from acme import test_util @@ -67,10 +68,12 @@ class HeaderTest(unittest.TestCase): from acme.jose.jws import Header header = Header(x5c=(CERT, CERT)) jobj = header.to_partial_json() - cert_b64 = base64.b64encode(CERT.dump()) + cert_asn1 = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) + cert_b64 = base64.b64encode(cert_asn1) self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) self.assertEqual(header, Header.from_json(jobj)) - jobj['x5c'][0] = base64.b64encode(b'xxx' + CERT.dump()) + jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) self.assertRaises(errors.DeserializationError, Header.from_json, jobj) def test_find_key(self): diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index 1d98aad4e..a7a129800 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -29,50 +29,50 @@ class abstractclassmethod(classmethod): class ComparableX509(object): # pylint: disable=too-few-public-methods """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. - Wraps around: - - - :class:`OpenSSL.crypto.X509` - - :class:`OpenSSL.crypto.X509Req` + :ivar wrapped: Wrapped certificate or certificate request. + :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. """ def __init__(self, wrapped): assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( wrapped, OpenSSL.crypto.X509Req) - self._wrapped = wrapped + self.wrapped = wrapped def __getattr__(self, name): - return getattr(self._wrapped, name) + return getattr(self.wrapped, name) - def dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): + def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): """Dumps the object into a buffer with the specified encoding. :param int filetype: The desired encoding. Should be one of - OpenSSL.crypto.FILETYPE_ASN1, OpenSSL.crypto.FILETYPE_PEM, - or OpenSSL.crypto.FILETYPE_TEXT. + `OpenSSL.crypto.FILETYPE_ASN1`, + `OpenSSL.crypto.FILETYPE_PEM`, or + `OpenSSL.crypto.FILETYPE_TEXT`. :returns: Encoded X509 object. :rtype: str """ - if isinstance(self._wrapped, OpenSSL.crypto.X509): + if isinstance(self.wrapped, OpenSSL.crypto.X509): func = OpenSSL.crypto.dump_certificate else: # assert in __init__ makes sure this is X509Req func = OpenSSL.crypto.dump_certificate_request - return func(filetype, self._wrapped) + return func(filetype, self.wrapped) def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented - return self.dump() == other.dump() + # pylint: disable=protected-access + return self._dump() == other._dump() def __hash__(self): - return hash((self.__class__, self.dump())) + return hash((self.__class__, self._dump())) def __ne__(self, other): return not self == other def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) + return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) class ComparableKey(object): # pylint: disable=too-few-public-methods diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1a141f556..dfebde13c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -391,9 +391,9 @@ def _auth_from_domains(le_client, config, domains): new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) # TODO: Check whether it worked! <- or make sure errors are thrown (jdk) lineage.save_successor( - lineage.latest_common_version(), - new_certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), new_key.pem, - crypto_util.dump_pyopenssl_chain(new_chain)) + lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), + new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 59ac11a72..c2dfca1bf 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -299,7 +299,8 @@ class Client(object): "by your operating system package manager") lineage = storage.RenewableCert.new_lineage( - domains[0], certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), + domains[0], OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), key.pem, crypto_util.dump_pyopenssl_chain(chain), params, config, cli_config) return lineage @@ -328,7 +329,8 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = certr.body.dump(OpenSSL.crypto.FILETYPE_PEM) + cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, act_cert_path = le_util.unique_file(cert_path, 0o644) try: cert_file.write(cert_pem) diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index f897ec852..730c32398 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -271,7 +271,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): def _dump_cert(cert): if isinstance(cert, jose.ComparableX509): # pylint: disable=protected-access - cert = cert._wrapped + cert = cert.wrapped return OpenSSL.crypto.dump_certificate(filetype, cert) # assumes that OpenSSL.crypto.dump_certificate includes ending diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 6e2366d82..ed4910ef9 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -102,7 +102,8 @@ def renew(cert, old_version): # new_key if the old key is to be used (since save_successor # already understands this distinction!) return cert.save_successor( - old_version, new_certr.body.dump(OpenSSL.crypto.FILETYPE_PEM), + old_version, OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) # TODO: Notify results else: From c271dc58ce4efb884666217a182d2785a9a4bf42 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 24 Dec 2015 20:55:44 -0800 Subject: [PATCH 032/127] Fix the letsencrypt-auto update script --- tests/letstest/scripts/test_leauto_upgrades.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 70f8a2293..b7849755a 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -7,7 +7,9 @@ cd letsencrypt #git checkout v0.1.0 use --branch instead SAVE="$PIP_EXTRA_INDEX_URL" unset PIP_EXTRA_INDEX_URL +export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" ./letsencrypt-auto -v --debug --version +unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" From dc5c0933df775a49d93e0401cfc7306127fcb733 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 24 Dec 2015 21:39:28 -0800 Subject: [PATCH 033/127] Definitely need --debug for this --- tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh index 476ad8bde..234e70f68 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh @@ -4,4 +4,4 @@ cd letsencrypt # help installs virtualenv and does nothing else -./letsencrypt-auto -v --help all +./letsencrypt-auto -v --debug --help all From e0837ad41f1e2bee0df17ccb77948abc1767f0c9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 02:39:05 -0800 Subject: [PATCH 034/127] [letstest] create & reuse a persistent boulder server Reduces minimum multitester.py runtime to just a bit under 10 minutes --- tests/letstest/multitester.py | 87 ++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 5aca79b7a..60be10447 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -77,6 +77,12 @@ parser.add_argument('--saveinstances', parser.add_argument('--alt_pip', default='', help="server from which to pull candidate release packages") +parser.add_argument('--killboulder', + action='store_true', + help="do not leave a persistent boulder server running") +parser.add_argument('--boulderonly', + action='store_true', + help="only make a boulder server") cl_args = parser.parse_args() # Credential Variables @@ -292,6 +298,29 @@ def grab_letsencrypt_log(): sudo('if [ -f ./letsencrypt.log ]; then \ cat ./letsencrypt.log; else echo "[nolocallog]"; fi') +def create_client_instances(targetlist): + "Create a fleet of client instances" + instances = [] + print("Creating instances: ", end="") + for target in targetlist: + if target['virt'] == 'hvm': + machine_type = 't2.micro' + else: + machine_type = 't1.micro' + if 'userdata' in target.keys(): + userdata = target['userdata'] + else: + userdata = '' + name = 'le-%s'%target['name'] + print(name, end=" ") + instances.append(make_instance(name, + target['ami'], + KEYNAME, + machine_type=machine_type, + userdata=userdata)) + print() + return instances + #------------------------------------------------------------------------------- # SCRIPT BEGINS #------------------------------------------------------------------------------- @@ -352,30 +381,28 @@ if not sg_exists: make_security_group() time.sleep(30) +boulder_preexists = False +boulder_servers = EC2.instances.filter(Filters=[ + {'Name': 'tag:Name', 'Values': ['le-boulderserver']}, + {'Name': 'instance-state-name', 'Values': ['running']}]) + +boulder_server = next(iter(boulder_servers), None) + print("Requesting Instances...") -boulder_server = make_instance('le-boulderserver', - BOULDER_AMI, - KEYNAME, - #machine_type='t2.micro', - machine_type='t2.medium', - security_groups=['letsencrypt_test']) - -instances = [] -for target in targetlist: - if target['virt'] == 'hvm': - machine_type = 't2.micro' - else: - machine_type = 't1.micro' - if 'userdata' in target.keys(): - userdata = target['userdata'] - else: - userdata = '' - instances.append(make_instance('le-%s'%target['name'], - target['ami'], +if boulder_server: + print("Found existing boulder server:", boulder_server) + boulder_preexists = True +else: + print("Can't find a boulder server, starting one...") + boulder_server = make_instance('le-boulderserver', + BOULDER_AMI, KEYNAME, - machine_type=machine_type, - userdata=userdata)) + machine_type='t2.micro', + #machine_type='t2.medium', + security_groups=['letsencrypt_test']) +if not cl_args.boulderonly: + instances = create_client_instances(targetlist) # Configure and launch boulder server #------------------------------------------------------------------------------- @@ -383,21 +410,24 @@ print("Waiting on Boulder Server") boulder_server = block_until_instance_ready(boulder_server) print(" server %s"%boulder_server) -print("Configuring and Launching Boulder") # env.host_string defines the ssh user and host for connection env.host_string = "ubuntu@%s"%boulder_server.public_ip_address print("Boulder Server at (SSH):", env.host_string) -config_and_launch_boulder(boulder_server) -# blocking often unnecessary, but cheap EC2 VMs can get very slow -block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, - timeout=500) +if not boulder_preexists: + print("Configuring and Launching Boulder") + config_and_launch_boulder(boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) print("Boulder Server at (EC2 private ip): %s"%boulder_url) +if cl_args.boulderonly: + sys.exit(0) + # Install and launch client scripts in parallel #------------------------------------------------------------------------------- print("Uploading and running test script in parallel: %s"%cl_args.test_script) @@ -480,7 +510,8 @@ results_file.close() if not cl_args.saveinstances: print('Logs in ', LOGDIR) print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes') - boulder_server.terminate() + if cl_args.killboulder: + boulder_server.terminate() terminate_and_clean(instances) else: # print login information for the boxes for debugging From f5a0268f172433f5cf91c11fc7dd8c12f5f3e3f0 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Tue, 24 Nov 2015 13:25:28 +0200 Subject: [PATCH 035/127] Adding Python 2.6/2.7 note to the docs --- docs/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c71aefeec..ea9a9a16c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -365,10 +365,12 @@ are provided here mainly for the :ref:`developers ` reference. In general: * ``sudo`` is required as a suggested way of running privileged process +* `Python`_ 2.6/2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies +.. _Python: https://wiki.python.org/moin/BeginnersGuide/Download .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io From 3e1bc19e0f5e6ef417328c39c027c8e61ac50e2e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 25 Dec 2015 10:43:52 -0800 Subject: [PATCH 036/127] Optional flag for faster AMIs --- tests/letstest/multitester.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 60be10447..dee6968c3 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -83,6 +83,9 @@ parser.add_argument('--killboulder', parser.add_argument('--boulderonly', action='store_true', help="only make a boulder server") +parser.add_argument('--fast', + action='store_true', + help="use larger instance types to run faster (saves about a minute, probably not worth it)") cl_args = parser.parse_args() # Credential Variables @@ -304,9 +307,10 @@ def create_client_instances(targetlist): print("Creating instances: ", end="") for target in targetlist: if target['virt'] == 'hvm': - machine_type = 't2.micro' + machine_type = 't2.medium' if cl_args.fast else 't2.micro' else: - machine_type = 't1.micro' + # 32 bit systems + machine_type = 'c1.medium' if cl_args.fast else 't1.micro' if 'userdata' in target.keys(): userdata = target['userdata'] else: From f20cd73e8dc1e16ecff9d1161ec73076c05f836d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 26 Dec 2015 20:13:45 -0800 Subject: [PATCH 037/127] For now, disable apacheconftest in travis :( --- .travis.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e0849e3b..a5d6d8a85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: python services: - rabbitmq - mariadb - - apache2 + # apacheconftest + #- apache2 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml @@ -23,7 +24,9 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover - - TOXENV=apacheconftest +# Disabled for now due to requiring sudo -> causing more boulder integration +# DNS timeouts :( +# - TOXENV=apacheconftest # Only build pushes to the master branch, PRs, and branches beginning with @@ -35,7 +38,7 @@ branches: - /^test-.*$/ # container-based infrastructure -sudo: true +sudo: false addons: # make sure simplehttp simple verification works (custom /etc/hosts) @@ -61,11 +64,11 @@ addons: # For Boulder integration testing - rsyslog # for apacheconftest - - realpath - - apache2 - - libapache2-mod-wsgi - - libapache2-mod-macro - - sudo + #- realpath + #- apache2 + #- libapache2-mod-wsgi + #- libapache2-mod-macro + #- sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' From 83812dc16a081ddc43d8f4204d61f0b7e73e8768 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 12:56:44 +0200 Subject: [PATCH 038/127] Abstract config file matching to os based constants --- letsencrypt-apache/letsencrypt_apache/constants.py | 3 +++ letsencrypt-apache/letsencrypt_apache/parser.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index f8712c247..800959463 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -6,6 +6,7 @@ from letsencrypt import le_util CLI_DEFAULTS_DEBIAN = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/sites-available", + vhost_files="*", ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], @@ -19,6 +20,7 @@ CLI_DEFAULTS_DEBIAN = dict( CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", + vhost_files="*.conf", ctl="apachectl", version_cmd=['apachectl', '-v'], define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], @@ -32,6 +34,7 @@ CLI_DEFAULTS_CENTOS = dict( CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", + vhost_files="*.conf", ctl="apache2ctl", version_cmd=['/usr/sbin/apache2', '-v'], define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'], diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 12704c859..5f84e9f52 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -60,9 +60,10 @@ class ApacheParser(object): self.loc.update(self._set_locations()) # Must also attempt to parse virtual host root - self._parse_file(self.vhostroot + "/*.conf") + self._parse_file(self.vhostroot + "/" + + constants.os_constant("vhost_files")) - #check to see if there were unparsed define statements + # check to see if there were unparsed define statements if version < (2, 4): if self.find_dir("Define", exclude=False): raise errors.PluginError("Error parsing runtime variables") From e3358bb15346cbcedd229c37d072c05f2198cf4a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 13:47:14 +0200 Subject: [PATCH 039/127] Abstract the remaining commands to configurable ones --- .../letsencrypt_apache/configurator.py | 22 ++++++++----------- .../letsencrypt_apache/constants.py | 9 +++++--- .../letsencrypt_apache/parser.py | 16 ++++++++------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 6c6685257..5e2156edc 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -86,10 +86,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @classmethod def add_parser_arguments(cls, add): - add("ctl", default=constants.os_constant("ctl"), - help="Path to the 'apache2ctl' binary, used for 'configtest', " - "retrieving the Apache2 version number, and initialization " - "parameters.") add("enmod", default=constants.os_constant("enmod"), help="Path to the Apache 'a2enmod' binary.") add("dismod", default=constants.os_constant("dismod"), @@ -148,10 +144,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Verify Apache is installed - for exe in (self.conf("ctl"), self.conf("enmod"), self.conf("dismod")): - if exe is not None: - if not le_util.exe_exists(exe): - raise errors.NoInstallationError + for exe in constants.os_constant("restart_cmd")[0]: + if not le_util.exe_exists(exe): + raise errors.NoInstallationError # Make sure configuration is valid self.config_test() @@ -165,7 +160,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.conf("vhost-root"), - self.conf("ctl"), self.version) + self.version) # Check for errors in parsing files with Augeas self.check_parsing_errors("httpd.aug") @@ -1277,7 +1272,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Modules can enable additional config files. Variables may be defined # within these new configuration sections. # Reload is not necessary as DUMP_RUN_CFG uses latest config. - self.parser.update_runtime_variables(self.conf("ctl")) + self.parser.update_runtime_variables() def _add_parser_mod(self, mod_name): """Shortcut for updating parser modules.""" @@ -1315,7 +1310,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "graceful"]) + le_util.run_script(constants.os_constant("restart_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1326,7 +1321,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script([self.conf("ctl"), "configtest"]) + le_util.run_script(constants.os_constant("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1346,7 +1341,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): constants.os_constant("version_cmd")) except errors.SubprocessError: raise errors.PluginError( - "Unable to run %s -v" % self.conf("ctl")) + "Unable to run %s -v" % + constants.os_constant("version_cmd")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 800959463..8ac88b197 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -7,9 +7,10 @@ CLI_DEFAULTS_DEBIAN = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/sites-available", vhost_files="*", - ctl="apache2ctl", version_cmd=['apache2ctl', '-v'], define_cmd=['apache2ctl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], enmod="a2enmod", dismod="a2dismod", le_vhost_ext="-le-ssl.conf", @@ -21,9 +22,10 @@ CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", - ctl="apachectl", version_cmd=['apachectl', '-v'], define_cmd=['apachectl', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", @@ -35,9 +37,10 @@ CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - ctl="apache2ctl", version_cmd=['/usr/sbin/apache2', '-v'], define_cmd=['/usr/sbin/apache2', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apache2ctl', 'graceful'], + conftest_cmd=['apache2ctl', 'configtest'], enmod=None, dismod=None, le_vhost_ext="-le-ssl.conf", diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 5f84e9f52..593c807cc 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -28,7 +28,7 @@ class ApacheParser(object): arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") fnmatch_chars = set(["*", "?", "\\", "[", "]"]) - def __init__(self, aug, root, vhostroot, ctl, version=(2, 4)): + def __init__(self, aug, root, vhostroot, version=(2, 4)): # Note: Order is important here. # This uses the binary, so it can be done first. @@ -37,7 +37,7 @@ class ApacheParser(object): # This only handles invocation parameters and Define directives! self.variables = {} if version >= (2, 4): - self.update_runtime_variables(ctl) + self.update_runtime_variables() self.aug = aug # Find configuration root and make sure augeas can parse it. @@ -92,7 +92,7 @@ class ApacheParser(object): self.modules.add( os.path.basename(self.get_arg(match_filename))[:-2] + "c") - def update_runtime_variables(self, ctl): + def update_runtime_variables(self): """" .. note:: Compile time variables (apache2ctl -V) are not used within the @@ -102,7 +102,7 @@ class ApacheParser(object): .. todo:: Create separate compile time variables... simply for arg_get() """ - stdout = self._get_runtime_cfg(ctl) + stdout = self._get_runtime_cfg() variables = dict() matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) @@ -122,7 +122,7 @@ class ApacheParser(object): self.variables = variables - def _get_runtime_cfg(self, ctl): # pylint: disable=no-self-use + def _get_runtime_cfg(self): # pylint: disable=no-self-use """Get runtime configuration info. :returns: stdout from DUMP_RUN_CFG @@ -137,9 +137,11 @@ class ApacheParser(object): except (OSError, ValueError): logger.error( - "Error accessing %s for runtime parameters!%s", ctl, os.linesep) + "Error running command %s for runtime parameters!%s", + constants.os_constant("define_cmd"), os.linesep) raise errors.MisconfigurationError( - "Error accessing loaded Apache parameters: %s", ctl) + "Error accessing loaded Apache parameters: %s", + constants.os_constant("define_cmd")) # Small errors that do not impede if proc.returncode != 0: logger.warn("Error in checking parameter list: %s", stderr) From ab069741f2c4cf32b911180e1a166c809a144c0a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 14:03:50 +0200 Subject: [PATCH 040/127] Fix tests --- .../tests/constants_test.py | 9 +++++--- .../letsencrypt_apache/tests/parser_test.py | 22 ++++++++++--------- .../letsencrypt_apache/tests/util.py | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py index 63eb5c783..289b61bb1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/constants_test.py @@ -11,14 +11,17 @@ class ConstantsTest(unittest.TestCase): @mock.patch("letsencrypt.le_util.get_os_info") def test_get_debian_value(self, os_info): os_info.return_value = ('Debian', '', '') - self.assertEqual(constants.os_constant("ctl"), "apache2ctl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") @mock.patch("letsencrypt.le_util.get_os_info") def test_get_centos_value(self, os_info): os_info.return_value = ('CentOS Linux', '', '') - self.assertEqual(constants.os_constant("ctl"), "apachectl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/httpd/conf.d") @mock.patch("letsencrypt.le_util.get_os_info") def test_get_default_value(self, os_info): os_info.return_value = ('Nonexistent Linux', '', '') - self.assertEqual(constants.os_constant("ctl"), "apache2ctl") + self.assertEqual(constants.os_constant("vhost_root"), + "/etc/apache2/sites-available") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 023b3990a..b871f89b7 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -145,24 +145,26 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.update_runtime_variables("ctl") + self.parser.update_runtime_variables() self.assertEqual(self.parser.variables, expected_vars) @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_vars_bad_output(self, mock_cfg): mock_cfg.return_value = "Define: TLS=443=24" - self.parser.update_runtime_variables("ctl") + self.parser.update_runtime_variables() mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24" self.assertRaises( - errors.PluginError, self.parser.update_runtime_variables, "ctl") + errors.PluginError, self.parser.update_runtime_variables) + @mock.patch("letsencrypt_apache.constants.os_constant") @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen): + def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const): mock_popen.side_effect = OSError + mock_const.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) @mock.patch("letsencrypt_apache.parser.subprocess.Popen") def test_update_runtime_vars_bad_exit(self, mock_popen): @@ -170,7 +172,7 @@ class BasicParserTest(util.ParserTest): mock_popen.returncode = -1 self.assertRaises( errors.MisconfigurationError, - self.parser.update_runtime_variables, "ctl") + self.parser.update_runtime_variables) class ParserInitTest(util.ApacheTest): @@ -191,7 +193,7 @@ class ParserInitTest(util.ApacheTest): self.assertRaises( errors.PluginError, ApacheParser, self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath", "ctl", version=(2, 2, 22)) + "/dummy/vhostpath", version=(2, 2, 22)) def test_root_normalized(self): from letsencrypt_apache.parser import ApacheParser @@ -203,7 +205,7 @@ class ParserInitTest(util.ApacheTest): "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2") parser = ApacheParser(self.aug, path, - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -213,7 +215,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, os.path.relpath(self.config_path), - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) @@ -223,7 +225,7 @@ class ParserInitTest(util.ApacheTest): "update_runtime_variables"): parser = ApacheParser( self.aug, self.config_path + os.path.sep, - "/dummy/vhostpath", "dummy_ctl") + "/dummy/vhostpath") self.assertEqual(parser.root, self.config_path) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 95c95e6a9..798d4814b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -58,7 +58,7 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): self.parser = ApacheParser( - self.aug, self.config_path, self.vhost_path, "dummy_ctl_path") + self.aug, self.config_path, self.vhost_path) def get_apache_configurator( From 3fadfb5444b38e65fb60507c2592eaeac5cdde6f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 28 Dec 2015 15:56:24 +0200 Subject: [PATCH 041/127] Fixed the find exe condition --- letsencrypt-apache/letsencrypt_apache/configurator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 5e2156edc..836d77135 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -144,9 +144,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Verify Apache is installed - for exe in constants.os_constant("restart_cmd")[0]: - if not le_util.exe_exists(exe): - raise errors.NoInstallationError + if not le_util.exe_exists(constants.os_constant("restart_cmd")[0]): + raise errors.NoInstallationError # Make sure configuration is valid self.config_test() From fd4f6fb2eef3fd24d427836023918103bac08ada Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 29 Dec 2015 08:47:14 +0000 Subject: [PATCH 042/127] Use GH pages for IETF spec repo link --- acme/acme/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 0f5f0e4bd..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -3,10 +3,10 @@ This module is an implementation of the `ACME protocol`_. Latest supported version: `draft-ietf-acme-01`_. -.. _`ACME protocol`: https://github.com/ietf-wg-acme/acme/ + +.. _`ACME protocol`: https://ietf-wg-acme.github.io/acme .. _`draft-ietf-acme-01`: https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ From 7788799a9bdb6a67f25833ccba031799b7b6429a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 29 Dec 2015 08:55:13 +0000 Subject: [PATCH 043/127] Staging URI in dev-cli.ini example --- examples/cli.ini | 3 --- examples/dev-cli.ini | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cli.ini b/examples/cli.ini index 6b6b05d7d..f0c993c57 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -5,9 +5,6 @@ # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 -# Always use the staging/testing server -server = https://acme-staging.api.letsencrypt.org/directory - # Uncomment and update to register with the specified e-mail address # email = foo@example.com diff --git a/examples/dev-cli.ini b/examples/dev-cli.ini index be703814a..c02038ca1 100644 --- a/examples/dev-cli.ini +++ b/examples/dev-cli.ini @@ -1,3 +1,6 @@ +# Always use the staging/testing server - avoids rate limiting +server = https://acme-staging.api.letsencrypt.org/directory + # This is an example configuration file for developers config-dir = /tmp/le/conf work-dir = /tmp/le/conf From 8f984bd24f2779490bf526d47f7fbe14633fcfb1 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 1 Jan 2016 16:35:57 -0800 Subject: [PATCH 044/127] Better Nginx error handling. Raise MisconfigurationError on restart failure, so we don't attempt to continue with an authorization we know will fail. Log at debug level the config files that are about to be written out, so it's easier to debug restart failures. Fix https://github.com/letsencrypt/letsencrypt/issues/942: Error out if adding a conflicting directive. Remove unnecessarily-inserted access_log and error_log directives. These were added to make integration testing easier but are no longer needed. Incidentally this makes the plugin work with some configs where it wouldn't have worked previously. Change the semantics of add_server_directives with replace=True so only the first instance of a given directive is replaced, not all of them. This works fine with the one place in the code that calls add_server_directives with replace=True, because all of the involved directives aren't allowed to be duplicated in a given block. Make add_http_directives do inserts into the structure itself, since its needs were significantly different than the more general add_server_directives. This also allows us to narrow the scope of the `block.insert(0, directive)` hack that we inserted to work around https://trac.nginx.org/nginx/ticket/810, since it's only necessary for http blocks. --- .../letsencrypt_nginx/configurator.py | 24 ++---- letsencrypt-nginx/letsencrypt_nginx/parser.py | 81 +++++++++++++------ .../tests/configurator_test.py | 63 ++++++++------- .../letsencrypt_nginx/tests/parser_test.py | 29 ++++--- .../tests/boulder-integration.conf.sh | 6 +- 5 files changed, 115 insertions(+), 88 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index aaaf43c5f..89b1145e7 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -311,17 +311,11 @@ class NginxConfigurator(common.Plugin): """ snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() ssl_block = [['listen', '{0} ssl'.format(self.config.tls_sni_01_port)], - # access and error logs necessary for integration - # testing (non-root) - ['access_log', os.path.join( - self.config.work_dir, 'access.log')], - ['error_log', os.path.join( - self.config.work_dir, 'error.log')], ['ssl_certificate', snakeoil_cert], ['ssl_certificate_key', snakeoil_key], ['include', self.parser.loc["ssl_options"]]] self.parser.add_server_directives( - vhost.filep, vhost.names, ssl_block) + vhost.filep, vhost.names, ssl_block, replace=False) vhost.ssl = True vhost.raw.extend(ssl_block) vhost.addrs.add(obj.Addr( @@ -384,7 +378,7 @@ class NginxConfigurator(common.Plugin): [['return', '301 https://$host$request_uri']] ]] self.parser.add_server_directives( - vhost.filep, vhost.names, redirect_block) + vhost.filep, vhost.names, redirect_block, replace=False) logger.info("Redirecting all traffic to ssl in %s", vhost.filep) ###################################### @@ -393,11 +387,10 @@ class NginxConfigurator(common.Plugin): def restart(self): """Restarts nginx server. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If either the reload fails. """ - return nginx_restart(self.conf('ctl'), self.nginx_conf) + nginx_restart(self.conf('ctl'), self.nginx_conf) def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. @@ -631,19 +624,16 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): if nginx_proc.returncode != 0: # Enter recovery routine... - logger.error("Nginx Restart Failed!\n%s\n%s", stdout, stderr) - return False + raise errors.MisconfigurationError( + "nginx restart failed:\n%s\n%s" % (stdout, stderr)) except (OSError, ValueError): - logger.fatal("Nginx Restart Failed - Please Check the Configuration") - sys.exit(1) + raise errors.MisconfigurationError("nginx restart failed") # Nginx can take a moment to recognize a newly added TLS SNI servername, so sleep # for a second. TODO: Check for expected servername and loop until it # appears or return an error if looping too long. time.sleep(1) - return True - def temp_install(options_ssl): """Temporary install for convenience.""" diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index 14db2f8b7..1d424f834 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -213,6 +213,7 @@ class NginxParser(object): if ext: filename = filename + os.path.extsep + ext try: + logger.debug('Dumping to %s:\n%s', filename, nginxparser.dumps(tree)) with open(filename, 'w') as _file: nginxparser.dump(tree, _file) except IOError: @@ -252,7 +253,7 @@ class NginxParser(object): return server_names == names def add_server_directives(self, filename, names, directives, - replace=False): + replace): """Add or replace directives in the first server block with names. ..note :: If replace is True, this raises a misconfiguration error @@ -269,20 +270,27 @@ class NginxParser(object): :param bool replace: Whether to only replace existing directives """ - _do_for_subarray(self.parsed[filename], - lambda x: self._has_server_names(x, names), - lambda x: _add_directives(x, directives, replace)) + try: + _do_for_subarray(self.parsed[filename], + lambda x: self._has_server_names(x, names), + lambda x: _add_directives(x, directives, replace)) + except errors.MisconfigurationError as err: + raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message)) def add_http_directives(self, filename, directives): """Adds directives to the first encountered HTTP block in filename. + We insert new directives at the top of the block to work around + https://trac.nginx.org/nginx/ticket/810: If the first server block + doesn't enable OCSP stapling, stapling is broken for all blocks. + :param str filename: The absolute filename of the config file :param list directives: The directives to add """ _do_for_subarray(self.parsed[filename], lambda x: x[0] == ['http'], - lambda x: _add_directives(x[1], [directives], False)) + lambda x: x[1].insert(0, directives)) def get_all_certs_keys(self): """Gets all certs and keys in the nginx config. @@ -467,9 +475,14 @@ def _parse_server(server): return parsed_server -def _add_directives(block, directives, replace=False): - """Adds or replaces directives in a block. If the directive doesn't exist in - the entry already, raises a misconfiguration error. +def _add_directives(block, directives, replace): + """Adds or replaces directives in a config block. + + When replace=False, it's an error to try and add a directive that already + exists in the config block with a conflicting value. + + When replace=True, a directive with the same name MUST already exist in the + config block, and the first instance will be replaced. ..todo :: Find directives that are in included files. @@ -478,21 +491,39 @@ def _add_directives(block, directives, replace=False): """ for directive in directives: - if not replace: - # We insert new directives at the top of the block, mostly - # to work around https://trac.nginx.org/nginx/ticket/810 - # Only add directive if its not already in the block - if directive not in block: - block.insert(0, directive) - else: - changed = False - if len(directive) == 0: - continue - for index, line in enumerate(block): - if len(line) > 0 and line[0] == directive[0]: - block[index] = directive - changed = True - if not changed: + _add_directive(block, directive, replace) + +repeatable_directives = set(['server_name', 'listen', 'include']) + +def _add_directive(block, directive, replace): + """Adds or replaces a single directive in a config block. + + See _add_directives for more documentation. + + """ + location = -1 + # Find the index of a config line where the name of the directive matches + # the name of the directive we want to add. + for index, line in enumerate(block): + if len(line) > 0 and line[0] == directive[0]: + location = index + break + if replace: + if location == -1: + raise errors.MisconfigurationError( + 'expected directive for %s in the Nginx ' + 'config but did not find it.' % directive[0]) + block[location] = directive + else: + # Append directive. Fail if the name is not a repeatable directive name, + # and there is already a copy of that directive with a different value + # in the config file. + if location != -1 and directive[0].__str__() not in repeatable_directives: + if block[location][1] == directive[1]: + pass + else: raise errors.MisconfigurationError( - 'Let\'s Encrypt expected directive for %s in the Nginx ' - 'config but did not find it.' % directive[0]) + 'tried to insert directive "%s" but found conflicting "%s".' % ( + directive, block[location])) + else: + block.append(directive) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..f6a4f7eee 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -65,16 +65,19 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') self.config.parser.add_server_directives( filep, set(['.example.com', 'example.*']), - [['listen', '5001 ssl']]) + [['listen', '5001 ssl']], + replace=False) self.config.save() # pylint: disable=protected-access parsed = self.config.parser._parse_files(filep, override=True) - self.assertEqual([[['server'], [['listen', '5001 ssl'], + self.assertEqual([[['server'], [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + ['listen', '5001 ssl'] + ]]], parsed[0]) def test_choose_vhost(self): @@ -154,38 +157,36 @@ class NginxConfiguratorTest(util.NginxTest): parsed_server_conf = util.filter_comments(self.config.parser.parsed[server_conf]) parsed_nginx_conf = util.filter_comments(self.config.parser.parsed[nginx_conf]) - access_log = os.path.join(self.work_dir, "access.log") - error_log = os.path.join(self.work_dir, "error.log") self.assertEqual([[['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', 'example/key.pem'], - ['ssl_certificate', 'example/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - - ['listen', '5001 ssl'], + [ ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], - ['server_name', 'example.*']]]], + ['server_name', 'example.*'], + + ['listen', '5001 ssl'], + ['ssl_certificate', 'example/fullchain.pem'], + ['ssl_certificate_key', 'example/key.pem'], + ['include', self.config.parser.loc["ssl_options"]] + ]]], parsed_example_conf) self.assertEqual([['server_name', 'somename alias another.alias']], parsed_server_conf) - self.assertTrue(util.contains_at_depth(parsed_nginx_conf, - [['server'], - [['include', self.config.parser.loc["ssl_options"]], - ['ssl_certificate_key', '/etc/nginx/key.pem'], - ['ssl_certificate', '/etc/nginx/fullchain.pem'], - ['error_log', error_log], - ['access_log', access_log], - ['listen', '5001 ssl'], - ['listen', '8000'], - ['listen', 'somename:8080'], - ['include', 'server.conf'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html index.htm']]]]], - 2)) + self.assertTrue(util.contains_at_depth( + parsed_nginx_conf, + [['server'], + [ + ['listen', '8000'], + ['listen', 'somename:8080'], + ['include', 'server.conf'], + [['location', '/'], + [['root', 'html'], + ['index', 'index.html index.htm']]], + ['listen', '5001 ssl'], + ['ssl_certificate', '/etc/nginx/fullchain.pem'], + ['ssl_certificate_key', '/etc/nginx/key.pem'], + ['include', self.config.parser.loc["ssl_options"]]]], + 2)) def test_get_all_certs_keys(self): nginx_conf = self.config.parser.abs_path('nginx.conf') @@ -297,19 +298,19 @@ class NginxConfiguratorTest(util.NginxTest): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 0 - self.assertTrue(self.config.restart()) + self.config.restart() @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_nginx_restart_fail(self, mock_popen): mocked = mock_popen() mocked.communicate.return_value = ('', '') mocked.returncode = 1 - self.assertFalse(self.config.restart()) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_no_nginx_start(self, mock_popen): mock_popen.side_effect = OSError("Can't find program") - self.assertRaises(SystemExit, self.config.restart) + self.assertRaises(errors.MisconfigurationError, self.config.restart) @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") def test_config_test(self, mock_popen): diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 2d6156429..6559a5df6 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -127,7 +127,8 @@ class NginxParserTest(util.NginxTest): set(['localhost', r'~^(www\.)?(example|bar)\.']), [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert.pem']]) + '/etc/ssl/cert.pem']], + replace=False) ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') dump = nginxparser.dumps(nparser.parsed[nparser.abs_path('nginx.conf')]) self.assertEqual(1, len(re.findall(ssl_re, dump))) @@ -136,12 +137,15 @@ class NginxParserTest(util.NginxTest): names = set(['alias', 'another.alias', 'somename']) nparser.add_server_directives(server_conf, names, [['foo', 'bar'], ['ssl_certificate', - '/etc/ssl/cert2.pem']]) - nparser.add_server_directives(server_conf, names, [['foo', 'bar']]) + '/etc/ssl/cert2.pem']], + replace=False) + nparser.add_server_directives(server_conf, names, [['foo', 'bar']], + replace=False) self.assertEqual(nparser.parsed[server_conf], - [['ssl_certificate', '/etc/ssl/cert2.pem'], + [['server_name', 'somename alias another.alias'], ['foo', 'bar'], - ['server_name', 'somename alias another.alias']]) + ['ssl_certificate', '/etc/ssl/cert2.pem'] + ]) def test_add_http_directives(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) @@ -165,17 +169,19 @@ class NginxParserTest(util.NginxTest): target = set(['.example.com', 'example.*']) filep = nparser.abs_path('sites-enabled/example.com') nparser.add_server_directives( - filep, target, [['server_name', 'foo bar']], True) + filep, target, [['server_name', 'foobar.com']], replace=True) self.assertEqual( nparser.parsed[filep], [[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], - ['server_name', 'foo bar'], - ['server_name', 'foo bar']]]]) + ['server_name', 'foobar.com'], + ['server_name', 'example.*'], + ]]]) self.assertRaises(errors.MisconfigurationError, nparser.add_server_directives, - filep, set(['foo', 'bar']), - [['ssl_certificate', 'cert.pem']], True) + filep, set(['foobar.com', 'example.*']), + [['ssl_certificate', 'cert.pem']], + replace=True) def test_get_best_match(self): target_name = 'www.eff.org' @@ -217,7 +223,8 @@ class NginxParserTest(util.NginxTest): set(['.example.com', 'example.*']), [['ssl_certificate', 'foo.pem'], ['ssl_certificate_key', 'bar.key'], - ['listen', '443 ssl']]) + ['listen', '443 ssl']], + replace=False) c_k = nparser.get_all_certs_keys() self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k) diff --git a/letsencrypt-nginx/tests/boulder-integration.conf.sh b/letsencrypt-nginx/tests/boulder-integration.conf.sh index 12610d895..d77669a76 100755 --- a/letsencrypt-nginx/tests/boulder-integration.conf.sh +++ b/letsencrypt-nginx/tests/boulder-integration.conf.sh @@ -20,13 +20,14 @@ events { } http { - # Set an array of temp and cache file options that will otherwise default to + # Set an array of temp, cache and log file options that will otherwise default to # restricted locations accessible only to root. client_body_temp_path $root/client_body; fastcgi_temp_path $root/fastcgi_temp; proxy_temp_path $root/proxy_temp; #scgi_temp_path $root/scgi_temp; #uwsgi_temp_path $root/uwsgi_temp; + access_log $root/error.log; # This should be turned off in a Virtualbox VM, as it can cause some # interesting issues with data corruption in delivered files. @@ -54,9 +55,6 @@ http { root $root/webroot; - access_log $root/access.log; - error_log $root/error.log; - location / { # First attempt to serve request as file, then as directory, then fall # back to index.html. From dc3a2da9b11b26be54b035ac1520128e8efbfe22 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 3 Jan 2016 10:49:50 -0500 Subject: [PATCH 045/127] Fixed a typo in a comment --- acme/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index ba2c88394..7314152cd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -31,7 +31,7 @@ else: install_requires.append('mock') if sys.version_info < (2, 7, 9): - # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) + # For secure SSL connection with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') install_requires.append('pyasn1') From 0454031cce4b88fef44e3e129e879a35b49c2314 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 3 Jan 2016 14:37:08 -0500 Subject: [PATCH 046/127] Fixed a pair of typos in docstrings --- acme/acme/jose/json_util.py | 2 +- acme/acme/jose/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 7b95e3fce..977a06622 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -226,7 +226,7 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): :param str name: Name of the field to be encoded. - :raises erors.SerializationError: if field cannot be serialized + :raises errors.SerializationError: if field cannot be serialized :raises errors.Error: if field could not be found """ diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index ab3606efc..600077b20 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -130,7 +130,7 @@ class ImmutableMap(collections.Mapping, collections.Hashable): """Immutable key to value mapping with attribute access.""" __slots__ = () - """Must be overriden in subclasses.""" + """Must be overridden in subclasses.""" def __init__(self, **kwargs): if set(kwargs) != set(self.__slots__): From a718cfede0f44cbc0c40ae396911fa3f510c3c7e Mon Sep 17 00:00:00 2001 From: sagi Date: Sun, 3 Jan 2016 22:03:47 +0000 Subject: [PATCH 047/127] Copy only relevant lines from http vhost to ssl vhost skeleton --- .../letsencrypt_apache/configurator.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 836d77135..0d6749638 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -712,8 +712,19 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): with open(avail_fp, "r") as orig_file: with open(ssl_fp, "w") as new_file: new_file.write("\n") + + # In some cases we wouldn't want to copy the exact + # directives used in an http vhost to a ssl vhost. + # An example: + # If there's a redirect rewrite rule directive installed in + # the http vhost - copying it to the ssl vhost would cause + # a redirection loop. + blacklist_set = set(['RewriteRule', 'RewriteEngine']) + for line in orig_file: - new_file.write(line) + line_set = set(line.split()) + if not line_set & blacklist_set: # & -> Intersection + new_file.write(line) new_file.write("\n") except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") From e722a381978afe3faaeb265eefbfc45b9a83e35f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 4 Jan 2016 11:18:13 -0800 Subject: [PATCH 048/127] Clarify parser check for duplicate values. --- letsencrypt-nginx/letsencrypt_nginx/parser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index 1d424f834..c60d0102a 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -518,8 +518,12 @@ def _add_directive(block, directive, replace): # Append directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. - if location != -1 and directive[0].__str__() not in repeatable_directives: - if block[location][1] == directive[1]: + directive_name = directive[0] + directive_value = directive[1] + if location != -1 and directive_name.__str__() not in repeatable_directives: + if block[location][1] == directive_value: + # There's a conflict, but the existing value matches the one we + # want to insert, so it's fine. pass else: raise errors.MisconfigurationError( From e1b4797cbfae87b59d1b3282992a6b605e578a7f Mon Sep 17 00:00:00 2001 From: Wilfried Teiken Date: Tue, 5 Jan 2016 01:12:21 -0500 Subject: [PATCH 049/127] Change the semantics of query_registration and update_registration to set new_authzr_uri from the server if available --- acme/acme/client.py | 12 +++++------- acme/acme/client_test.py | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3e28ef47..3d84cb3bf 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -66,15 +66,13 @@ class Client(object): # pylint: disable=too-many-instance-attributes @classmethod def _regr_from_response(cls, response, uri=None, new_authzr_uri=None, terms_of_service=None): - terms_of_service = ( - response.links['terms-of-service']['url'] - if 'terms-of-service' in response.links else terms_of_service) + if 'terms-of-service' in response.links: + terms_of_service = response.links['terms-of-service']['url'] + if 'next' in response.links: + new_authzr_uri = response.links['next']['url'] if new_authzr_uri is None: - try: - new_authzr_uri = response.links['next']['url'] - except KeyError: - raise errors.ClientError('"next" link missing') + raise errors.ClientError(response, 'missing "new_authrz_uri"') return messages.RegistrationResource( body=messages.Registration.from_json(response.json()), diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 58f55b293..9e3ea3d88 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -127,6 +127,13 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) + def test_query_registration_updates_new_authzr_uri(self): + self.response.json.return_value = self.regr.body.to_json() + self.response.links = {'next': {'url': 'UPDATED'}} + self.assertEqual( + 'UPDATED', + self.client.query_registration(self.regr).new_authzr_uri) + def test_agree_to_tos(self): self.client.update_registration = mock.Mock() self.client.agree_to_tos(self.regr) From 10a49532ae459ee1b1cc45b3b72072d8bdd37310 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 5 Jan 2016 11:25:21 +0200 Subject: [PATCH 050/127] Add / replace functionality to augeas transform paths to overwrite old narrower scope if needed --- .../letsencrypt_apache/parser.py | 64 ++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 593c807cc..04b61b2d4 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -35,6 +35,7 @@ class ApacheParser(object): # https://httpd.apache.org/docs/2.4/mod/core.html#define # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine # This only handles invocation parameters and Define directives! + self.parser_paths = {} self.variables = {} if version >= (2, 4): self.update_runtime_variables() @@ -471,16 +472,61 @@ class ApacheParser(object): :param str filepath: Apache config file path """ + use_new, remove_old = self._check_path_actions(filepath) # Test if augeas included file for Httpd.lens # Note: This works for augeas globs, ie. *.conf - inc_test = self.aug.match( - "/augeas/load/Httpd/incl [. ='%s']" % filepath) - if not inc_test: - # Load up files - # This doesn't seem to work on TravisCI - # self.aug.add_transform("Httpd.lns", [filepath]) - self._add_httpd_transform(filepath) - self.aug.load() + if use_new: + inc_test = self.aug.match( + "/augeas/load/Httpd/incl [. ='%s']" % filepath) + if not inc_test: + # Load up files + # This doesn't seem to work on TravisCI + # self.aug.add_transform("Httpd.lns", [filepath]) + if remove_old: + self._remove_httpd_transform(filepath) + self._add_httpd_transform(filepath) + self.aug.load() + + def _check_path_actions(self, filepath): + """Determine actions to take with a new augeas path + + This helper function will return a tuple that defines + if we should try to append the new filepath to augeas + parser paths, and / or remove the old one with more + narrow matching. + + :param str filepath: filepath to check the actions for + + """ + + try: + use_new = False + remove_old = False + new_file_match = os.path.basename(filepath) + existing_match = self.parser_paths[os.path.dirname(filepath)] + if existing_match == new_file_match: + # True here to let augeas verify that the path is parsed + use_new = True + elif new_file_match == "*": + use_new = True + remove_old = True + except KeyError: + use_new = True + return use_new, remove_old + + def _remove_httpd_transform(self, filepath): + """Remove path from Augeas transform + + :param str filepath: filepath to remove + """ + + remove_basename = self.parser_paths[os.path.dirname(filepath)] + remove_dirname = os.path.dirname(filepath) + remove_path = remove_dirname + "/" + remove_basename + remove_inc = self.aug.match( + "/augeas/load/Httpd/incl [. ='%s']" % remove_path) + self.aug.remove(remove_inc[0]) + self.parser_paths.pop(remove_dirname) def _add_httpd_transform(self, incl): """Add a transform to Augeas. @@ -502,6 +548,8 @@ class ApacheParser(object): # Augeas uses base 1 indexing... insert at beginning... self.aug.set("/augeas/load/Httpd/lens", "Httpd.lns") self.aug.set("/augeas/load/Httpd/incl", incl) + # Add included path to paths dictionary + self.parser_paths[os.path.dirname(incl)] = os.path.basename(incl) def standardize_excl(self): """Standardize the excl arguments for the Httpd lens in Augeas. From 63f311eea492433b5a12ac685907b5d49105c44f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 5 Jan 2016 13:16:49 +0200 Subject: [PATCH 051/127] Refactored the checking method to be easier to read --- letsencrypt-apache/letsencrypt_apache/parser.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 04b61b2d4..23ca05ae0 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -500,18 +500,19 @@ class ApacheParser(object): """ try: - use_new = False - remove_old = False new_file_match = os.path.basename(filepath) existing_match = self.parser_paths[os.path.dirname(filepath)] - if existing_match == new_file_match: - # True here to let augeas verify that the path is parsed - use_new = True - elif new_file_match == "*": + if existing_match == "*": + use_new = False + else: use_new = True + if new_file_match == "*": remove_old = True + else: + remove_old = False except KeyError: use_new = True + remove_old = False return use_new, remove_old def _remove_httpd_transform(self, filepath): From 32d5375b9bb101f7e9239627c2ccb17a728046da Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 5 Jan 2016 13:21:18 +0200 Subject: [PATCH 052/127] Change test to check out adding a file not already in the augeas paths --- letsencrypt-apache/letsencrypt_apache/tests/parser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index b871f89b7..9b78bf6d6 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -36,7 +36,7 @@ class BasicParserTest(util.ParserTest): """ file_path = os.path.join( - self.config_path, "sites-available", "letsencrypt.conf") + self.config_path, "not-parsed-by-default", "letsencrypt.conf") self.parser._parse_file(file_path) # pylint: disable=protected-access From ffeef67e542dfb170d051d60b3e519fa689bbe03 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Tue, 5 Jan 2016 14:36:42 +0200 Subject: [PATCH 053/127] Use lists to handle multiple different matching wildcards in same directory --- .../letsencrypt_apache/parser.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 23ca05ae0..82effad2b 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -501,8 +501,8 @@ class ApacheParser(object): try: new_file_match = os.path.basename(filepath) - existing_match = self.parser_paths[os.path.dirname(filepath)] - if existing_match == "*": + existing_matches = self.parser_paths[os.path.dirname(filepath)] + if "*" in existing_matches: use_new = False else: use_new = True @@ -521,12 +521,13 @@ class ApacheParser(object): :param str filepath: filepath to remove """ - remove_basename = self.parser_paths[os.path.dirname(filepath)] + remove_basenames = self.parser_paths[os.path.dirname(filepath)] remove_dirname = os.path.dirname(filepath) - remove_path = remove_dirname + "/" + remove_basename - remove_inc = self.aug.match( - "/augeas/load/Httpd/incl [. ='%s']" % remove_path) - self.aug.remove(remove_inc[0]) + for name in remove_basenames: + remove_path = remove_dirname + "/" + name + remove_inc = self.aug.match( + "/augeas/load/Httpd/incl [. ='%s']" % remove_path) + self.aug.remove(remove_inc[0]) self.parser_paths.pop(remove_dirname) def _add_httpd_transform(self, incl): @@ -550,7 +551,12 @@ class ApacheParser(object): self.aug.set("/augeas/load/Httpd/lens", "Httpd.lns") self.aug.set("/augeas/load/Httpd/incl", incl) # Add included path to paths dictionary - self.parser_paths[os.path.dirname(incl)] = os.path.basename(incl) + try: + self.parser_paths[os.path.dirname(incl)].append( + os.path.basename(incl)) + except KeyError: + self.parser_paths[os.path.dirname(incl)] = [ + os.path.basename(incl)] def standardize_excl(self): """Standardize the excl arguments for the Httpd lens in Augeas. From 4d3d6ff03157fe9ec77ba60e2fd59cca78ee8ce9 Mon Sep 17 00:00:00 2001 From: watercrossing Date: Tue, 5 Jan 2016 16:51:34 +0000 Subject: [PATCH 054/127] Fix for listen bug --- .../letsencrypt_apache/configurator.py | 2 ++ .../tests/configurator_test.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 836d77135..ff12055ac 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -558,6 +558,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] + if port in listens: + return for listen in listens: # For any listen statement, check if the machine also listens on Port 443. # If not, add such a listen statement. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 218c085f9..9838b4f52 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -461,6 +461,25 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:8080", "https"]) self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + def test_prepare_server_https_mixed_listen(self): + + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:8080", "443"] + mock_add_dir = mock.Mock() + mock_enable = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir_to_ifmodssl = mock_add_dir + self.config.enable_mod = mock_enable + + # Test Listen statements with specific ip listeed + self.config.prepare_server_https("443") + # Should only be 2 here, as the third interface already listens to the correct port + self.assertEqual(mock_add_dir.call_count, 0) + def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) From f7fad9a4affc399a4c6953e330d59b20d44d9d2e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 23 Dec 2015 23:25:41 -0800 Subject: [PATCH 055/127] Show error detail to the user. Previously this code would show the user a hardcoded message based on the error type, but it's much more useful to show the error detail field, which often helps debug the problem. --- letsencrypt/auth_handler.py | 7 +------ letsencrypt/tests/auth_handler_test.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 027c11158..37b71775d 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -543,13 +543,8 @@ def _generate_failed_chall_msg(failed_achalls): msg = [ "The following '{0}' errors were reported by the server:".format(typ)] - problems = dict() for achall in failed_achalls: - problems.setdefault(achall.error.description, set()).add(achall.domain) - for problem in problems: - msg.append("\n\nDomains: ") - msg.append(", ".join(sorted(problems[problem]))) - msg.append("\nError: {0}".format(problem)) + msg.append("\n\nDomain: %s\nDetail: %s\n" % (achall.domain, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index be19ab036..199954278 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -467,7 +467,7 @@ class ReportFailedChallsTest(unittest.TestCase): auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) - self.assertTrue("Domains: example.com\n" in call_list[0][0][0]) + self.assertTrue("Domain: example.com\nDetail: detail" in call_list[0][0][0]) @mock.patch("letsencrypt.auth_handler.zope.component.getUtility") def test_different_errors_and_domains(self, mock_zope): From 4adc2826318168f8318a7784a5b190599a03a045 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 5 Jan 2016 11:02:03 -0800 Subject: [PATCH 056/127] Fix formatting of error messages. --- letsencrypt/auth_handler.py | 4 ++-- letsencrypt/tests/auth_handler_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 37b71775d..e77f83248 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -541,10 +541,10 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ msg = [ - "The following '{0}' errors were reported by the server:".format(typ)] + "The following errors were reported by the server:".format(typ)] for achall in failed_achalls: - msg.append("\n\nDomain: %s\nDetail: %s\n" % (achall.domain, achall.error.detail)) + msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % (achall.domain, achall.error.typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index 199954278..5b4c2bfc7 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -467,7 +467,7 @@ class ReportFailedChallsTest(unittest.TestCase): auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) - self.assertTrue("Domain: example.com\nDetail: detail" in call_list[0][0][0]) + self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @mock.patch("letsencrypt.auth_handler.zope.component.getUtility") def test_different_errors_and_domains(self, mock_zope): From 74237d101041897231881a5ba55142e7fb255a4d Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 12:00:20 -0500 Subject: [PATCH 057/127] Requires chain_path for nginx versions supporting OCSP stapling --chain-path config is not mandatory, so we require this property if nginx supports OCSP stapling. Alternatively, we could disable OCSP stapling on supported nginx versions if --chain-path is missing. --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 11 ++++++++++- .../letsencrypt_nginx/tests/configurator_test.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index aaaf43c5f..99a864141 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -122,7 +122,7 @@ class NginxConfigurator(common.Plugin): # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, - chain_path, fullchain_path): + chain_path=None, fullchain_path=None): # pylint: disable=unused-argument """Deploys certificate to specified virtual host. @@ -136,6 +136,9 @@ class NginxConfigurator(common.Plugin): .. note:: This doesn't save the config files! + :raises errors.PluginError: When unable to deploy certificate due to + a lack of directives or configuration + """ vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], @@ -150,6 +153,12 @@ class NginxConfigurator(common.Plugin): ['ssl_stapling', 'on'], ['ssl_stapling_verify', 'on']] + if len(stapling_directives) != 0 and not chain_path: + raise errors.PluginError( + "--chain-path is required to enable " + "Online Certificate Status Protocol (OCSP) stapling " + "on nginx >= 1.3.7.") + try: self.parser.add_server_directives(vhost.filep, vhost.names, cert_directives, replace=True) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..35a55befd 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -125,6 +125,15 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(util.contains_at_depth(generated_conf, ['ssl_trusted_certificate', 'example/chain.pem'], 2)) + def test_deploy_cert_stapling_requires_chain_path(self): + self.config.version = (1, 3, 7) + self.assertRaises(errors.PluginError, self.config.deploy_cert, + "www.example.com", + "example/cert.pem", + "example/key.pem", + None, + "example/fullchain.pem") + def test_deploy_cert(self): server_conf = self.config.parser.abs_path('server.conf') nginx_conf = self.config.parser.abs_path('nginx.conf') From e8fc2eca0139df1d8cde1e23c12eaca184371100 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 15:02:15 -0500 Subject: [PATCH 058/127] nginx plugin requires fullchain_path --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 4 ++++ .../letsencrypt_nginx/tests/configurator_test.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 99a864141..59a977f09 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -140,6 +140,10 @@ class NginxConfigurator(common.Plugin): a lack of directives or configuration """ + if not fullchain_path: + raise errors.PluginError( + "--fullchain-path is required for nginx plugin.") + vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], ['ssl_certificate_key', key_path]] diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 35a55befd..bab107582 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -134,6 +134,15 @@ class NginxConfiguratorTest(util.NginxTest): None, "example/fullchain.pem") + def test_deploy_cert_requires_fullchain_path(self): + self.config.version = (1, 3, 1) + self.assertRaises(errors.PluginError, self.config.deploy_cert, + "www.example.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + None) + def test_deploy_cert(self): server_conf = self.config.parser.abs_path('server.conf') nginx_conf = self.config.parser.abs_path('nginx.conf') From 5b5051b6ced180c10311cea89b37900d147929c1 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 15:16:30 -0500 Subject: [PATCH 059/127] The notes should display the fullchain_path See d01b17f1 and dd8c6d65 --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 59a977f09..963ae8a42 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -181,7 +181,7 @@ class NginxConfigurator(common.Plugin): self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, ", ".join(str(addr) for addr in vhost.addrs))) - self.save_notes += "\tssl_certificate %s\n" % cert_path + self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path ####################### From 0b9f505ed5beb7a861c9cb027b5e4349a67eb60d Mon Sep 17 00:00:00 2001 From: Fan Jiang Date: Mon, 4 Jan 2016 15:29:26 -0500 Subject: [PATCH 060/127] update document for --chain-path when required by Nginx >= 1.3.7 --- docs/using.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 5da13f02c..bd40dec02 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -237,7 +237,9 @@ The following files are available: server certificate, i.e. root and intermediate certificates only. This is what Apache < 2.4.8 needs for `SSLCertificateChainFile - `_. + `_, + and what nginx >= 1.3.7 needs for `ssl_trusted_certificate + `_. ``fullchain.pem`` All certificates, **including** server certificate. This is From cf74446b58fa1229c6b2ad6bf5465f3b9ebe41ef Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 23:04:18 -0500 Subject: [PATCH 061/127] Add test for redirect enhancement --- .../letsencrypt_nginx/tests/configurator_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 56ad5110c..5e237b04a 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -330,6 +330,17 @@ class NginxConfiguratorTest(util.NginxTest): OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_file.read()) + def test_redirect_enhance(self): + expected = [ + ['if', '($scheme != "https")'], + [['return', '301 https://$host$request_uri']] + ] + + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.enhance("www.example.com", "redirect") + + generated_conf = self.config.parser.parsed[example_conf] + self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) if __name__ == "__main__": unittest.main() # pragma: no cover From adcb7934aed5780ba96e9f5c861c467bff8f616d Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Mon, 4 Jan 2016 22:48:01 -0500 Subject: [PATCH 062/127] Improve choose_vhost() test by verifying the output file --- .../tests/configurator_test.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 5e237b04a..9a962a55f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -91,12 +91,26 @@ class NginxConfiguratorTest(util.NginxTest): 'test.www.example.com': foo_conf, 'abc.www.foo.com': foo_conf, 'www.bar.co.uk': localhost_conf} + + conf_path = {'localhost': "etc_nginx/nginx.conf", + 'alias': "etc_nginx/nginx.conf", + 'example.com': "etc_nginx/sites-enabled/example.com", + 'example.com.uk.test': "etc_nginx/sites-enabled/example.com", + 'www.example.com': "etc_nginx/sites-enabled/example.com", + 'test.www.example.com': "etc_nginx/foo.conf", + 'abc.www.foo.com': "etc_nginx/foo.conf", + 'www.bar.co.uk': "etc_nginx/nginx.conf"} + bad_results = ['www.foo.com', 'example', 't.www.bar.co', '69.255.225.155'] for name in results: - self.assertEqual(results[name], - self.config.choose_vhost(name).names) + vhost = self.config.choose_vhost(name) + path = os.path.relpath(vhost.filep, self.temp_dir) + + self.assertEqual(results[name], vhost.names) + self.assertEqual(conf_path[name], path) + for name in bad_results: self.assertEqual(set([name]), self.config.choose_vhost(name).names) From 20829e05ed6a665a224fdef43e4f91cd5c002199 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Tue, 5 Jan 2016 15:15:29 -0500 Subject: [PATCH 063/127] Add missing test condition for prepare() --- .../tests/configurator_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index 9a962a55f..3ad0b834f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -40,6 +40,23 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEquals((1, 6, 2), self.config.version) self.assertEquals(5, len(self.config.parser.parsed)) + @mock.patch("letsencrypt_nginx.configurator.le_util.exe_exists") + @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") + def test_prepare_initializes_version(self, mock_popen, mock_exe_exists): + mock_popen().communicate.return_value = ( + "", "\n".join(["nginx version: nginx/1.6.2", + "built by clang 6.0 (clang-600.0.56)" + " (based on LLVM 3.5svn)", + "TLS SNI support enabled", + "configure arguments: --prefix=/usr/local/Cellar/" + "nginx/1.6.2 --with-http_ssl_module"])) + + mock_exe_exists.return_value = True + + self.config.version = None + self.config.prepare() + self.assertEquals((1, 6, 2), self.config.version) + @mock.patch("letsencrypt_nginx.configurator.socket.gethostbyaddr") def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) From 7bd7e7ca23420439215e9fe07c9b9ec65f773c54 Mon Sep 17 00:00:00 2001 From: wteiken Date: Tue, 5 Jan 2016 19:51:45 -0500 Subject: [PATCH 064/127] Remove response argument from exception and fix eror messages. --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 3d84cb3bf..f3c2c6053 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -72,7 +72,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes new_authzr_uri = response.links['next']['url'] if new_authzr_uri is None: - raise errors.ClientError(response, 'missing "new_authrz_uri"') + raise errors.ClientError('"next" link missing') return messages.RegistrationResource( body=messages.Registration.from_json(response.json()), From 0a04efe40c55029ba66a5c8b69f0d3f646286aab Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 5 Jan 2016 17:23:32 -0800 Subject: [PATCH 065/127] Fix lint error and remove extra format() --- letsencrypt/auth_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index e77f83248..c63d8c8d4 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -540,11 +540,11 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ - msg = [ - "The following errors were reported by the server:".format(typ)] + msg = ["The following errors were reported by the server:"] for achall in failed_achalls: - msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % (achall.domain, achall.error.typ, achall.error.detail)) + msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( + achall.domain, achall.error.typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") From 6719d0d3804987914e6b53b6227d3ec0c7d4a010 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 12:40:44 -0500 Subject: [PATCH 066/127] Rewrote _pyopenssl_cert_or_req_san --- acme/acme/crypto_util.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 15890175f..ecec351c2 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -1,6 +1,7 @@ """Crypto utilities.""" import contextlib import logging +import re import socket import sys @@ -160,26 +161,22 @@ def _pyopenssl_cert_or_req_san(cert_or_req): """ # constants based on PyOpenSSL certificate/CSR text dump - label = "DNS" - parts_separator = ", " part_separator = ":" - prefix = label + part_separator - title = "X509v3 Subject Alternative Name:" + parts_separator = ", " + prefix = "DNS" + part_separator if isinstance(cert_or_req, OpenSSL.crypto.X509): func = OpenSSL.crypto.dump_certificate else: func = OpenSSL.crypto.dump_certificate_request - text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req) - - lines = iter(text.decode("utf-8").splitlines()) - sans = [next(lines).split(parts_separator) - for line in lines if title in line] + text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") + match = re.search(r"X509v3 Subject Alternative Name:\s*(.*)", text) + sans_parts = [] if match is None else match.group(1).split(parts_separator) # WARNING: this function assumes that no SAN can include # parts_separator, hence the split! - return [part.split(part_separator)[1] for parts in sans - for part in parts if part.lstrip().startswith(prefix)] + return [part.split(part_separator)[1] + for part in sans_parts if part.startswith(prefix)] def gen_ss_cert(key, domains, not_before=None, From 858dadd85b2b387f8e5c0485e370efa57bb7fe18 Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Wed, 6 Jan 2016 13:36:52 -0500 Subject: [PATCH 067/127] Update error message This is supposed to not happen once #1391 is fixed. --- letsencrypt-nginx/letsencrypt_nginx/configurator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 963ae8a42..43ad36c7f 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -142,7 +142,8 @@ class NginxConfigurator(common.Plugin): """ if not fullchain_path: raise errors.PluginError( - "--fullchain-path is required for nginx plugin.") + "The nginx plugin currently requires --fullchain-path to " + "install a cert.") vhost = self.choose_vhost(domain) cert_directives = [['ssl_certificate', fullchain_path], From 1af997158dbc6a3f2233e3c0a6ea7e7b9bc3fc7a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 13:39:14 -0500 Subject: [PATCH 068/127] Fix repr differences between PyOpenSSL versions --- acme/acme/jose/util_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py index 392f81777..0038a6cc1 100644 --- a/acme/acme/jose/util_test.py +++ b/acme/acme/jose/util_test.py @@ -44,8 +44,8 @@ class ComparableX509Test(unittest.TestCase): def test_repr(self): for x509 in self.req1, self.cert1: - self.assertTrue(repr(x509).startswith( - ''.format(x509.wrapped)) class ComparableRSAKeyTest(unittest.TestCase): From 32650a6d08803d6f07b4880a7a037e25bba1de96 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 14:10:57 -0500 Subject: [PATCH 069/127] Added 100 SANs cert and csr --- acme/acme/testdata/cert-100sans.pem | 44 +++++++++++++++++++++++++++++ acme/acme/testdata/csr-100sans.pem | 41 +++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 acme/acme/testdata/cert-100sans.pem create mode 100644 acme/acme/testdata/csr-100sans.pem diff --git a/acme/acme/testdata/cert-100sans.pem b/acme/acme/testdata/cert-100sans.pem new file mode 100644 index 000000000..3fdc9404f --- /dev/null +++ b/acme/acme/testdata/cert-100sans.pem @@ -0,0 +1,44 @@ +-----BEGIN CERTIFICATE----- +MIIHxDCCB26gAwIBAgIJAOGrG1Un9lHiMA0GCSqGSIb3DQEBCwUAMGQxCzAJBgNV +BAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMScwJQYDVQQLDB5FbGVjdHJv +bmljIEZyb250aWVyIEZvdW5kYXRpb24xFDASBgNVBAMMC2V4YW1wbGUuY29tMB4X +DTE2MDEwNjE5MDkzN1oXDTE2MDEwNzE5MDkzN1owZDELMAkGA1UECAwCQ0ExFjAU +BgNVBAcMDVNhbiBGcmFuY2lzY28xJzAlBgNVBAsMHkVsZWN0cm9uaWMgRnJvbnRp +ZXIgRm91bmRhdGlvbjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0B +AQEFAANLADBIAkEArHVztFHtH92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580 +rv2+6QWE30cWgdmJS86ObRz6lUTor4R0T+3C5QIDAQABo4IGATCCBf0wCQYDVR0T +BAIwADALBgNVHQ8EBAMCBeAwggXhBgNVHREEggXYMIIF1IIMZXhhbXBsZTEuY29t +ggxleGFtcGxlMi5jb22CDGV4YW1wbGUzLmNvbYIMZXhhbXBsZTQuY29tggxleGFt +cGxlNS5jb22CDGV4YW1wbGU2LmNvbYIMZXhhbXBsZTcuY29tggxleGFtcGxlOC5j +b22CDGV4YW1wbGU5LmNvbYINZXhhbXBsZTEwLmNvbYINZXhhbXBsZTExLmNvbYIN +ZXhhbXBsZTEyLmNvbYINZXhhbXBsZTEzLmNvbYINZXhhbXBsZTE0LmNvbYINZXhh +bXBsZTE1LmNvbYINZXhhbXBsZTE2LmNvbYINZXhhbXBsZTE3LmNvbYINZXhhbXBs +ZTE4LmNvbYINZXhhbXBsZTE5LmNvbYINZXhhbXBsZTIwLmNvbYINZXhhbXBsZTIx +LmNvbYINZXhhbXBsZTIyLmNvbYINZXhhbXBsZTIzLmNvbYINZXhhbXBsZTI0LmNv +bYINZXhhbXBsZTI1LmNvbYINZXhhbXBsZTI2LmNvbYINZXhhbXBsZTI3LmNvbYIN +ZXhhbXBsZTI4LmNvbYINZXhhbXBsZTI5LmNvbYINZXhhbXBsZTMwLmNvbYINZXhh +bXBsZTMxLmNvbYINZXhhbXBsZTMyLmNvbYINZXhhbXBsZTMzLmNvbYINZXhhbXBs +ZTM0LmNvbYINZXhhbXBsZTM1LmNvbYINZXhhbXBsZTM2LmNvbYINZXhhbXBsZTM3 +LmNvbYINZXhhbXBsZTM4LmNvbYINZXhhbXBsZTM5LmNvbYINZXhhbXBsZTQwLmNv +bYINZXhhbXBsZTQxLmNvbYINZXhhbXBsZTQyLmNvbYINZXhhbXBsZTQzLmNvbYIN +ZXhhbXBsZTQ0LmNvbYINZXhhbXBsZTQ1LmNvbYINZXhhbXBsZTQ2LmNvbYINZXhh +bXBsZTQ3LmNvbYINZXhhbXBsZTQ4LmNvbYINZXhhbXBsZTQ5LmNvbYINZXhhbXBs +ZTUwLmNvbYINZXhhbXBsZTUxLmNvbYINZXhhbXBsZTUyLmNvbYINZXhhbXBsZTUz +LmNvbYINZXhhbXBsZTU0LmNvbYINZXhhbXBsZTU1LmNvbYINZXhhbXBsZTU2LmNv +bYINZXhhbXBsZTU3LmNvbYINZXhhbXBsZTU4LmNvbYINZXhhbXBsZTU5LmNvbYIN +ZXhhbXBsZTYwLmNvbYINZXhhbXBsZTYxLmNvbYINZXhhbXBsZTYyLmNvbYINZXhh +bXBsZTYzLmNvbYINZXhhbXBsZTY0LmNvbYINZXhhbXBsZTY1LmNvbYINZXhhbXBs +ZTY2LmNvbYINZXhhbXBsZTY3LmNvbYINZXhhbXBsZTY4LmNvbYINZXhhbXBsZTY5 +LmNvbYINZXhhbXBsZTcwLmNvbYINZXhhbXBsZTcxLmNvbYINZXhhbXBsZTcyLmNv +bYINZXhhbXBsZTczLmNvbYINZXhhbXBsZTc0LmNvbYINZXhhbXBsZTc1LmNvbYIN +ZXhhbXBsZTc2LmNvbYINZXhhbXBsZTc3LmNvbYINZXhhbXBsZTc4LmNvbYINZXhh +bXBsZTc5LmNvbYINZXhhbXBsZTgwLmNvbYINZXhhbXBsZTgxLmNvbYINZXhhbXBs +ZTgyLmNvbYINZXhhbXBsZTgzLmNvbYINZXhhbXBsZTg0LmNvbYINZXhhbXBsZTg1 +LmNvbYINZXhhbXBsZTg2LmNvbYINZXhhbXBsZTg3LmNvbYINZXhhbXBsZTg4LmNv +bYINZXhhbXBsZTg5LmNvbYINZXhhbXBsZTkwLmNvbYINZXhhbXBsZTkxLmNvbYIN +ZXhhbXBsZTkyLmNvbYINZXhhbXBsZTkzLmNvbYINZXhhbXBsZTk0LmNvbYINZXhh +bXBsZTk1LmNvbYINZXhhbXBsZTk2LmNvbYINZXhhbXBsZTk3LmNvbYINZXhhbXBs +ZTk4LmNvbYINZXhhbXBsZTk5LmNvbYIOZXhhbXBsZTEwMC5jb20wDQYJKoZIhvcN +AQELBQADQQBEunJbKUXcyNKTSfA0pKRyWNiKmkoBqYgfZS6eHNrNH/hjFzHtzyDQ +XYHHK6kgEWBvHfRXGmqhFvht+b1tQKkG +-----END CERTIFICATE----- diff --git a/acme/acme/testdata/csr-100sans.pem b/acme/acme/testdata/csr-100sans.pem new file mode 100644 index 000000000..199814126 --- /dev/null +++ b/acme/acme/testdata/csr-100sans.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIHNTCCBt8CAQAwZDELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lz +Y28xJzAlBgNVBAsMHkVsZWN0cm9uaWMgRnJvbnRpZXIgRm91bmRhdGlvbjEUMBIG +A1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHt +H92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580rv2+6QWE30cWgdmJS86ObRz6 +lUTor4R0T+3C5QIDAQABoIIGFDCCBhAGCSqGSIb3DQEJDjGCBgEwggX9MAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgXgMIIF4QYDVR0RBIIF2DCCBdSCDGV4YW1wbGUxLmNv +bYIMZXhhbXBsZTIuY29tggxleGFtcGxlMy5jb22CDGV4YW1wbGU0LmNvbYIMZXhh +bXBsZTUuY29tggxleGFtcGxlNi5jb22CDGV4YW1wbGU3LmNvbYIMZXhhbXBsZTgu +Y29tggxleGFtcGxlOS5jb22CDWV4YW1wbGUxMC5jb22CDWV4YW1wbGUxMS5jb22C +DWV4YW1wbGUxMi5jb22CDWV4YW1wbGUxMy5jb22CDWV4YW1wbGUxNC5jb22CDWV4 +YW1wbGUxNS5jb22CDWV4YW1wbGUxNi5jb22CDWV4YW1wbGUxNy5jb22CDWV4YW1w +bGUxOC5jb22CDWV4YW1wbGUxOS5jb22CDWV4YW1wbGUyMC5jb22CDWV4YW1wbGUy +MS5jb22CDWV4YW1wbGUyMi5jb22CDWV4YW1wbGUyMy5jb22CDWV4YW1wbGUyNC5j +b22CDWV4YW1wbGUyNS5jb22CDWV4YW1wbGUyNi5jb22CDWV4YW1wbGUyNy5jb22C +DWV4YW1wbGUyOC5jb22CDWV4YW1wbGUyOS5jb22CDWV4YW1wbGUzMC5jb22CDWV4 +YW1wbGUzMS5jb22CDWV4YW1wbGUzMi5jb22CDWV4YW1wbGUzMy5jb22CDWV4YW1w +bGUzNC5jb22CDWV4YW1wbGUzNS5jb22CDWV4YW1wbGUzNi5jb22CDWV4YW1wbGUz +Ny5jb22CDWV4YW1wbGUzOC5jb22CDWV4YW1wbGUzOS5jb22CDWV4YW1wbGU0MC5j +b22CDWV4YW1wbGU0MS5jb22CDWV4YW1wbGU0Mi5jb22CDWV4YW1wbGU0My5jb22C +DWV4YW1wbGU0NC5jb22CDWV4YW1wbGU0NS5jb22CDWV4YW1wbGU0Ni5jb22CDWV4 +YW1wbGU0Ny5jb22CDWV4YW1wbGU0OC5jb22CDWV4YW1wbGU0OS5jb22CDWV4YW1w +bGU1MC5jb22CDWV4YW1wbGU1MS5jb22CDWV4YW1wbGU1Mi5jb22CDWV4YW1wbGU1 +My5jb22CDWV4YW1wbGU1NC5jb22CDWV4YW1wbGU1NS5jb22CDWV4YW1wbGU1Ni5j +b22CDWV4YW1wbGU1Ny5jb22CDWV4YW1wbGU1OC5jb22CDWV4YW1wbGU1OS5jb22C +DWV4YW1wbGU2MC5jb22CDWV4YW1wbGU2MS5jb22CDWV4YW1wbGU2Mi5jb22CDWV4 +YW1wbGU2My5jb22CDWV4YW1wbGU2NC5jb22CDWV4YW1wbGU2NS5jb22CDWV4YW1w +bGU2Ni5jb22CDWV4YW1wbGU2Ny5jb22CDWV4YW1wbGU2OC5jb22CDWV4YW1wbGU2 +OS5jb22CDWV4YW1wbGU3MC5jb22CDWV4YW1wbGU3MS5jb22CDWV4YW1wbGU3Mi5j +b22CDWV4YW1wbGU3My5jb22CDWV4YW1wbGU3NC5jb22CDWV4YW1wbGU3NS5jb22C +DWV4YW1wbGU3Ni5jb22CDWV4YW1wbGU3Ny5jb22CDWV4YW1wbGU3OC5jb22CDWV4 +YW1wbGU3OS5jb22CDWV4YW1wbGU4MC5jb22CDWV4YW1wbGU4MS5jb22CDWV4YW1w +bGU4Mi5jb22CDWV4YW1wbGU4My5jb22CDWV4YW1wbGU4NC5jb22CDWV4YW1wbGU4 +NS5jb22CDWV4YW1wbGU4Ni5jb22CDWV4YW1wbGU4Ny5jb22CDWV4YW1wbGU4OC5j +b22CDWV4YW1wbGU4OS5jb22CDWV4YW1wbGU5MC5jb22CDWV4YW1wbGU5MS5jb22C +DWV4YW1wbGU5Mi5jb22CDWV4YW1wbGU5My5jb22CDWV4YW1wbGU5NC5jb22CDWV4 +YW1wbGU5NS5jb22CDWV4YW1wbGU5Ni5jb22CDWV4YW1wbGU5Ny5jb22CDWV4YW1w +bGU5OC5jb22CDWV4YW1wbGU5OS5jb22CDmV4YW1wbGUxMDAuY29tMA0GCSqGSIb3 +DQEBCwUAA0EAW05UMFavHn2rkzMyUfzsOvWzVNlm43eO2yHu5h5TzDb23gkDnNEo +duUAbQ+CLJHYd+MvRCmPQ+3ZnaPy7l/0Hg== +-----END CERTIFICATE REQUEST----- From ba93c576977eafa754c41bb7c4410cc876143b4a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 14:22:13 -0500 Subject: [PATCH 070/127] Added large sans cert and csr test --- acme/acme/crypto_util_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index b3c39c388..e926fc317 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -82,6 +82,10 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self.assertEqual(self._call_cert('cert-san.pem'), ['example.com', 'www.example.com']) + def test_cert_hundred_sans(self): + self.assertEqual(self._call_cert('cert-100sans.pem'), + ['example{0}.com'.format(i) for i in range(1, 101)]) + def test_csr_no_sans(self): self.assertEqual(self._call_csr('csr-nosans.pem'), []) @@ -98,6 +102,10 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): "example.info", "subdomain.example.com", "other.subdomain.example.com"]) + def test_csr_hundred_sans(self): + self.assertEqual(self._call_csr('csr-100sans.pem'), + ['example{0}.com'.format(i) for i in range(1, 101)]) + if __name__ == "__main__": unittest.main() # pragma: no cover From 96114ba84e2a6177cca9533e5d5fb9fafa7fa4b1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 15:10:08 -0500 Subject: [PATCH 071/127] Add IDN SANs CSR and cert --- acme/acme/testdata/cert-idnsans.pem | 30 +++++++++++++++++++++++++++++ acme/acme/testdata/csr-idnsans.pem | 27 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 acme/acme/testdata/cert-idnsans.pem create mode 100644 acme/acme/testdata/csr-idnsans.pem diff --git a/acme/acme/testdata/cert-idnsans.pem b/acme/acme/testdata/cert-idnsans.pem new file mode 100644 index 000000000..932649692 --- /dev/null +++ b/acme/acme/testdata/cert-idnsans.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFNjCCBOCgAwIBAgIJAP4rNqqOKifCMA0GCSqGSIb3DQEBCwUAMGQxCzAJBgNV +BAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMScwJQYDVQQLDB5FbGVjdHJv +bmljIEZyb250aWVyIEZvdW5kYXRpb24xFDASBgNVBAMMC2V4YW1wbGUuY29tMB4X +DTE2MDEwNjIwMDg1OFoXDTE2MDEwNzIwMDg1OFowZDELMAkGA1UECAwCQ0ExFjAU +BgNVBAcMDVNhbiBGcmFuY2lzY28xJzAlBgNVBAsMHkVsZWN0cm9uaWMgRnJvbnRp +ZXIgRm91bmRhdGlvbjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0B +AQEFAANLADBIAkEArHVztFHtH92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580 +rv2+6QWE30cWgdmJS86ObRz6lUTor4R0T+3C5QIDAQABo4IDczCCA28wCQYDVR0T +BAIwADALBgNVHQ8EBAMCBeAwggNTBgNVHREEggNKMIIDRoJiz4PPhM+Fz4bPh8+I +z4nPis+Lz4zPjc+Oz4/PkM+Rz5LPk8+Uz5XPls+Xz5jPmc+az5vPnM+dz57Pn8+g +z6HPos+jz6TPpc+mz6fPqM+pz6rPq8+sz63Prs+vLmludmFsaWSCYs+wz7HPss+z +z7TPtc+2z7fPuM+5z7rPu8+8z73Pvs+/2YHZgtmD2YTZhdmG2YfZiNmJ2YrZi9mM +2Y3ZjtmP2ZDZkdmS2ZPZlNmV2ZbZl9mY2ZnZmtmb2ZzZnS5pbnZhbGlkgmLZntmf +2aDZodmi2aPZpNml2abZp9mo2anZqtmr2azZrdmu2a/ZsNmx2bLZs9m02bXZttm3 +2bjZudm62bvZvNm92b7Zv9qA2oHagtqD2oTahdqG2ofaiNqJ2oouaW52YWxpZIJi +2ovajNqN2o7aj9qQ2pHaktqT2pTaldqW2pfamNqZ2pram9qc2p3antqf2qDaodqi +2qPapNql2qbap9qo2qnaqtqr2qzardqu2q/asNqx2rLas9q02rXattq3LmludmFs +aWSCYtq42rnautq72rzavdq+2r/bgNuB24Lbg9uE24XbhtuH24jbiduK24vbjNuN +247bj9uQ25HbktuT25TblduW25fbmNuZ25rbm9uc253bntuf26Dbodui26PbpC5p +bnZhbGlkgnjbpdum26fbqNup26rbq9us263brtuv27Dbsduy27PbtNu127bbt9u4 +27nbutu74aCg4aCh4aCi4aCj4aCk4aCl4aCm4aCn4aCo4aCp4aCq4aCr4aCs4aCt +4aCu4aCv4aCw4aCx4aCy4aCz4aC04aC1LmludmFsaWSCgY/hoLbhoLfhoLjhoLnh +oLrhoLvhoLzhoL3hoL7hoL/hoYDhoYHhoYLhoYPhoYThoYXhoYbhoYfhoYjhoYnh +oYrhoYvhoYzhoY3hoY7hoY/hoZDhoZHhoZLhoZPhoZThoZXhoZbhoZfhoZjhoZnh +oZrhoZvhoZzhoZ3hoZ7hoZ/hoaDhoaHhoaIuaW52YWxpZIJE4aGj4aGk4aGl4aGm +4aGn4aGo4aGp4aGq4aGr4aGs4aGt4aGu4aGv4aGw4aGx4aGy4aGz4aG04aG14aG2 +LmludmFsaWQwDQYJKoZIhvcNAQELBQADQQAzOQL/54yXxln87/YvEQbBm9ik9zoT +TxEkvnZ4kmTRhDsUPtRjMXhY2FH7LOtXKnJQ7POUB7AsJ2Z6uq2w623G +-----END CERTIFICATE----- diff --git a/acme/acme/testdata/csr-idnsans.pem b/acme/acme/testdata/csr-idnsans.pem new file mode 100644 index 000000000..d6e91a420 --- /dev/null +++ b/acme/acme/testdata/csr-idnsans.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEpzCCBFECAQAwZDELMAkGA1UECAwCQ0ExFjAUBgNVBAcMDVNhbiBGcmFuY2lz +Y28xJzAlBgNVBAsMHkVsZWN0cm9uaWMgRnJvbnRpZXIgRm91bmRhdGlvbjEUMBIG +A1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHt +H92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580rv2+6QWE30cWgdmJS86ObRz6 +lUTor4R0T+3C5QIDAQABoIIDhjCCA4IGCSqGSIb3DQEJDjGCA3MwggNvMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgXgMIIDUwYDVR0RBIIDSjCCA0aCYs+Dz4TPhc+Gz4fP +iM+Jz4rPi8+Mz43Pjs+Pz5DPkc+Sz5PPlM+Vz5bPl8+Yz5nPms+bz5zPnc+ez5/P +oM+hz6LPo8+kz6XPps+nz6jPqc+qz6vPrM+tz67Pry5pbnZhbGlkgmLPsM+xz7LP +s8+0z7XPts+3z7jPuc+6z7vPvM+9z77Pv9mB2YLZg9mE2YXZhtmH2YjZidmK2YvZ +jNmN2Y7Zj9mQ2ZHZktmT2ZTZldmW2ZfZmNmZ2ZrZm9mc2Z0uaW52YWxpZIJi2Z7Z +n9mg2aHZotmj2aTZpdmm2afZqNmp2arZq9ms2a3Zrtmv2bDZsdmy2bPZtNm12bbZ +t9m42bnZutm72bzZvdm+2b/agNqB2oLag9qE2oXahtqH2ojaidqKLmludmFsaWSC +YtqL2ozajdqO2o/akNqR2pLak9qU2pXaltqX2pjamdqa2pvanNqd2p7an9qg2qHa +otqj2qTapdqm2qfaqNqp2qraq9qs2q3artqv2rDasdqy2rPatNq12rbaty5pbnZh +bGlkgmLauNq52rrau9q82r3avtq/24DbgduC24PbhNuF24bbh9uI24nbituL24zb +jduO24/bkNuR25Lbk9uU25XbltuX25jbmdua25vbnNud257bn9ug26Hbotuj26Qu +aW52YWxpZIJ426Xbptun26jbqduq26vbrNut267br9uw27Hbstuz27Tbtdu227fb +uNu527rbu+GgoOGgoeGgouGgo+GgpOGgpeGgpuGgp+GgqOGgqeGgquGgq+GgrOGg +reGgruGgr+GgsOGgseGgsuGgs+GgtOGgtS5pbnZhbGlkgoGP4aC24aC34aC44aC5 +4aC64aC74aC84aC94aC+4aC/4aGA4aGB4aGC4aGD4aGE4aGF4aGG4aGH4aGI4aGJ +4aGK4aGL4aGM4aGN4aGO4aGP4aGQ4aGR4aGS4aGT4aGU4aGV4aGW4aGX4aGY4aGZ +4aGa4aGb4aGc4aGd4aGe4aGf4aGg4aGh4aGiLmludmFsaWSCROGho+GhpOGhpeGh +puGhp+GhqOGhqeGhquGhq+GhrOGhreGhruGhr+GhsOGhseGhsuGhs+GhtOGhteGh +ti5pbnZhbGlkMA0GCSqGSIb3DQEBCwUAA0EAeNkY0M0+kMnjRo6dEUoGE4dX9fEr +dfGrpPUBcwG0P5QBdZJWvZxTfRl14yuPYHbGHULXeGqRdkU6HK5pOlzpng== +-----END CERTIFICATE REQUEST----- From 9dc4af5cee5445e1a8bdfab97d56ecbceea6d659 Mon Sep 17 00:00:00 2001 From: Ben Ubois Date: Wed, 6 Jan 2016 12:10:21 -0800 Subject: [PATCH 072/127] Document webroot request path. It's handy to know the implementation details of the webroot plugin so that a server can be configured to properly the ACME challenge files. --- docs/using.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 5da13f02c..f7a59cf7c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -139,9 +139,20 @@ Would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and ``/var/www/eg`` for the second two. +The webroot plugin works by creating a temporary file for each of your requested +domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's +Encrypt validation server makes HTTP requests to validate that the DNS for each +requested domain resolves to the server running letsencrypt. An example request +made to your web server would look like: + +:: + + 66.133.109.36 - - [05/Jan/2016:20:11:24 -0500] "GET /.well-known/acme-challenge/HGr8U1IeTW4kY_Z6UIyaakzOkyQgPr_7ArlLgtZE8SX HTTP/1.1" 200 87 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" + Note that to use the webroot plugin, your server must be configured to serve files from hidden directories. + Manual ------ From 1cdff156c90e075c888829f928fab644d46b98f9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 15:33:36 -0500 Subject: [PATCH 073/127] Add IDN test --- acme/acme/crypto_util_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index e926fc317..0f3c9225b 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -1,9 +1,11 @@ """Tests for acme.crypto_util.""" +import itertools import socket import threading import time import unittest +import six from six.moves import socketserver # pylint: disable=import-error from acme import errors @@ -69,6 +71,14 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): from acme.crypto_util import _pyopenssl_cert_or_req_san return _pyopenssl_cert_or_req_san(loader(name)) + @classmethod + def _get_idn_names(cls): + chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400), + range(0x641, 0x6fc), + range(0x1820, 0x1877))] + return [''.join(chars[i: i + 45]) + '.invalid' + for i in range(0, len(chars), 45)] + def _call_cert(self, name): return self._call(test_util.load_cert, name) @@ -86,6 +96,10 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self.assertEqual(self._call_cert('cert-100sans.pem'), ['example{0}.com'.format(i) for i in range(1, 101)]) + def test_cert_idn_sans(self): + self.assertEqual(self._call_cert('cert-idnsans.pem'), + self._get_idn_names()) + def test_csr_no_sans(self): self.assertEqual(self._call_csr('csr-nosans.pem'), []) @@ -106,6 +120,10 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self.assertEqual(self._call_csr('csr-100sans.pem'), ['example{0}.com'.format(i) for i in range(1, 101)]) + def test_csr(self): + self.assertEqual(self._call_csr('csr-idnsans.pem'), + self._get_idn_names()) + if __name__ == "__main__": unittest.main() # pragma: no cover From 51bc1311a219b81feba302dc8a7e1a9756201bf8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 15:34:42 -0500 Subject: [PATCH 074/127] Fixed rogue quotes --- acme/acme/crypto_util_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 0f3c9225b..446c19eec 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -112,9 +112,9 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): def test_csr_six_sans(self): self.assertEqual(self._call_csr('csr-6sans.pem'), - ["example.com", "example.org", "example.net", - "example.info", "subdomain.example.com", - "other.subdomain.example.com"]) + ['example.com', 'example.org', 'example.net', + 'example.info', 'subdomain.example.com', + 'other.subdomain.example.com']) def test_csr_hundred_sans(self): self.assertEqual(self._call_csr('csr-100sans.pem'), @@ -125,5 +125,5 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self._get_idn_names()) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() # pragma: no cover From 2f569f77836d1628a1cf04afe41fc688c0aae516 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 21:40:26 -0500 Subject: [PATCH 075/127] Tox fanciness --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index dbd6d51fa..c1d23b69f 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ # acme and letsencrypt are not yet on pypi, so when Tox invokes # "install *.zip", it will not find deps skipsdist = true -envlist = py26,py27,py33,py34,py35,cover,lint +envlist = py{26,27,33,34,35},cover,lint # nosetest -v => more verbose output, allows to detect busy waiting # loops, especially on Travis From 90f0b15c9d8f4b34d6fcbaa599446fffd0f53dba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 21:51:42 -0500 Subject: [PATCH 076/127] Add old dependency test --- tox.ini | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c1d23b69f..e2d7900e0 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ # acme and letsencrypt are not yet on pypi, so when Tox invokes # "install *.zip", it will not find deps skipsdist = true -envlist = py{26,27,33,34,35},cover,lint +envlist = py{26,27,33,34,35},py{26,27}-oldest,cover,lint # nosetest -v => more verbose output, allows to detect busy waiting # loops, especially on Travis @@ -31,6 +31,15 @@ setenv = PYTHONHASHSEED = 0 # https://testrun.org/tox/latest/example/basic.html#special-handling-of-pythonhas + +deps = + py{26,27}-oldest: cryptography==0.8 + py{26,27}-oldest: configargparse==0.10.0 + py{26,27}-oldest: psutil==2.1.0 + py{26,27}-oldest: PyOpenSSL==0.13 + py{26,27}-oldest: pyparsing==1.5.5 + py{26,27}-oldest: python2-pythondialog==3.2.2rc1 + [testenv:py33] commands = pip install -e acme[testing] From 07b3a9fd955121d1ea976bab03d2d6fd7f7ad42d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 21:52:14 -0500 Subject: [PATCH 077/127] Add old dependency tests to Travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a5d6d8a85..2532216f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=py26-oldest + - TOXENV=py27-oldest - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration From 8a1931e23c015f40cd21c5b003ed56d5c1550bec Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 21:56:52 -0500 Subject: [PATCH 078/127] Run integration tests with old deps --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2532216f1..c3e2e92aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,8 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=py26-oldest - - TOXENV=py27-oldest + - TOXENV=py26-oldest BOULDER_INTEGRATION=1 + - TOXENV=py27-oldest BOULDER_INTEGRATION=1 - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration From 94508b00dfbcd1265967ee23c13f0dda944b9d30 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 21:57:15 -0500 Subject: [PATCH 079/127] Don't pin pyparsing version --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index e2d7900e0..f91285bec 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,6 @@ deps = py{26,27}-oldest: configargparse==0.10.0 py{26,27}-oldest: psutil==2.1.0 py{26,27}-oldest: PyOpenSSL==0.13 - py{26,27}-oldest: pyparsing==1.5.5 py{26,27}-oldest: python2-pythondialog==3.2.2rc1 [testenv:py33] From 0b1e1d0937da0087f252807e72fba27848ed1cda Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 Jan 2016 22:07:45 -0500 Subject: [PATCH 080/127] Use test_util.load_cert --- acme/acme/challenges_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 6b277ac27..4f2d06167 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -264,9 +264,7 @@ class TLSSNI01ResponseTest(unittest.TestCase): def test_verify_bad_cert(self): self.assertFalse(self.response.verify_cert( - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, - test_util.load_vector('cert.pem')))) + test_util.load_cert('cert.pem'))) def test_simple_verify_bad_key_authorization(self): key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) From a815ddbafd2856700b9f30ea71b594ef8961da9e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 10:05:33 -0500 Subject: [PATCH 081/127] Remove excessive newline --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index f91285bec..45c7c43d2 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,6 @@ setenv = PYTHONHASHSEED = 0 # https://testrun.org/tox/latest/example/basic.html#special-handling-of-pythonhas - deps = py{26,27}-oldest: cryptography==0.8 py{26,27}-oldest: configargparse==0.10.0 From e7da21eec0cadfe80f8836d44f1aa64cd89130de Mon Sep 17 00:00:00 2001 From: Reinaldo de Souza Jr Date: Thu, 7 Jan 2016 11:27:40 -0500 Subject: [PATCH 082/127] Makes NginxParser aware of directive Fixes #2059 --- letsencrypt-nginx/letsencrypt_nginx/parser.py | 6 ++++-- .../letsencrypt_nginx/tests/parser_test.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/parser.py b/letsencrypt-nginx/letsencrypt_nginx/parser.py index c60d0102a..3b1dd049e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/parser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/parser.py @@ -113,7 +113,7 @@ class NginxParser(object): for filename in servers: for server in servers[filename]: # Parse the server block into a VirtualHost object - parsed_server = _parse_server(server) + parsed_server = parse_server(server) vhost = obj.VirtualHost(filename, parsed_server['addrs'], parsed_server['ssl'], @@ -451,7 +451,7 @@ def _get_servernames(names): return names.split(' ') -def _parse_server(server): +def parse_server(server): """Parses a list of server directives. :param list server: list of directives in a server block @@ -471,6 +471,8 @@ def _parse_server(server): elif directive[0] == 'server_name': parsed_server['names'].update( _get_servernames(directive[1])) + elif directive[0] == 'ssl' and directive[1] == 'on': + parsed_server['ssl'] = True return parsed_server diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py index 6559a5df6..b64f1dee3 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/parser_test.py @@ -228,6 +228,26 @@ class NginxParserTest(util.NginxTest): c_k = nparser.get_all_certs_keys() self.assertEqual(set([('foo.pem', 'bar.key', filep)]), c_k) + def test_parse_server_ssl(self): + server = parser.parse_server([ + ['listen', '443'] + ]) + self.assertFalse(server['ssl']) + + server = parser.parse_server([ + ['listen', '443 ssl'] + ]) + self.assertTrue(server['ssl']) + + server = parser.parse_server([ + ['listen', '443'], ['ssl', 'off'] + ]) + self.assertFalse(server['ssl']) + + server = parser.parse_server([ + ['listen', '443'], ['ssl', 'on'] + ]) + self.assertTrue(server['ssl']) if __name__ == "__main__": unittest.main() # pragma: no cover From bbf25a2c5edf66acc8df06708cab3181a7928b4f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 08:57:13 -0800 Subject: [PATCH 083/127] Setenvif may be required for conf tests on ubuntu 12.04? --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index 4e0443bb7..0afddbb33 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -49,7 +49,7 @@ if [ "$1" = --debian-modules ] ; then sudo apt-get install -y libapache2-mod-wsgi sudo apt-get install -y libapache2-mod-macro - for mod in ssl rewrite macro wsgi deflate userdir version mime ; do + for mod in ssl rewrite macro wsgi deflate userdir version mime setenvif ; do sudo a2enmod $mod done fi From 4bdd96a29e98cd0aa5b29c1565c986d7b1a8f8e3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 08:59:53 -0800 Subject: [PATCH 084/127] Verbosity --- .../letsencrypt_apache/tests/apache-conf-files/apache-conf-test | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test index 0afddbb33..7b3f83d13 100755 --- a/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test +++ b/letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test @@ -50,6 +50,7 @@ if [ "$1" = --debian-modules ] ; then sudo apt-get install -y libapache2-mod-macro for mod in ssl rewrite macro wsgi deflate userdir version mime setenvif ; do + echo -n enabling $mod sudo a2enmod $mod done fi From dc7f479fe3ac34388da208f97abfff4af98eaef0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 09:44:13 -0800 Subject: [PATCH 085/127] [deb bootstrap] Add precise-backports for libaugeas0 --- bootstrap/_deb_common.sh | 44 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 8b96fe6f1..180caf1a1 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -35,25 +35,39 @@ fi augeas_pkg=libaugeas0 AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` +AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$0" + BACKPORT_SOURCELINE="$1" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list + apt-get update + fi + fi + +} + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then if lsb_release -a | grep -q wheezy ; then - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then - /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." - sleep 1s - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - - echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list - apt-get update - fi - fi + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 augeas_pkg= + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" echo "Let's Encrypt apache plugin..." From 491a08e4d47201b8b30891aa9da636c403664776 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 09:46:45 -0800 Subject: [PATCH 086/127] fixen --- bootstrap/_deb_common.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 180caf1a1..410f4e10e 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -37,16 +37,16 @@ AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut - AddBackportRepo() { # ARGS: - BACKPORT_NAME="$0" - BACKPORT_SOURCELINE="$1" + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..." + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..." + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..." + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." sleep 1s if echo $BACKPORT_NAME | grep -q wheezy ; then /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' From 154cd47c83cf5753354563ceb0c92febb2c868ce Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 09:52:10 -0800 Subject: [PATCH 087/127] precision --- bootstrap/_deb_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 410f4e10e..72f8e85c8 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -67,7 +67,7 @@ if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then augeas_pkg= elif lsb_release -a | grep -q precise ; then # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" echo "Let's Encrypt apache plugin..." From caf9b1f26111d0a51212aa3b5b70d13d7347b547 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 13:00:29 -0500 Subject: [PATCH 088/127] Clarify _get_idn_sans method --- acme/acme/crypto_util_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 446c19eec..8b510e43e 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -73,6 +73,7 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): @classmethod def _get_idn_names(cls): + """Returns expected names from '{cert,csr}-idnsans.pem'.""" chars = [six.unichr(i) for i in itertools.chain(range(0x3c3, 0x400), range(0x641, 0x6fc), range(0x1820, 0x1877))] From 0f239e0029e77f302401c3c6089e67f45c19d51c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 13:04:29 -0500 Subject: [PATCH 089/127] Add comment about dependency version --- acme/setup.py | 1 + letsencrypt-apache/setup.py | 1 + letsencrypt-nginx/setup.py | 1 + setup.py | 1 + 4 files changed, 4 insertions(+) diff --git a/acme/setup.py b/acme/setup.py index 3b1a42327..4993d7584 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -6,6 +6,7 @@ from setuptools import find_packages version = '0.2.0.dev0' +# Please update tox.ini when modifying dependency version requirements install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 58008e1e4..a5c5e8a7a 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -6,6 +6,7 @@ from setuptools import find_packages version = '0.2.0.dev0' +# Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 1d42fe488..bfb3c3758 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -6,6 +6,7 @@ from setuptools import find_packages version = '0.2.0.dev0' +# Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), 'letsencrypt=={0}'.format(version), diff --git a/setup.py b/setup.py index f95f672ff..ad7fb6909 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ readme = read_file(os.path.join(here, 'README.rst')) changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] +# Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), 'configobj', From 99616864766e7cf8acbf8cda7625153159c56738 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 10:09:20 -0800 Subject: [PATCH 090/127] We might need -t afterall --- bootstrap/_deb_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 72f8e85c8..2b6ee11be 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -54,6 +54,8 @@ AddBackportRepo() { echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list apt-get update + apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" libaugeas0 + augeas_pkg= fi fi @@ -63,8 +65,6 @@ AddBackportRepo() { if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then if lsb_release -a | grep -q wheezy ; then AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0 - augeas_pkg= elif lsb_release -a | grep -q precise ; then # XXX add ARM case AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" From 710eb59f41e9ba0f258bc9b6d74761a6158513ac Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 16:19:21 -0500 Subject: [PATCH 091/127] Fix IDN CSR test name --- acme/acme/crypto_util_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 8b510e43e..147cd5a2a 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -121,7 +121,7 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self.assertEqual(self._call_csr('csr-100sans.pem'), ['example{0}.com'.format(i) for i in range(1, 101)]) - def test_csr(self): + def test_csr_idn_sans(self): self.assertEqual(self._call_csr('csr-idnsans.pem'), self._get_idn_names()) From 6548f343bfe8b9fc3e00e1ef2abe356ff4c3c13c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 21:20:14 +0000 Subject: [PATCH 092/127] Add invalidEmail error type to acme Related to: - #1923 - https://github.com/ietf-wg-acme/acme/pull/65 --- acme/acme/messages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0b73864ec..3b5739da1 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -25,6 +25,8 @@ class Error(jose.JSONObjectWithFields, errors.Error): ('connection', 'The server could not connect to the client to ' 'verify the domain'), ('dnssec', 'The server could not validate a DNSSEC signed domain'), + ('invalidEmail', + 'The provided email for a registration was invalid'), ('malformed', 'The request message was malformed'), ('rateLimited', 'There were too many requests of a given type'), ('serverInternal', 'The server experienced an internal error'), From 32957cc5eccc9dc6d61f3c02327edabc860f5d31 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 16:25:23 -0500 Subject: [PATCH 093/127] Comment _pyopenssl_cert_or_req_san method --- acme/acme/crypto_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index ecec351c2..0d0e78df6 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -169,11 +169,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): func = OpenSSL.crypto.dump_certificate else: func = OpenSSL.crypto.dump_certificate_request + + # This method of finding SANs is used to support PyOpenSSL version 0.13. text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") match = re.search(r"X509v3 Subject Alternative Name:\s*(.*)", text) - sans_parts = [] if match is None else match.group(1).split(parts_separator) + # WARNING: this function assumes that no SAN can include # parts_separator, hence the split! + sans_parts = [] if match is None else match.group(1).split(parts_separator) return [part.split(part_separator)[1] for part in sans_parts if part.startswith(prefix)] From 946f4474da7be9c784b732224f395ab2c22ed621 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 16:45:46 -0500 Subject: [PATCH 094/127] Add warning about multiple SANs extensions --- acme/acme/crypto_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 0d0e78df6..a25819cf5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -169,11 +169,11 @@ def _pyopenssl_cert_or_req_san(cert_or_req): func = OpenSSL.crypto.dump_certificate else: func = OpenSSL.crypto.dump_certificate_request - # This method of finding SANs is used to support PyOpenSSL version 0.13. text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") + # WARNING: this function does not support multiple SANs extensions. + # Multiple X509v3 extensions of the same type is disallowed by RFC 5280. match = re.search(r"X509v3 Subject Alternative Name:\s*(.*)", text) - # WARNING: this function assumes that no SAN can include # parts_separator, hence the split! sans_parts = [] if match is None else match.group(1).split(parts_separator) From cc168c8ef13bb2d7a53c3ce488806632d4f2ca31 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 22:17:40 +0000 Subject: [PATCH 095/127] Generate fresh pylintrc pylint 1.4.2, -generate-rcfile --- acme/.pylintrc | 380 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 acme/.pylintrc diff --git a/acme/.pylintrc b/acme/.pylintrc new file mode 100644 index 000000000..a31ace48d --- /dev/null +++ b/acme/.pylintrc @@ -0,0 +1,380 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# DEPRECATED +include-ids=no + +# DEPRECATED +symbols=no + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception From dba69d079f734f13437f1845f0008d1f1e81f6fa Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 7 Jan 2016 22:18:21 +0000 Subject: [PATCH 096/127] Separate pylintrc for acme --- acme/.pylintrc | 23 +++++++++++++---------- acme/acme/challenges.py | 2 +- acme/acme/crypto_util.py | 2 +- acme/acme/messages.py | 4 ++-- tox.ini | 2 +- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/acme/.pylintrc b/acme/.pylintrc index a31ace48d..33650310d 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -19,7 +19,7 @@ persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=linter_plugin # DEPRECATED include-ids=no @@ -60,7 +60,10 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636 +disable=fixme,locally-disabled,abstract-class-not-used +# bstract-class-not-used cannot be disabled locally (at least in +# pylint 1.4.1/2) + [REPORTS] @@ -133,7 +136,7 @@ notes=FIXME,XXX,TODO # Logging modules to check that the string format arguments are in logging # function parameter format -logging-modules=logging +logging-modules=logging,logger [SPELLING] @@ -200,7 +203,7 @@ init-import=no # A regular expression matching the name of dummy variables (i.e. expectedly # not used). -dummy-variables-rgx=_$|dummy +dummy-variables-rgx=_$|dummy|unused # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. @@ -220,7 +223,7 @@ required-attributes= bad-functions=map,filter,input # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +good-names=i,j,k,ex,Run,_,logger # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata @@ -233,7 +236,7 @@ name-group= include-naming-hint=no # Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ +function-rgx=[a-z_][a-z0-9_]{2,40}$ # Naming hint for function names function-name-hint=[a-z_][a-z0-9_]{2,30}$ @@ -287,14 +290,14 @@ module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ +method-rgx=[a-z_][a-z0-9_]{2,49}$ # Naming hint for method names method-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match function or class names that do # not require a docstring. -no-docstring-rgx=__.*__ +no-docstring-rgx=__.*__|test_[A-Za-z0-9_]*|_.*|.*Test # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. @@ -324,7 +327,7 @@ exclude-protected=_asdict,_fields,_replace,_source,_make [DESIGN] # Maximum number of arguments for function / method -max-args=5 +max-args=6 # Argument names that match this expression will be ignored. Default to name # with leading underscore @@ -343,7 +346,7 @@ max-branches=12 max-statements=50 # Maximum number of parents for a class (see R0901). -max-parents=7 +max-parents=12 # Maximum number of attributes for a class (see R0902). max-attributes=7 diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 1e456d325..fc414d93b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -336,7 +336,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ @property - def z(self): + def z(self): # pylint: disable=invalid-name """``z`` value used for verification. :rtype bytes: diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 72a93141a..757f7b5a1 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -70,7 +70,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" - # pylint: disable=missing-docstring + # pylint: disable=too-few-public-methods,missing-docstring def __init__(self, connection): self._wrapped = connection diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0b73864ec..6ef00399d 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -23,13 +23,13 @@ class Error(jose.JSONObjectWithFields, errors.Error): ('badCSR', 'The CSR is unacceptable (e.g., due to a short key)'), ('badNonce', 'The client sent an unacceptable anti-replay nonce'), ('connection', 'The server could not connect to the client to ' - 'verify the domain'), + 'verify the domain'), ('dnssec', 'The server could not validate a DNSSEC signed domain'), ('malformed', 'The request message was malformed'), ('rateLimited', 'There were too many requests of a given type'), ('serverInternal', 'The server experienced an internal error'), ('tls', 'The server experienced a TLS error during domain ' - 'verification'), + 'verification'), ('unauthorized', 'The client lacks sufficient authorization'), ('unknownHost', 'The server could not resolve a domain name'), ) diff --git a/tox.ini b/tox.ini index dbd6d51fa..36e3632c0 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,7 @@ commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt ./pep8.travis.sh pylint --rcfile=.pylintrc letsencrypt - pylint --rcfile=.pylintrc acme/acme + pylint --rcfile=acme/.pylintrc acme/acme pylint --rcfile=.pylintrc letsencrypt-apache/letsencrypt_apache pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx pylint --rcfile=.pylintrc letsencrypt-compatibility-test/letsencrypt_compatibility_test From 639cbeb7d02801b79429f17f9cd55dad09e962f6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 7 Jan 2016 21:11:09 -0500 Subject: [PATCH 097/127] sans_text_dump_comment += 1 --- acme/acme/crypto_util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a25819cf5..a1a1be73b 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -160,6 +160,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req): :rtype: `list` of `unicode` """ + # This function finds SANs by dumping the certificate/CSR to text and + # searching for "X509v3 Subject Alternative Name" in the text. This method + # is used to support PyOpenSSL version 0.13 where the + # `_subjectAltNameString` and `get_extensions` methods are not available + # for CSRs. + # constants based on PyOpenSSL certificate/CSR text dump part_separator = ":" parts_separator = ", " @@ -169,7 +175,6 @@ def _pyopenssl_cert_or_req_san(cert_or_req): func = OpenSSL.crypto.dump_certificate else: func = OpenSSL.crypto.dump_certificate_request - # This method of finding SANs is used to support PyOpenSSL version 0.13. text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") # WARNING: this function does not support multiple SANs extensions. # Multiple X509v3 extensions of the same type is disallowed by RFC 5280. From fc3acc69b69da9875c054c2e1ac602462a2ab533 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 7 Jan 2016 19:12:19 -0800 Subject: [PATCH 098/127] Try updating augeas-lenses as well --- bootstrap/_deb_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 2b6ee11be..84ab9e35a 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -54,7 +54,7 @@ AddBackportRepo() { echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list apt-get update - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" libaugeas0 + apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" libaugeas0 augeas-lenses augeas_pkg= fi fi From 6fedd22dc8c2db00665fa4343779f4aa801d571e Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 02:00:47 -0800 Subject: [PATCH 099/127] don't iDisplay if logging --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt/reverter.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 1baa06128..3a0b90fc0 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1271,7 +1271,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self.config_test() - logger.debug(self.reverter.view_config_changes()) + logger.debug(self.reverter.view_config_changes(logging=True)) self._reload() def _reload(self): diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index d5114ae71..e2bc28cb9 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -94,7 +94,7 @@ class Reverter(object): "Unable to load checkpoint during rollback") rollback -= 1 - def view_config_changes(self): + def view_config_changes(self, logging=False): """Displays all saved checkpoints. All checkpoints are printed by @@ -144,6 +144,8 @@ class Reverter(object): output.append(os.linesep) + if logging: + return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( os.linesep.join(output), display_util.HEIGHT) From 287be6be8eeac0e4da55c6088bda74ec3d48a8d7 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 02:47:59 -0800 Subject: [PATCH 100/127] fix linting issue --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt/reverter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index d936309ae..2a9fb0250 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1302,7 +1302,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self.config_test() - logger.debug(self.reverter.view_config_changes(logging=True)) + logger.debug(self.reverter.view_config_changes(for_logging=True)) self._reload() def _reload(self): diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index e2bc28cb9..863074374 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -94,7 +94,7 @@ class Reverter(object): "Unable to load checkpoint during rollback") rollback -= 1 - def view_config_changes(self, logging=False): + def view_config_changes(self, for_logging=False): """Displays all saved checkpoints. All checkpoints are printed by @@ -144,7 +144,7 @@ class Reverter(object): output.append(os.linesep) - if logging: + if for_logging: return os.linesep.join(output) zope.component.getUtility(interfaces.IDisplay).notification( os.linesep.join(output), display_util.HEIGHT) From 2c8d042974dbbf78a03a95dad187ed39c867664a Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 8 Jan 2016 04:58:17 -0800 Subject: [PATCH 101/127] added in tls_sni logging from #2002 --- letsencrypt-apache/letsencrypt_apache/tls_sni_01.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index 2049eb574..ca7985f35 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -1,12 +1,14 @@ """A class that performs TLS-SNI-01 challenges for Apache""" import os +import logging from letsencrypt.plugins import common from letsencrypt_apache import obj from letsencrypt_apache import parser +logger = logging.getLogger(__name__) class ApacheTlsSni01(common.TLSSNI01): """Class that performs TLS-SNI-01 challenges within the Apache configurator @@ -104,6 +106,7 @@ class ApacheTlsSni01(common.TLSSNI01): self.configurator.reverter.register_file_creation( True, self.challenge_conf) + logger.debug("writing a config file with text: %s", config_text) with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) From b039c884d8ffa16e2b3ac9663cca634bfd2f8bc3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 8 Jan 2016 14:09:44 -0500 Subject: [PATCH 102/127] Don't use cryptography version 1.2 --- acme/setup.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 7314152cd..54c4d82d9 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,7 +9,7 @@ version = '0.2.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8', + 'cryptography>=0.8,<1.2', # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) 'PyOpenSSL>=0.15', 'pyrfc3339', diff --git a/setup.py b/setup.py index f95f672ff..4005b0973 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), 'configobj', - 'cryptography>=0.7', # load_pem_x509_certificate + 'cryptography>=0.7,<1.2', # load_pem_x509_certificate 'parsedatetime', 'psutil>=2.1.0', # net_connections introduced in 2.1.0 'PyOpenSSL', From b1e67f241e64137077215f4f432995ae10439006 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 8 Jan 2016 14:31:30 -0500 Subject: [PATCH 103/127] Fix merge conflicts properly --- acme/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index 82a08b6ef..c10c95546 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -10,7 +10,6 @@ version = '0.2.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) -<<<<<<< HEAD 'cryptography>=0.8,<1.2', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13', From 96b55c8f349ac29e314e5468c406d0e07f4b4917 Mon Sep 17 00:00:00 2001 From: bmw Date: Fri, 8 Jan 2016 17:02:35 -0500 Subject: [PATCH 104/127] Revert "Don't use cryptography version 1.2" --- acme/setup.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 54c4d82d9..7314152cd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,7 +9,7 @@ version = '0.2.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8,<1.2', + 'cryptography>=0.8', # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15) 'PyOpenSSL>=0.15', 'pyrfc3339', diff --git a/setup.py b/setup.py index 4005b0973..f95f672ff 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ version = meta['version'] install_requires = [ 'acme=={0}'.format(version), 'configobj', - 'cryptography>=0.7,<1.2', # load_pem_x509_certificate + 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', 'psutil>=2.1.0', # net_connections introduced in 2.1.0 'PyOpenSSL', From 705032bc67afe3705af7916d767487f4eeb38bc1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 8 Jan 2016 19:12:30 -0800 Subject: [PATCH 105/127] [Always] Install augeas-lenses - But do we need the augeas-lenses package? - Install augeas from backports even if the backports were already available (this is the third time fixing that bug!) --- bootstrap/_deb_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 84ab9e35a..58ea67a45 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -32,7 +32,7 @@ if apt-cache show python-virtualenv > /dev/null 2>&1; then virtualenv="$virtualenv python-virtualenv" fi -augeas_pkg=libaugeas0 +augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` AddBackportRepo() { @@ -54,7 +54,7 @@ AddBackportRepo() { echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list apt-get update - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" libaugeas0 augeas-lenses + apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg augeas_pkg= fi fi From 7728f4d28a14724783bd8ee42d54ab047d645500 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:24:39 +0000 Subject: [PATCH 106/127] Python 3 Travis testing for acme. Despite its description, https://github.com/letsencrypt/letsencrypt/pull/630, removed not only Python 2.6 support, but also Travis tests against Python 3. ACME library supports Python 3 and Travis should tests it. This must be merged before any pending PRs agains acme library. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a5d6d8a85..6cadbd36e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ env: matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 + - TOXENV=py33 + - TOXENV=py34 + - TOXENV=py35 - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration From 34010a0168d0df97a08bdd5b985e7694a37123f2 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:31:50 +0000 Subject: [PATCH 107/127] Python 3.5 needs explicit Travis setting --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6cadbd36e..680dfeb8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,12 +24,15 @@ env: - TOXENV=py27 BOULDER_INTEGRATION=1 - TOXENV=py33 - TOXENV=py34 - - TOXENV=py35 - TOXENV=lint - TOXENV=cover # Disabled for now due to requiring sudo -> causing more boulder integration # DNS timeouts :( # - TOXENV=apacheconftest +matrix: + include: + - env: TOXENV=py35 + python: 3.5 # Only build pushes to the master branch, PRs, and branches beginning with From b26dda3afe859ca6586d6d89d5b500ee416d2841 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 9 Jan 2016 13:38:12 +0000 Subject: [PATCH 108/127] Add Python 3.5 trove classifier to acme --- acme/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/setup.py b/acme/setup.py index 7314152cd..f585b3cdd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -66,6 +66,7 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], From 5b3bd890b70f831c492c904d05e26fec9c5b9201 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 9 Jan 2016 15:15:05 -0800 Subject: [PATCH 109/127] Default: renew 30 days before expiry, rather than 10 - gives more time for various fallback strategies if renewal doesn't work the first time --- letsencrypt/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index ac71bd9fe..fc58ffb8d 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -565,7 +565,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes return True # Renewals on the basis of expiry time - interval = self.configuration.get("renew_before_expiry", "10 days") + interval = self.configuration.get("renew_before_expiry", "30 days") expiry = crypto_util.notAfter(self.version( "cert", self.latest_common_version())) now = pytz.UTC.fromutc(datetime.datetime.utcnow()) From 3a7565afe5633abb5329b06c954c72f225e462c0 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 01:03:50 -0800 Subject: [PATCH 110/127] trigger travis rerun --- letsencrypt/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index fc58ffb8d..f7e5c3ad3 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -564,7 +564,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes logger.debug("Should renew, certificate is revoked.") return True - # Renewals on the basis of expiry time + # Renews some period before expiry time interval = self.configuration.get("renew_before_expiry", "30 days") expiry = crypto_util.notAfter(self.version( "cert", self.latest_common_version())) From c10bfd6efc6ff27efe94e8a397b809c4faf55986 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 10 Jan 2016 14:01:34 +0000 Subject: [PATCH 111/127] Fix wrong doc comment: account_public_key is None --- acme/acme/challenges.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 68bf3fce4..13d19d3c4 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -236,10 +236,8 @@ class HTTP01Response(KeyAuthorizationChallengeResponse): :param challenges.SimpleHTTP chall: Corresponding challenge. :param unicode domain: Domain name being verified. - :param account_public_key: Public key for the key pair - being authorized. If ``None`` key verification is not - performed! - :param JWK account_public_key: + :param JWK account_public_key: Public key for the key pair + being authorized. :param int port: Port used in the validation. :returns: ``True`` iff validation is successful, ``False`` From f5862a7a4f1313c9b1bf63dc8acedb2b07dac62f Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 10 Jan 2016 18:38:53 +0200 Subject: [PATCH 112/127] Parse all included paths in apache root configuration --- .../letsencrypt_apache/configurator.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 2a9fb0250..1d4618825 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -488,15 +488,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: list """ - # Search vhost-root, httpd.conf for possible virtual hosts - paths = self.aug.match( - ("/files%s//*[label()=~regexp('%s')]" % - (self.conf("vhost-root"), parser.case_i("VirtualHost")))) - + # Search base config, and all included paths for VirtualHosts vhs = [] + vhost_paths = {} + for vhost_path in self.parser.parser_paths.keys(): + paths = self.aug.match( + ("/files%s//*[label()=~regexp('%s')]" % + (vhost_path, parser.case_i("VirtualHost")))) + for path in paths: + new_vhost = self._create_vhost(path) + realpath = os.path.realpath(new_vhost.filep) + if realpath not in vhost_paths.keys(): + vhs.append(new_vhost) + vhost_paths[realpath] = new_vhost.filep + elif realpath == new_vhost.filep: + # Prefer "real" vhost paths instead of symlinked ones + # ex: sites-enabled/vh.conf -> sites-available/vh.conf - for path in paths: - vhs.append(self._create_vhost(path)) + # remove old (most likely) symlinked one + vhs = [v for v in vhs if v.filep != vhost_paths[realpath]] + vhs.append(new_vhost) + vhost_paths[realpath] = realpath return vhs From 4c02902762dc09b2c2ece0f23ba5da60102f8dad Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 10 Jan 2016 09:01:34 -0800 Subject: [PATCH 113/127] [bootstrap/_deb_common] Re-fix the always-install-backports * This bug seems to come back every time it's fixed :( --- bootstrap/_deb_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh index 58ea67a45..c2f58db75 100755 --- a/bootstrap/_deb_common.sh +++ b/bootstrap/_deb_common.sh @@ -54,10 +54,10 @@ AddBackportRepo() { echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list apt-get update - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= fi fi + apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= } From 39e4053b82b4649b6413606c4896c4b2c5cc987a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Sun, 10 Jan 2016 19:15:09 +0200 Subject: [PATCH 114/127] Removed some now obsolete mock code from tests --- .../letsencrypt_apache/tests/configurator_test.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 9838b4f52..212f128f2 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -128,20 +128,10 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(found, 6) # Handle case of non-debian layout get_virtual_hosts - orig_conf = self.config.conf with mock.patch( "letsencrypt_apache.configurator.ApacheConfigurator.conf" - ) as mock_conf: - def conf_sideeffect(key): - """Handle calls to configurator.conf() - :param key: configuration key - :return: configuration value - """ - if key == "handle-sites": - return False - else: - return orig_conf(key) - mock_conf.side_effect = conf_sideeffect + ) as mock_conf: + mock_conf.return_value = False vhs = self.config.get_virtual_hosts() self.assertEqual(len(vhs), 6) From 0a536d50bea24df181788693ff2751411ff6ed4f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 10 Jan 2016 17:31:50 +0000 Subject: [PATCH 115/127] Remove dead code (error in except) --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3e28ef47..7ff740354 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -539,7 +539,7 @@ class ClientNetwork(object): # TODO: response.json() is called twice, once here, and # once in _get and _post clients jobj = response.json() - except ValueError as error: + except ValueError: jobj = None if not response.ok: From bdd9fa44854b2e25211fd35425337fdc1d73b6cd Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 10 Jan 2016 18:47:04 +0000 Subject: [PATCH 116/127] Quickfix too-many-instance-attributes. https://github.com/letsencrypt/letsencrypt/pull/2135#issuecomment-170381179 --- acme/acme/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 7ff740354..b65577c53 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -483,7 +483,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes 'Successful revocation must return HTTP OK status') -class ClientNetwork(object): +class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Client network.""" JSON_CONTENT_TYPE = 'application/json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' From bf74b2cc644c0ffa1e29f0694b25af214b09bf18 Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 19:12:30 +0000 Subject: [PATCH 117/127] Change test RewriteRule so that it conforms with Apaches spec. --- .../letsencrypt_apache/tests/configurator_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 9838b4f52..599334a29 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -857,7 +857,8 @@ class TwoVhost80Test(util.ApacheTest): # Create a preexisting rewrite rule self.config.parser.add_dir( - self.vh_truth[3].path, "RewriteRule", ["Unknown"]) + self.vh_truth[3].path, "RewriteRule", ["UnknownPattern", + "UnknownTarget"]) self.config.save() # This will create an ssl vhost for letsencrypt.demo @@ -872,7 +873,7 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(len(rw_engine), 1) # three args to rw_rule + 1 arg for the pre existing rewrite - self.assertEqual(len(rw_rule), 4) + self.assertEqual(len(rw_rule), 5) self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path)) self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path)) From 6c18a7d318c477ed0ffae1aa3e57f5bd197fa1aa Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 19:15:23 +0000 Subject: [PATCH 118/127] Revise RewriteRule sifting algorithm --- .../letsencrypt_apache/configurator.py | 63 ++++++++++++++++--- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 948514f9b..288ea9dc8 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -696,6 +696,42 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return non_ssl_vh_fp[:-(len(".conf"))] + self.conf("le_vhost_ext") else: return non_ssl_vh_fp + self.conf("le_vhost_ext") + + def _sift_line(self, line): + """ Decides whether a line shouldn't be copied from a http vhost to a + SSL vhost. + + A canonical example of when sifting a line is required: + When the http vhost contains a RewriteRule that unconditionally redirects + any request to the https version of the same site. + e.g: RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent] + Copying the above line to the ssl vhost would cause a redirection loop. + + :param str line: a line extracted from the http vhost config file. + + :returns: True - don't copy line from http vhost to SSL vhost. + :rtype: (bool) + """ + + rewrite_rule = "RewriteRule" + if line.lstrip()[:len(rewrite_rule)] != rewrite_rule: + return False + # line starts with RewriteRule. + + # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html + # The syntax of a RewriteRule is: + # RewriteRule pattern target [Flag1,Flag2,Flag3] + # i.e. target is required, so it must exist. + target = line.split()[2].strip() + + https_prefix = "https://" + if len(target) skeleton. @@ -714,20 +750,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): with open(avail_fp, "r") as orig_file: with open(ssl_fp, "w") as new_file: new_file.write("\n") - - # In some cases we wouldn't want to copy the exact - # directives used in an http vhost to a ssl vhost. - # An example: - # If there's a redirect rewrite rule directive installed in - # the http vhost - copying it to the ssl vhost would cause - # a redirection loop. - blacklist_set = set(['RewriteRule', 'RewriteEngine']) + sift = False for line in orig_file: - line_set = set(line.split()) - if not line_set & blacklist_set: # & -> Intersection + if self._sift_line(line): + if not sift: + new_file.write("# The following rewrite rules" + "were *not* enabled on your HTTPS site, " + "because they have the potential to create " + "redirection loops:") + sift = True + new_file.write("# " + line) + else: new_file.write(line) + new_file.write("\n") + + if sift: + logger.warn("Some rewrite rules were *not* enabled on " + "your HTTPS site, because they have the " + "potential to create redirection loops.") + except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") raise errors.PluginError("Unable to write/read in make_vhost_ssl") From ae572fe0840bfc6052c3c86acfe8df7f530a26e7 Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 19:20:29 +0000 Subject: [PATCH 119/127] Make lint happy --- .../letsencrypt_apache/configurator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 288ea9dc8..13c7c91a8 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -696,11 +696,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return non_ssl_vh_fp[:-(len(".conf"))] + self.conf("le_vhost_ext") else: return non_ssl_vh_fp + self.conf("le_vhost_ext") - + def _sift_line(self, line): - """ Decides whether a line shouldn't be copied from a http vhost to a + """ Decides whether a line shouldn't be copied from a http vhost to a SSL vhost. - + A canonical example of when sifting a line is required: When the http vhost contains a RewriteRule that unconditionally redirects any request to the https version of the same site. @@ -709,7 +709,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str line: a line extracted from the http vhost config file. - :returns: True - don't copy line from http vhost to SSL vhost. + :returns: True - don't copy line from http vhost to SSL vhost. :rtype: (bool) """ @@ -718,14 +718,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return False # line starts with RewriteRule. - # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html + # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html # The syntax of a RewriteRule is: # RewriteRule pattern target [Flag1,Flag2,Flag3] - # i.e. target is required, so it must exist. + # i.e. target is required, so it must exist. target = line.split()[2].strip() https_prefix = "https://" - if len(target) Date: Mon, 11 Jan 2016 19:48:17 +0000 Subject: [PATCH 120/127] Dequote possible quoted target --- letsencrypt-apache/letsencrypt_apache/configurator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 13c7c91a8..383db3d98 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -716,6 +716,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): rewrite_rule = "RewriteRule" if line.lstrip()[:len(rewrite_rule)] != rewrite_rule: return False + # line starts with RewriteRule. # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html @@ -723,6 +724,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # RewriteRule pattern target [Flag1,Flag2,Flag3] # i.e. target is required, so it must exist. target = line.split()[2].strip() + + # target may be surrounded with double quotes + if len(target) > 0 and target[0]==target[-1]=="\"" + target = target[1:-1] https_prefix = "https://" if len(target) < len(https_prefix): From a43e7b11f1d926c4bfe89a402b8597a1fde8fc4c Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 19:55:15 +0000 Subject: [PATCH 121/127] Add colon --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 383db3d98..af00e7527 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -726,7 +726,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): target = line.split()[2].strip() # target may be surrounded with double quotes - if len(target) > 0 and target[0]==target[-1]=="\"" + if len(target) > 0 and target[0]==target[-1]=="\"": target = target[1:-1] https_prefix = "https://" From 9c2a0362a763a0aa804748de389650051d351a6c Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 19:55:55 +0000 Subject: [PATCH 122/127] Add rewrite tests: normal, small, quoted, etc. --- .../letsencrypt_apache/tests/configurator_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 599334a29..78714b881 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -922,6 +922,16 @@ class TwoVhost80Test(util.ApacheTest): self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access self.assertEqual(len(self.config.vhosts), 7) + def test_sift_line(self): + # pylint: disable=protected-access + small_quoted_target = "RewriteRule ^ \"http://\"" + self.assertFalse(self.config._sift_line(small_quoted_target)) + + https_target = "RewriteRule ^ https://satoshi" + self.assertTrue(self.config._sift_line(https_target)) + + normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]" + self.assertFalse(self.config._sift_line(normal_target)) def get_achalls(self): """Return testing achallenges.""" From c89dcad313e933630e1b8e7b81b4878ebc4c5534 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 11 Jan 2016 12:22:22 -0800 Subject: [PATCH 123/127] This default shouldn't be a magic string --- letsencrypt/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index f7e5c3ad3..08b48ff5e 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -565,7 +565,8 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes return True # Renews some period before expiry time - interval = self.configuration.get("renew_before_expiry", "30 days") + default_interval = constants.RENEWER_DEFAULTS["renew_before_expiry"] + interval = self.configuration.get("renew_before_expiry", default_interval) expiry = crypto_util.notAfter(self.version( "cert", self.latest_common_version())) now = pytz.UTC.fromutc(datetime.datetime.utcnow()) From 4645bf8329f24b59c7be742c5ad4befed5dd3037 Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 20:58:52 +0000 Subject: [PATCH 124/127] Make lint happy --- .../letsencrypt_apache/configurator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index af00e7527..16f264747 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -724,9 +724,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # RewriteRule pattern target [Flag1,Flag2,Flag3] # i.e. target is required, so it must exist. target = line.split()[2].strip() - + # target may be surrounded with double quotes - if len(target) > 0 and target[0]==target[-1]=="\"": + if len(target) > 0 and target[0] == target[-1] == "\"": target = target[1:-1] https_prefix = "https://" @@ -760,10 +760,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for line in orig_file: if self._sift_line(line): if not sift: - new_file.write("# The following rewrite rules" - "were *not* enabled on your HTTPS site, " - "because they have the potential to create " - "redirection loops:") + new_file.write("# The following rewrite rules " + "were *not* enabled on your HTTPS site,\n" + "# because they have the potential to create " + "redirection loops:\n") sift = True new_file.write("# " + line) else: From b28b5b08d7f325f7bd089b47250450788240e670 Mon Sep 17 00:00:00 2001 From: sagi Date: Mon, 11 Jan 2016 20:59:19 +0000 Subject: [PATCH 125/127] More tests; Make Nose happy --- .../tests/configurator_test.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 78714b881..954560aa6 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -930,9 +930,35 @@ class TwoVhost80Test(util.ApacheTest): https_target = "RewriteRule ^ https://satoshi" self.assertTrue(self.config._sift_line(https_target)) - normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]" + normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]" self.assertFalse(self.config._sift_line(normal_target)) + def test_make_vhost_ssl_with_http_vhost_redirect_rewrite_rule(self): + self.config.parser.modules.add("rewrite_module") + + http_vhost = self.vh_truth[0] + + self.config.parser.add_dir( + http_vhost.path, "RewriteEngine", "on") + + self.config.parser.add_dir( + http_vhost.path, "RewriteRule", + ["^", + "https://%{SERVER_NAME}%{REQUEST_URI}", + "[L,QSA,R=permanent]"]) + self.config.save() + + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) + + #import ipdb; ipdb.set_trace() + self.assertTrue(self.config.parser.find_dir( + "RewriteEngine", "on", ssl_vhost.path, False)) + + conf_text = open(ssl_vhost.filep).read() + commented_rewrite_rule = \ + "# RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent]" + self.assertTrue(commented_rewrite_rule in conf_text) + def get_achalls(self): """Return testing achallenges.""" account_key = self.rsa512jwk From 10df56bab6b5d7a6fb4ee791b1239b149f67ee6a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Jan 2016 18:21:33 -0800 Subject: [PATCH 126/127] Added revisions --- .../letsencrypt_apache/configurator.py | 63 +++++++++---------- .../tests/configurator_test.py | 6 +- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 16f264747..de179966a 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -8,6 +8,7 @@ import shutil import socket import time +import zope.component import zope.interface from acme import challenges @@ -698,42 +699,36 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return non_ssl_vh_fp + self.conf("le_vhost_ext") def _sift_line(self, line): - """ Decides whether a line shouldn't be copied from a http vhost to a - SSL vhost. + """Decides whether a line should be copied to a SSL vhost. A canonical example of when sifting a line is required: - When the http vhost contains a RewriteRule that unconditionally redirects - any request to the https version of the same site. - e.g: RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent] - Copying the above line to the ssl vhost would cause a redirection loop. + When the http vhost contains a RewriteRule that unconditionally + redirects any request to the https version of the same site. + e.g: + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent] + Copying the above line to the ssl vhost would cause a + redirection loop. - :param str line: a line extracted from the http vhost config file. + :param str line: a line extracted from the http vhost. :returns: True - don't copy line from http vhost to SSL vhost. - :rtype: (bool) + :rtype: bool + """ - - rewrite_rule = "RewriteRule" - if line.lstrip()[:len(rewrite_rule)] != rewrite_rule: + if not line.lstrip().startswith("RewriteRule"): return False - # line starts with RewriteRule. - # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html # The syntax of a RewriteRule is: # RewriteRule pattern target [Flag1,Flag2,Flag3] # i.e. target is required, so it must exist. target = line.split()[2].strip() - # target may be surrounded with double quotes - if len(target) > 0 and target[0] == target[-1] == "\"": + # target may be surrounded with quotes + if target[0] in ("'", '"') and target[0] == target[-1]: target = target[1:-1] - https_prefix = "https://" - if len(target) < len(https_prefix): - return False - - if target[:len(https_prefix)] == https_prefix: + if target.startswith("https://"): return True return False @@ -750,36 +745,38 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # First register the creation so that it is properly removed if # configuration is rolled back self.reverter.register_file_creation(False, ssl_fp) + sift = False try: with open(avail_fp, "r") as orig_file: with open(ssl_fp, "w") as new_file: new_file.write("\n") - sift = False - for line in orig_file: if self._sift_line(line): if not sift: - new_file.write("# The following rewrite rules " - "were *not* enabled on your HTTPS site,\n" - "# because they have the potential to create " - "redirection loops:\n") + new_file.write( + "# Some rewrite rules in this file were " + "were disabled on your HTTPS site,\n" + "# because they have the potential to " + "create redirection loops.\n") sift = True new_file.write("# " + line) else: new_file.write(line) - new_file.write("\n") - - if sift: - logger.warn("Some rewrite rules were *not* enabled on " - "your HTTPS site, because they have the " - "potential to create redirection loops.") - except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") raise errors.PluginError("Unable to write/read in make_vhost_ssl") + if sift: + reporter = zope.component.getUtility(interfaces.IReporter) + reporter.add_message( + "Some rewrite rules copied from {0} were disabled in the " + "vhost for your HTTPS site located at {1} because they have " + "the potential to create redirection loops.".format(avail_fp, + ssl_fp), + reporter.MEDIUM_PRIORITY) + def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() ssl_addr_p = self.aug.match(vh_path + "/arg") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 954560aa6..5905e281b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -933,7 +933,8 @@ class TwoVhost80Test(util.ApacheTest): normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]" self.assertFalse(self.config._sift_line(normal_target)) - def test_make_vhost_ssl_with_http_vhost_redirect_rewrite_rule(self): + @mock.patch("letsencrypt_apache.configurator.zope.component.getUtility") + def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): self.config.parser.modules.add("rewrite_module") http_vhost = self.vh_truth[0] @@ -950,7 +951,6 @@ class TwoVhost80Test(util.ApacheTest): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - #import ipdb; ipdb.set_trace() self.assertTrue(self.config.parser.find_dir( "RewriteEngine", "on", ssl_vhost.path, False)) @@ -958,6 +958,8 @@ class TwoVhost80Test(util.ApacheTest): commented_rewrite_rule = \ "# RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent]" self.assertTrue(commented_rewrite_rule in conf_text) + mock_get_utility().add_message.assert_called_once_with(mock.ANY, + mock.ANY) def get_achalls(self): """Return testing achallenges.""" From 4cdf63c55e971929f3c14b2c940c6cfc3c9c19f1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 11 Jan 2016 18:27:01 -0800 Subject: [PATCH 127/127] Fix a couple nits --- letsencrypt-apache/letsencrypt_apache/configurator.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 9b1b3a961..4066d6264 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -740,10 +740,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if target[0] in ("'", '"') and target[0] == target[-1]: target = target[1:-1] - if target.startswith("https://"): - return True - - return False + # Sift line if it redirects the request to a HTTPS site + return target.startswith("https://") def _copy_create_ssl_vhost_skeleton(self, avail_fp, ssl_fp): """Copies over existing Vhost with IfModule mod_ssl.c> skeleton. @@ -771,7 +769,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "were disabled on your HTTPS site,\n" "# because they have the potential to " "create redirection loops.\n") - sift = True + sift = True new_file.write("# " + line) else: new_file.write(line)