From a74eff5fbd4f68a584bff9d2805f28954d95f978 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 28 Aug 2015 06:40:19 +0000 Subject: [PATCH 01/31] Revert "Revert PR #708." This reverts commit 70e311b43f6910c070dce3bab3396db495636d00. --- acme/setup.py | 10 +++++++--- letsencrypt-apache/setup.py | 8 +++++++- letsencrypt-compatibility-test/setup.py | 8 +++++++- letsencrypt-nginx/setup.py | 8 +++++++- letshelp-letsencrypt/setup.py | 8 ++++---- setup.py | 10 +++++++--- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 6d8208414..5f1da2391 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,7 +9,6 @@ install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', - 'mock<1.1.0', # py26 'pyrfc3339', 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) 'pyasn1', # urllib3 InsecurePlatformWarning (#304) @@ -23,8 +22,13 @@ install_requires = [ # env markers in extras_require cause problems with older pip: #517 if sys.version_info < (2, 7): - # only some distros recognize stdlib argparse as already satisfying - install_requires.append('argparse') + install_requires.extend([ + # only some distros recognize stdlib argparse as already satisfying + 'argparse', + 'mock<1.1.0', + ]) +else: + install_requires.append('mock') testing_extras = [ 'nose', diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 39f4b68e1..5f1b0a95d 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -1,3 +1,5 @@ +import sys + from setuptools import setup from setuptools import find_packages @@ -5,12 +7,16 @@ from setuptools import find_packages install_requires = [ 'acme', 'letsencrypt', - 'mock<1.1.0', # py26 'python-augeas', 'zope.component', 'zope.interface', ] +if sys.version_info < (2, 7): + install_requires.append('mock<1.1.0') +else: + install_requires.append('mock') + setup( name='letsencrypt-apache', packages=find_packages(), diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index f02041e55..99e66f54f 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -1,3 +1,5 @@ +import sys + from setuptools import setup from setuptools import find_packages @@ -7,10 +9,14 @@ install_requires = [ 'letsencrypt-apache', 'letsencrypt-nginx', 'docker-py', - 'mock<1.1.0', # py26 'zope.interface', ] +if sys.version_info < (2, 7): + install_requires.append('mock<1.1.0') +else: + install_requires.append('mock') + setup( name='letsencrypt-compatibility-test', packages=find_packages(), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 92b974974..64742e7b6 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -1,3 +1,5 @@ +import sys + from setuptools import setup from setuptools import find_packages @@ -6,10 +8,14 @@ install_requires = [ 'acme', 'letsencrypt', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - 'mock<1.1.0', # py26 'zope.interface', ] +if sys.version_info < (2, 7): + install_requires.append('mock<1.1.0') +else: + install_requires.append('mock') + setup( name='letsencrypt-nginx', packages=find_packages(), diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 6b89a6d09..7d42af88b 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -6,17 +6,17 @@ from setuptools import find_packages install_requires = [] if sys.version_info < (2, 7): - install_requires.append("mock<1.1.0") + install_requires.append('mock<1.1.0') else: - install_requires.append("mock") + install_requires.append('mock') setup( - name="letshelp-letsencrypt", + name='letshelp-letsencrypt', packages=find_packages(), install_requires=install_requires, entry_points={ 'console_scripts': [ - "letshelp-letsencrypt-apache = letshelp_letsencrypt.apache:main", + 'letshelp-letsencrypt-apache = letshelp_letsencrypt.apache:main', ], }, include_package_data=True, diff --git a/setup.py b/setup.py index f816c6c56..e15ade595 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ install_requires = [ 'ConfigArgParse', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate - 'mock<1.1.0', # py26 'parsedatetime', 'psutil>=2.1.0', # net_connections introduced in 2.1.0 'PyOpenSSL', @@ -47,8 +46,13 @@ install_requires = [ # env markers in extras_require cause problems with older pip: #517 if sys.version_info < (2, 7): - # only some distros recognize stdlib argparse as already satisfying - install_requires.append('argparse') + install_requires.extend([ + # only some distros recognize stdlib argparse as already satisfying + 'argparse', + 'mock<1.1.0', + ]) +else: + install_requires.append('mock') dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 From 6649af94790407a1d0a2bde72cc927ca881a64ad Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 27 Sep 2015 20:57:31 +0000 Subject: [PATCH 02/31] Developer virtualenv bootstrap scripts. --- bootstrap/dev/README | 1 + bootstrap/dev/_venv_common.sh | 25 +++++++++++++++ bootstrap/dev/venv.sh | 11 +++++++ bootstrap/dev/venv3.sh | 8 +++++ docs/contributing.rst | 59 ++++++++++++++++++----------------- docs/using.rst | 2 ++ 6 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 bootstrap/dev/README create mode 100755 bootstrap/dev/_venv_common.sh create mode 100755 bootstrap/dev/venv.sh create mode 100755 bootstrap/dev/venv3.sh diff --git a/bootstrap/dev/README b/bootstrap/dev/README new file mode 100644 index 000000000..1509c24e0 --- /dev/null +++ b/bootstrap/dev/README @@ -0,0 +1 @@ +This directory contains developer setup. \ No newline at end of file diff --git a/bootstrap/dev/_venv_common.sh b/bootstrap/dev/_venv_common.sh new file mode 100755 index 000000000..2d84dc39b --- /dev/null +++ b/bootstrap/dev/_venv_common.sh @@ -0,0 +1,25 @@ +#!/bin/sh -xe + +VENV_NAME=${VENV_NAME:-venv} + +# .egg-info directories tend to cause bizzaire problems (e.g. `pip -e +# .` might unexpectedly install letshelp-letsencrypt only, in case +# `python letshelp-letsencrypt/setup.py build` has been called +# earlier) +rm -rf *.egg-info + +# virtualenv setup is NOT idempotent: shutil.Error: +# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and +# `venv/bin/python2` are the same file +mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true +virtualenv --no-site-packages $VENV_NAME $VENV_ARGS +. ./$VENV_NAME/bin/activate + +# Separately install setuptools and pip to make sure following +# invocations use latest +pip install -U setuptools +pip install -U pip +pip install "$@" + +echo "Please run the following command to activate developer environment:" +echo "source $VENV_NAME/bin/activate" diff --git a/bootstrap/dev/venv.sh b/bootstrap/dev/venv.sh new file mode 100755 index 000000000..90088ac9b --- /dev/null +++ b/bootstrap/dev/venv.sh @@ -0,0 +1,11 @@ +#!/bin/sh -xe +# Developer virtualenv setup for Let's Encrypt client + +./bootstrap/dev/_venv_common.sh \ + -r requirements.txt \ + -e acme[testing] \ + -e .[dev,docs,testing] \ + -e letsencrypt-apache \ + -e letsencrypt-nginx \ + -e letshelp-letsencrypt \ + -e letsencrypt-compatibility-test diff --git a/bootstrap/dev/venv3.sh b/bootstrap/dev/venv3.sh new file mode 100755 index 000000000..ccffffb83 --- /dev/null +++ b/bootstrap/dev/venv3.sh @@ -0,0 +1,8 @@ +#!/bin/sh -xe +# Developer Python3 virtualenv setup for Let's Encrypt + +export VENV_NAME="${VENV_NAME:-venv3}" +export VENV_ARGS="--python python3" + +./bootstrap/dev/_venv_common.sh \ + -e acme[testing] \ diff --git a/docs/contributing.rst b/docs/contributing.rst index c6443e3b2..6a9682494 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,39 +7,40 @@ Contributing Hacking ======= -Start by :doc:`installing dependencies and setting up Let's Encrypt -`. - -When you're done activate the virtualenv: - -.. code-block:: shell - - source ./venv/bin/activate - -This step should prepend you prompt with ``(venv)`` and save you from -typing ``./venv/bin/...``. It is also required to run some of the -`testing`_ tools. Virtualenv can be disabled at any time by typing -``deactivate``. More information can be found in `virtualenv -documentation`_. - -Install the development packages: - -.. code-block:: shell - - pip install -r requirements.txt -e acme -e .[dev,docs,testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt - -.. note:: `-e` (short for `--editable`) turns on *editable mode* in - which any source code changes in the current working - directory are "live" and no further `pip install ...` - invocations are necessary while developing. - - This is roughly equivalent to `python setup.py develop`. For - more info see `man pip`. - The code base, including your pull requests, **must** have 100% unit test coverage, pass our `integration`_ tests **and** be compliant with the :ref:`coding style `. + +Bootstrap +--------- + +Start by :ref:`installing Let's Encrypt prerequisites +`. Then run: + +.. code-block:: shell + + ./bootstrap/dev/venv.sh + ./bootstrap/dev/venv3.sh + +Both of the commands suggest to activate the virtualenv (you can +activate one at a time only): + +.. code-block:: shell + + source ./$VENV_NAME/bin/activate + +This step should prepend you prompt with ``($VENV_NAME)`` and save you +from typing ``./$VENV_NAME/bin/...``. It is also required to run some +of the `testing`_ tools. Virtualenv can be disabled at any time by +typing ``deactivate``. More information can be found in `virtualenv +documentation`_. + +Note that packages are installed in so called *editable mode*, in +which any source code changes in the current working directory are +"live" and no further ``./bootstrap/dev/venv.sh`` or ``pip install +...`` invocations are necessary while developing. + .. _`virtualenv documentation`: https://virtualenv.pypa.io diff --git a/docs/using.rst b/docs/using.rst index cfce29bae..53ecd72b5 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -42,6 +42,8 @@ above method instead. https://github.com/letsencrypt/letsencrypt/archive/master.zip +.. _prerequisites: + Prerequisites ============= From 85e5165b5d82f3700ba4c960b3f957f7c4d7d3d9 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 27 Sep 2015 20:59:07 +0000 Subject: [PATCH 03/31] nit: missing EOF newline --- bootstrap/dev/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/dev/README b/bootstrap/dev/README index 1509c24e0..759496187 100644 --- a/bootstrap/dev/README +++ b/bootstrap/dev/README @@ -1 +1 @@ -This directory contains developer setup. \ No newline at end of file +This directory contains developer setup. From c976c0abdfdbeb7edc580b252243d94c5c0296ce Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Oct 2015 13:03:23 -0700 Subject: [PATCH 04/31] Removed duplicated code --- letsencrypt/client.py | 14 +++++--------- letsencrypt/storage.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index c82131af3..7a78add38 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -268,19 +268,15 @@ class Client(object): :param .RenewableCert cert: Newly issued certificate """ - if ("autorenew" not in cert.configuration or - cert.configuration.as_bool("autorenew")): - if ("autodeploy" not in cert.configuration or - cert.configuration.as_bool("autodeploy")): + if cert.autorenewal_is_enabled(): + if cert.autodeployment_is_enabled(): msg = "Automatic renewal and deployment has " else: msg = "Automatic renewal but not automatic deployment has " + elif cert.autodeployment_is_enabled(): + msg = "Automatic deployment but not automatic renewal has " else: - if ("autodeploy" not in cert.configuration or - cert.configuration.as_bool("autodeploy")): - msg = "Automatic deployment but not automatic renewal has " - else: - msg = "Automatic renewal and deployment has not " + msg = "Automatic renewal and deployment has not " msg += ("been enabled for your certificate. These settings can be " "configured in the directories under {0}.").format( diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 08dff25a1..435bb29d5 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -439,6 +439,18 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes with open(target) as f: return crypto_util.get_sans_from_cert(f.read()) + def autodeployment_is_enabled(self): + """Is automatic deployment enabled for this cert? + + If autodeploy is not specified, defaults to True. + + :returns: True if automatic deployment is enabled + :rtype: bool + + """ + return ("autodeploy" not in self.configuration or + self.configuration.as_bool("autodeploy")) + def should_autodeploy(self): """Should this lineage now automatically deploy a newer version? @@ -453,8 +465,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :rtype: bool """ - if ("autodeploy" not in self.configuration or - self.configuration.as_bool("autodeploy")): + if self.autodeployment_is_enabled(): if self.has_pending_deployment(): interval = self.configuration.get("deploy_before_expiry", "5 days") @@ -488,6 +499,18 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # certificate is not revoked). return False + def autorenewal_is_enabled(self): + """Is automatic renewal enabled for this cert? + + If autorenew is not specified, defaults to True. + + :returns: True if automatic renewal is enabled + :rtype: bool + + """ + return ("autorenew" not in self.configuration or + self.configuration.as_bool("autorenew")) + def should_autorenew(self): """Should we now try to autorenew the most recent cert version? @@ -504,8 +527,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :rtype: bool """ - if ("autorenew" not in self.configuration or - self.configuration.as_bool("autorenew")): + if self.autorenewal_is_enabled(): # Consider whether to attempt to autorenew this cert now # Renewals on the basis of revocation From 59348ad30c4276fc6fb89487e659d24db23dc453 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 1 Oct 2015 13:33:22 -0700 Subject: [PATCH 05/31] Made methods private and updated tests --- letsencrypt/storage.py | 8 ++++---- letsencrypt/tests/client_test.py | 15 ++++++--------- letsencrypt/tests/renewer_test.py | 32 +++++++++++++++++-------------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 435bb29d5..ceae4bfa9 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -129,7 +129,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes self.chain = self.configuration["chain"] self.fullchain = self.configuration["fullchain"] - def consistent(self): + def _consistent(self): """Are the files associated with this lineage self-consistent? :returns: Whether the files stored in connection with this @@ -187,7 +187,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # for x in ALL_FOUR))) == 1 return True - def fix(self): + def _fix(self): """Attempt to fix defects or inconsistencies in this lineage. .. todo:: Currently unimplemented. @@ -347,7 +347,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes smallest_current = min(self.current_version(x) for x in ALL_FOUR) return smallest_current < self.latest_common_version() - def update_link_to(self, kind, version): + def _update_link_to(self, kind, version): """Make the specified item point at the specified version. (Note that this method doesn't verify that the specified version @@ -379,7 +379,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :param int version: the desired version""" for kind in ALL_FOUR: - self.update_link_to(kind, version) + self._update_link_to(kind, version) def _notafterbefore(self, method, version): """Internal helper function for finding notbefore/notafter.""" diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 1a232bccb..1e63bdbb6 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -4,14 +4,12 @@ import shutil import tempfile import unittest -import configobj import OpenSSL import mock from acme import jose from letsencrypt import account -from letsencrypt import configuration from letsencrypt import errors from letsencrypt import le_util @@ -120,29 +118,28 @@ class ClientTest(unittest.TestCase): def test_report_renewal_status(self, mock_zope): # pylint: disable=protected-access cert = mock.MagicMock() - cert.configuration = configobj.ConfigObj() - cert.cli_config = configuration.RenewerConfiguration(self.config) + cert.cli_config.renewal_configs_dir = "/foo/bar/baz" - cert.configuration["autorenew"] = "True" - cert.configuration["autodeploy"] = "True" + cert.autorenewal_is_enabled.return_value = True + cert.autodeployment_is_enabled.return_value = True self.client._report_renewal_status(cert) msg = mock_zope().add_message.call_args[0][0] self.assertTrue("renewal and deployment has been" in msg) self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - cert.configuration["autorenew"] = "False" + cert.autorenewal_is_enabled.return_value = False self.client._report_renewal_status(cert) msg = mock_zope().add_message.call_args[0][0] self.assertTrue("deployment but not automatic renewal" in msg) self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - cert.configuration["autodeploy"] = "False" + cert.autodeployment_is_enabled.return_value = False self.client._report_renewal_status(cert) msg = mock_zope().add_message.call_args[0][0] self.assertTrue("renewal and deployment has not" in msg) self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - cert.configuration["autorenew"] = "True" + cert.autorenewal_is_enabled.return_value = True self.client._report_renewal_status(cert) msg = mock_zope().add_message.call_args[0][0] self.assertTrue("renewal but not automatic deployment" in msg) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index e67631605..b7b2843d5 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -123,46 +123,47 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertRaises( errors.CertStorageError, storage.RenewableCert, config, defaults) - def test_consistent(self): # pylint: disable=too-many-statements + def test_consistent(self): + # pylint: disable=too-many-statements,protected-access oldcert = self.test_rc.cert self.test_rc.cert = "relative/path" # Absolute path for item requirement - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) self.test_rc.cert = oldcert # Items must exist requirement - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) # Items must be symlinks requirements fill_with_sample_data(self.test_rc) - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) unlink_all(self.test_rc) # Items must point to desired place if they are relative for kind in ALL_FOUR: os.symlink(os.path.join("..", kind + "17.pem"), getattr(self.test_rc, kind)) - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) unlink_all(self.test_rc) # Items must point to desired place if they are absolute for kind in ALL_FOUR: os.symlink(os.path.join(self.tempdir, kind + "17.pem"), getattr(self.test_rc, kind)) - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) unlink_all(self.test_rc) # Items must point to things that exist for kind in ALL_FOUR: os.symlink(os.path.join("..", "..", "archive", "example.org", kind + "17.pem"), getattr(self.test_rc, kind)) - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) # This version should work fill_with_sample_data(self.test_rc) - self.assertTrue(self.test_rc.consistent()) + self.assertTrue(self.test_rc._consistent()) # Items must point to things that follow the naming convention os.unlink(self.test_rc.fullchain) os.symlink(os.path.join("..", "..", "archive", "example.org", "fullchain_17.pem"), self.test_rc.fullchain) with open(self.test_rc.fullchain, "w") as f: f.write("wrongly-named fullchain") - self.assertFalse(self.test_rc.consistent()) + self.assertFalse(self.test_rc._consistent()) def test_current_target(self): # Relative path logic @@ -259,14 +260,15 @@ class RenewableCertTests(BaseRenewableCertTest): with open(where, "w") as f: f.write(kind) self.assertEqual(ver, self.test_rc.current_version(kind)) - self.test_rc.update_link_to("cert", 3) - self.test_rc.update_link_to("privkey", 2) + # pylint: disable=protected-access + self.test_rc._update_link_to("cert", 3) + self.test_rc._update_link_to("privkey", 2) self.assertEqual(3, self.test_rc.current_version("cert")) self.assertEqual(2, self.test_rc.current_version("privkey")) self.assertEqual(5, self.test_rc.current_version("chain")) self.assertEqual(5, self.test_rc.current_version("fullchain")) # Currently we are allowed to update to a version that doesn't exist - self.test_rc.update_link_to("chain", 3000) + self.test_rc._update_link_to("chain", 3000) # However, current_version doesn't allow querying the resulting # version (because it's a broken link). self.assertEqual(os.path.basename(os.readlink(self.test_rc.chain)), @@ -507,7 +509,8 @@ class RenewableCertTests(BaseRenewableCertTest): self.defaults, self.cli_config) # This consistency check tests most relevant properties about the # newly created cert lineage. - self.assertTrue(result.consistent()) + # pylint: disable=protected-access + self.assertTrue(result._consistent()) self.assertTrue(os.path.exists(os.path.join( self.cli_config.renewal_configs_dir, "the-lineage.com.conf"))) with open(result.fullchain) as f: @@ -578,9 +581,10 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertRaises( errors.CertStorageError, self.test_rc.newest_available_version, "elephant") + # pylint: disable=protected-access self.assertRaises( errors.CertStorageError, - self.test_rc.update_link_to, "elephant", 17) + self.test_rc._update_link_to, "elephant", 17) def test_ocsp_revoked(self): # XXX: This is currently hardcoded to False due to a lack of an From 895faa7dc94b554881b4330f20354670c276df19 Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Fri, 2 Oct 2015 22:36:56 -0400 Subject: [PATCH 06/31] Add OS X bootstrap for integration enviornment Installs requirements and sets up environment to run boulder and integration tests --- tests/mac-bootstrap.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 tests/mac-bootstrap.sh diff --git a/tests/mac-bootstrap.sh b/tests/mac-bootstrap.sh new file mode 100755 index 000000000..38db6a969 --- /dev/null +++ b/tests/mac-bootstrap.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +#Check Homebrew +if ! hash brew 2>/dev/null; then + echo "Homebrew Not Installed\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + +brew install libtool mariadb rabbitmq coreutils go + +mysql.server start + +rabbit_pid=`ps | grep rabbitmq | grep -v grep | awk '{ print $1}'` +if [ -n rabbit_pid ]; then + echo "RabbitMQ already running" +else + rabbitmq-server & +fi + +hosts_entry=`cat /etc/hosts | grep "127.0.0.1 le.wtf"` +if [ -z hosts_entry ]; then + echo "Adding hosts entry for le.wtf..." + sudo sh -c "echo 127.0.0.1 le.wtf >> /etc/hosts" +fi + +./tests/boulder-start.sh From 0868a5962f4ae812f9bcb1b39e4c3bace207074b Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Fri, 2 Oct 2015 22:37:22 -0400 Subject: [PATCH 07/31] Add documentation for OS X bootstrap script --- docs/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c6443e3b2..7b0768efb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -67,6 +67,8 @@ The following tools are there to help you: Integration ~~~~~~~~~~~ +Mac OS X users: Run `./tests/mac-integration.sh` to configure the +integration tests environment and start boulder First, install `Go`_ 1.5, libtool-ltdl, mariadb-server and rabbitmq-server and then start Boulder_, an ACME CA server:: From 8409c9c658b247421caef540d57410dc8a00ef41 Mon Sep 17 00:00:00 2001 From: Brandon Kreisel Date: Sat, 3 Oct 2015 11:27:39 -0400 Subject: [PATCH 08/31] Meddle with more documentation and learn how to bash flag --- docs/contributing.rst | 6 +++--- tests/mac-bootstrap.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b0768efb..c746c6ae7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -67,10 +67,10 @@ The following tools are there to help you: Integration ~~~~~~~~~~~ -Mac OS X users: Run `./tests/mac-integration.sh` to configure the -integration tests environment and start boulder +Mac OS X users: Run `./tests/mac-bootstrap.sh` instead of `boulder-start.sh` to +install dependencies, configure the environment, and start boulder. -First, install `Go`_ 1.5, libtool-ltdl, mariadb-server and +Otherwise, install `Go`_ 1.5, libtool-ltdl, mariadb-server and rabbitmq-server and then start Boulder_, an ACME CA server:: ./tests/boulder-start.sh diff --git a/tests/mac-bootstrap.sh b/tests/mac-bootstrap.sh index 38db6a969..66036ce56 100755 --- a/tests/mac-bootstrap.sh +++ b/tests/mac-bootstrap.sh @@ -11,14 +11,14 @@ brew install libtool mariadb rabbitmq coreutils go mysql.server start rabbit_pid=`ps | grep rabbitmq | grep -v grep | awk '{ print $1}'` -if [ -n rabbit_pid ]; then +if [ -n "$rabbit_pid" ]; then echo "RabbitMQ already running" else rabbitmq-server & fi hosts_entry=`cat /etc/hosts | grep "127.0.0.1 le.wtf"` -if [ -z hosts_entry ]; then +if [ -z "$hosts_entry" ]; then echo "Adding hosts entry for le.wtf..." sudo sh -c "echo 127.0.0.1 le.wtf >> /etc/hosts" fi From 4925e0b811616915c9c7752d506eab2739374ba3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 07:47:14 +0000 Subject: [PATCH 09/31] Update MANIFEST.in for compatibility-test --- letsencrypt-compatibility-test/MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt-compatibility-test/MANIFEST.in b/letsencrypt-compatibility-test/MANIFEST.in index a6aa14443..225d19085 100644 --- a/letsencrypt-compatibility-test/MANIFEST.in +++ b/letsencrypt-compatibility-test/MANIFEST.in @@ -1 +1,4 @@ +include letsencrypt_compatibility_test/configurators/apache/a2enmod.sh +include letsencrypt_compatibility_test/configurators/apache/a2dismod.sh +include letsencrypt_compatibility_test/configurators/apache/Dockerfile recursive-include letsencrypt_compatibility_test/testdata * From d20088a43532ec503ecc3f7f7e34eba2ee120cf3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 08:55:13 +0000 Subject: [PATCH 10/31] docs: pip install -U setuptools pip --- docs/using.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index cfce29bae..16a551c1b 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -121,11 +121,12 @@ Installation ============ .. "pip install acme" doesn't search for "acme" in cwd, just like "pip - install -e acme" does + install -e acme" does; `-U setuptools pip` necessary for #722 .. code-block:: shell virtualenv --no-site-packages -p python2 venv + ./venv/bin/pip install -U setuptools pip ./venv/bin/pip install -r requirements.txt acme/ . letsencrypt-apache/ letsencrypt-nginx/ .. warning:: Please do **not** use ``python setup.py install``. Please From 5d4e1b68cddc0a45e24adbff2d56319b572fec31 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 08:57:53 +0000 Subject: [PATCH 11/31] autospec=False for socket.socket (quick-fixes: #779). Also, https://github.com/testing-cabal/mock/issues/323 --- letsencrypt/plugins/manual_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index cfe47b833..78bc4ae0e 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -68,7 +68,7 @@ class AuthenticatorTest(unittest.TestCase): mock_popen.side_effect = OSError self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) - @mock.patch("letsencrypt.plugins.manual.socket.socket", autospec=True) + @mock.patch("letsencrypt.plugins.manual.socket.socket") @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) def test_perform_test_command_run_failure( @@ -78,7 +78,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises( errors.Error, self.auth_test_mode.perform, self.achalls) - @mock.patch("letsencrypt.plugins.manual.socket.socket", autospec=True) + @mock.patch("letsencrypt.plugins.manual.socket.socket") @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify", autospec=True) From 32da607ae5580a164933b17e32599faefbe9b396 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Sun, 4 Oct 2015 10:39:38 -0500 Subject: [PATCH 12/31] crypto_util: Remove asn1_generalizedtime_to_dt(...) Not used by any other code AFAIK (ack'd entire codebase). --- letsencrypt/crypto_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 79cd24ed6..73e4f9bdd 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -274,11 +274,6 @@ def asn1_generalizedtime_to_dt(timestamp): return datetime.datetime.strptime(timestamp, '%Y%m%d%H%M%SZ') -def pyopenssl_x509_name_as_text(x509name): - """Convert `OpenSSL.crypto.X509Name` to text.""" - return "/".join("{0}={1}" for key, value in x509name.get_components()) - - def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. From 6994dad59b37d26007f14030558d560144bce81e Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Sun, 4 Oct 2015 11:17:07 -0500 Subject: [PATCH 13/31] crypto_util: Remove `asn1_generalizedtime_to_dt(...)` --- letsencrypt/crypto_util.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 73e4f9bdd..1007f18c7 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -261,19 +261,6 @@ def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): csr, OpenSSL.crypto.load_certificate_request, typ) -def asn1_generalizedtime_to_dt(timestamp): - """Convert ASN.1 GENERALIZEDTIME to datetime. - - Useful for deserialization of `OpenSSL.crypto.X509.get_notAfter` and - `OpenSSL.crypto.X509.get_notAfter` outputs. - - .. todo:: This function support only one format: `%Y%m%d%H%M%SZ`. - Implement remaining two. - - """ - return datetime.datetime.strptime(timestamp, '%Y%m%d%H%M%SZ') - - def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. From 884d8e9905527d6df4f4bf42954763f4469fe788 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Sun, 4 Oct 2015 11:28:03 -0500 Subject: [PATCH 14/31] crypto_util: Remove unused import --- letsencrypt/crypto_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 1007f18c7..2f24c4fb6 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -4,7 +4,6 @@ is capable of handling the signatures. """ -import datetime import logging import os From d5ebc38b33eabf19fa033d4c5b7b266be05639b8 Mon Sep 17 00:00:00 2001 From: Liam Marshall Date: Sun, 4 Oct 2015 20:39:19 -0500 Subject: [PATCH 15/31] Fix pep8 warnings (down to only one now!) --- letsencrypt/cli.py | 25 ++++++++++++++----------- letsencrypt/tests/cli_test.py | 2 -- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 73dd24bdb..0bd5f537e 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -729,11 +729,13 @@ def create_parser(plugins, args): return helpful.parser, helpful.args + # For now unfortunately this constant just needs to match the code below; # there isn't an elegant way to autogenerate it in time. VERBS = ["run", "auth", "install", "revoke", "rollback", "config_changes", "plugins"] HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + VERBS + def _create_subparsers(helpful): subparsers = helpful.parser.add_subparsers(metavar="SUBCOMMAND") @@ -741,7 +743,7 @@ def _create_subparsers(helpful): if name == "plugins": func = plugins_cmd else: - func = eval(name) # pylint: disable=eval-used + func = eval(name) # pylint: disable=eval-used h = func.__doc__.splitlines()[0] subparser = subparsers.add_parser(name, help=h, description=func.__doc__) subparser.set_defaults(func=func) @@ -762,22 +764,23 @@ def _create_subparsers(helpful): helpful.add_group("plugins", description="Plugin options") helpful.add("auth", - "--csr", type=read_file, help="Path to a Certificate Signing Request (CSR) in DER format.") + "--csr", type=read_file, + help="Path to a Certificate Signing Request (CSR) in DER format.") helpful.add("rollback", - "--checkpoints", type=int, metavar="N", - default=flag_default("rollback_checkpoints"), - help="Revert configuration N number of checkpoints.") + "--checkpoints", type=int, metavar="N", + default=flag_default("rollback_checkpoints"), + help="Revert configuration N number of checkpoints.") helpful.add("plugins", - "--init", action="store_true", help="Initialize plugins.") + "--init", action="store_true", help="Initialize plugins.") helpful.add("plugins", - "--prepare", action="store_true", help="Initialize and prepare plugins.") + "--prepare", action="store_true", help="Initialize and prepare plugins.") helpful.add("plugins", - "--authenticators", action="append_const", dest="ifaces", - const=interfaces.IAuthenticator, help="Limit to authenticator plugins only.") + "--authenticators", action="append_const", dest="ifaces", + const=interfaces.IAuthenticator, help="Limit to authenticator plugins only.") helpful.add("plugins", - "--installers", action="append_const", dest="ifaces", - const=interfaces.IInstaller, help="Limit to installer plugins only.") + "--installers", action="append_const", dest="ifaces", + const=interfaces.IInstaller, help="Limit to installer plugins only.") def _paths_parser(helpful): diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 0a92aba62..d0fae370d 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -57,7 +57,6 @@ class CLITest(unittest.TestCase): ret = cli.main(args) return ret, None, stderr, client - def test_no_flags(self): with mock.patch('letsencrypt.cli.run') as mock_run: self._call([]) @@ -91,7 +90,6 @@ class CLITest(unittest.TestCase): from letsencrypt import cli self.assertTrue(cli.USAGE in out) - def test_rollback(self): _, _, _, client = self._call(['rollback']) self.assertEqual(1, client.rollback.call_count) From a86ea53a795f2425b9e682b1dbba0bc44ef8da2b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Oct 2015 12:09:35 -0700 Subject: [PATCH 16/31] Added unit tests --- letsencrypt/tests/renewer_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index b7b2843d5..6f115abf9 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -407,6 +407,14 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.should_autodeploy(), result) self.assertEqual(self.test_rc.should_autorenew(), result) + def test_autodeployment_is_enabled(self): + self.assertTrue(self.test_rc.autodeployment_is_enabled()) + self.test_rc.configuration["autodeploy"] = "1" + self.assertTrue(self.test_rc.autodeployment_is_enabled()) + + self.test_rc.configuration["autodeploy"] = "0" + self.assertFalse(self.test_rc.autodeployment_is_enabled()) + def test_should_autodeploy(self): """Test should_autodeploy() on the basis of reasons other than expiry time window.""" @@ -427,6 +435,14 @@ class RenewableCertTests(BaseRenewableCertTest): f.write(kind) self.assertFalse(self.test_rc.should_autodeploy()) + def test_autorenewal_is_enabled(self): + self.assertTrue(self.test_rc.autorenewal_is_enabled()) + self.test_rc.configuration["autorenew"] = "1" + self.assertTrue(self.test_rc.autorenewal_is_enabled()) + + self.test_rc.configuration["autorenew"] = "0" + self.assertFalse(self.test_rc.autorenewal_is_enabled()) + @mock.patch("letsencrypt.storage.RenewableCert.ocsp_revoked") def test_should_autorenew(self, mock_ocsp): """Test should_autorenew on the basis of reasons other than From 1e18351041f330badd2db89a5b499ef4b03cbe5d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 5 Oct 2015 19:44:35 +0000 Subject: [PATCH 17/31] Fix #903: docs version parsing --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 2b4b2cd43..e2b360a6e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ here = os.path.abspath(os.path.dirname(__file__)) # read version number (and other metadata) from package init init_fn = os.path.join(here, '..', 'letsencrypt', '__init__.py') with codecs.open(init_fn, encoding='utf8') as fd: - meta = dict(re.findall(r"""__([a-z]+)__ = "([^"]+)""", fd.read())) + meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", fd.read())) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the From 0d880e334d3423787fb68a1fccf954750ac982a2 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 5 Oct 2015 20:06:48 +0000 Subject: [PATCH 18/31] separate pip install -U setuptools pip https://travis-ci.org/jsha/boulder/jobs/83762761#L557 Success: virtualenv --no-site-packages -p python2 ./venv ./venv/bin/pip install -U setuptools pip Downloading/unpacking distribute from https://pypi.python.org/packages/source/d/distribute/distribute-0.7.3.zip#md5=c6c59594a7b180af57af8a0cc0cf5b4a Downloading distribute-0.7.3.zip (145Kb): 145Kb downloaded Running setup.py egg_info for package distribute Downloading/unpacking pip from https://pypi.python.org/packages/source/p/pip/pip-7.1.2.tar.gz#md5=3823d2343d9f3aaab21cf9c917710196 Downloading pip-7.1.2.tar.gz (1.0Mb): 1.0Mb downloaded Running setup.py egg_info for package pip warning: no previously-included files found matching '.coveragerc' warning: no previously-included files found matching '.mailmap' warning: no previously-included files found matching '.travis.yml' warning: no previously-included files found matching 'pip/_vendor/Makefile' warning: no previously-included files found matching 'tox.ini' warning: no previously-included files found matching 'dev-requirements.txt' no previously-included directories found matching '.travis' no previously-included directories found matching 'docs/_build' no previously-included directories found matching 'contrib' no previously-included directories found matching 'tasks' no previously-included directories found matching 'tests' Downloading/unpacking setuptools>=0.7 (from distribute) Downloading setuptools-18.3.2.tar.gz (626Kb): 626Kb downloaded Running setup.py egg_info for package setuptools Installing collected packages: distribute, pip, setuptools Found existing installation: distribute 0.6.24 Uninstalling distribute: Successfully uninstalled distribute Running setup.py install for distribute Found existing installation: pip 1.1 Uninstalling pip: Successfully uninstalled pip Running setup.py install for pip Traceback (most recent call last): File "", line 1, in ImportError: No module named setuptools Complete output from command /home/travis/letsencrypt/venv/bin/python2 -c "import setuptools;__file__='/home/travis/letsencrypt/venv/build/pip/setup.py';exec(compile(open(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --single-version-externally-managed --record /tmp/pip-4lZMdG-record/install-record.txt --install-headers /home/travis/letsencrypt/venv/include/site/python2.7: Traceback (most recent call last): File "", line 1, in ImportError: No module named setuptools ---------------------------------------- Rolling back uninstall of pip Exception: Traceback (most recent call last): File "/home/travis/letsencrypt/venv/local/lib/python2.7/site-packages/pip-1.1-py2.7.egg/pip/basecommand.py", line 104, in main status = self.run(options, args) File "/home/travis/letsencrypt/venv/local/lib/python2.7/site-packages/pip-1.1-py2.7.egg/pip/commands/install.py", line 250, in run requirement_set.install(install_options, global_options) File "/home/travis/letsencrypt/venv/local/lib/python2.7/site-packages/pip-1.1-py2.7.egg/pip/req.py", line 1137, in install requirement.rollback_uninstall() File "/home/travis/letsencrypt/venv/local/lib/python2.7/site-packages/pip-1.1-py2.7.egg/pip/req.py", line 491, in rollback_uninstall self.uninstalled.rollback() File "/home/travis/letsencrypt/venv/local/lib/python2.7/site-packages/pip-1.1-py2.7.egg/pip/req.py", line 1450, in rollback pth.rollback() AttributeError: 'str' object has no attribute 'rollback' Storing complete log in /home/travis/.pip/pip.log [!] FAILURE: ./venv/bin/pip install -U setuptools pip ./venv/bin/pip install -r requirements.txt -e acme -e . -e letsencrypt-apache -e letsencrypt-nginx Traceback (most recent call last): File "./venv/bin/pip", line 5, in from pkg_resources import load_entry_point ImportError: No module named pkg_resources --- docs/using.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 16a551c1b..803a38705 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -126,7 +126,8 @@ Installation .. code-block:: shell virtualenv --no-site-packages -p python2 venv - ./venv/bin/pip install -U setuptools pip + ./venv/bin/pip install -U setuptools + ./venv/bin/pip install -U pip ./venv/bin/pip install -r requirements.txt acme/ . letsencrypt-apache/ letsencrypt-nginx/ .. warning:: Please do **not** use ``python setup.py install``. Please From eec5542cb3a479f57f6f69c407aa9aaae287e247 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 5 Oct 2015 21:39:34 +0000 Subject: [PATCH 19/31] lint archlinux bootstrap script --- bootstrap/archlinux.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh index fbe0987fe..fb54c83d5 100755 --- a/bootstrap/archlinux.sh +++ b/bootstrap/archlinux.sh @@ -1,2 +1,11 @@ #!/bin/sh -pacman -S git python2 python2-virtualenv gcc dialog augeas openssl libffi ca-certificates \ No newline at end of file +pacman -S \ + git \ + python2 \ + python2-virtualenv \ + gcc \ + dialog \ + augeas \ + openssl \ + libffi \ + ca-certificates \ From 4cd5a8e42c4a31ab7f8520de3e3759ee64aff590 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 5 Oct 2015 21:40:55 +0000 Subject: [PATCH 20/31] Archlinux bootstrap: python-virtualenv --- bootstrap/archlinux.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh index fb54c83d5..6de7c23d4 100755 --- a/bootstrap/archlinux.sh +++ b/bootstrap/archlinux.sh @@ -1,8 +1,12 @@ #!/bin/sh + +# "python-virtualenv" is Python3, but "python2-virtualenv" provides +# only "virtualenv2" binary, not "virtualenv" necessary in +# ./bootstrap/dev/_common_venv.sh pacman -S \ git \ python2 \ - python2-virtualenv \ + python-virtualenv \ gcc \ dialog \ augeas \ From 26e03dbba29beb32d8c64844f7ffb32a383706e7 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 5 Oct 2015 22:12:21 +0000 Subject: [PATCH 21/31] docs: remove venv3. --- docs/contributing.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9629ddcc5..3959ccee1 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -21,10 +21,8 @@ Start by :ref:`installing Let's Encrypt prerequisites .. code-block:: shell ./bootstrap/dev/venv.sh - ./bootstrap/dev/venv3.sh -Both of the commands suggest to activate the virtualenv (you can -activate one at a time only): +Activate the virtualenv: .. code-block:: shell From b275df13d3578cacfc8c199b7e2d54444d0298fa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Oct 2015 10:49:33 -0700 Subject: [PATCH 22/31] python2 and clarified coverage --- bootstrap/dev/venv.sh | 2 ++ docs/contributing.rst | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bootstrap/dev/venv.sh b/bootstrap/dev/venv.sh index 90088ac9b..d6cf95bb5 100755 --- a/bootstrap/dev/venv.sh +++ b/bootstrap/dev/venv.sh @@ -1,6 +1,8 @@ #!/bin/sh -xe # Developer virtualenv setup for Let's Encrypt client +export VENV_ARGS="--python python2" + ./bootstrap/dev/_venv_common.sh \ -r requirements.txt \ -e acme[testing] \ diff --git a/docs/contributing.rst b/docs/contributing.rst index 3959ccee1..3277d321a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,9 +7,9 @@ Contributing Hacking ======= -The code base, including your pull requests, **must** have 100% unit -test coverage, pass our `integration`_ tests **and** be compliant with -the :ref:`coding style `. +All changes in your pull request **must** have 100% unit test coverage, pass +our `integration`_ tests, **and** be compliant with the +:ref:`coding style `. Bootstrap From f7241af5cee6c395104dada8d0ce9c718520a522 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 18:58:06 +0000 Subject: [PATCH 23/31] Bump core coverage to 98% --- tox.cover.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.cover.sh b/tox.cover.sh index edfd9b81a..8418de9a8 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -16,7 +16,7 @@ fi cover () { if [ "$1" = "letsencrypt" ]; then - min=97 + min=98 elif [ "$1" = "acme" ]; then min=100 elif [ "$1" = "letsencrypt_apache" ]; then From ae66253ddfe621d3b6ef9da005373d6a67453e5f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 19:07:20 +0000 Subject: [PATCH 24/31] Don't save KGS in dist dir in dev release script (fixes #908). --- tools/dev-release.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 06f49f0a5..d93a6d21f 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -70,7 +70,7 @@ echo "Testing packages" cd "dist.$version" # start local PyPI python -m SimpleHTTPServer $PORT & -# cd .. is NOT done on purpose: we make sure that all subpacakges are +# cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) virtualenv --no-site-packages ../venv . ../venv/bin/activate @@ -82,15 +82,16 @@ pip install \ # stop local PyPI kill $! -# freeze before installing anythin else, so that we know end-user KGS -mkdir kgs -kgs="kgs/$version" +# freeze before installing anything else, so that we know end-user KGS +# make sure "twine upload" doesn't catch "kgs" +mkdir ../kgs +kgs="../kgs/$version" pip freeze | tee $kgs pip install nose # TODO: letsencrypt_apache fails due to symlink, c.f. #838 nosetests letsencrypt $SUBPKGS || true echo "New root: $root" -echo "KGS is at $root/$kgs" +echo "KGS is at $root/kgs" echo "In order to upload packages run the following command:" echo twine upload "$root/dist.$version/*/*" From 7e1b7ff7ae0d6f8b187b1ff19e5a94c04cb72bfd Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 20:56:03 +0000 Subject: [PATCH 25/31] Add naive JWK Thumbprint implementation --- acme/acme/jose/jwk.py | 31 +++++++++++++++++++++++++++++++ acme/acme/jose/jwk_test.py | 23 +++++++++++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py index 7a976f189..2b8fa0a34 100644 --- a/acme/acme/jose/jwk.py +++ b/acme/acme/jose/jwk.py @@ -1,10 +1,12 @@ """JSON Web Key.""" import abc import binascii +import json import logging import cryptography.exceptions from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives.asymmetric import rsa @@ -27,6 +29,32 @@ class JWK(json_util.TypedJSONObjectWithFields): cryptography_key_types = () """Subclasses should override.""" + required = NotImplemented + """Required members of public key's representation as defined by JWK/JWA.""" + + _thumbprint_json_dumps_params = { + # "no whitespace or line breaks before or after any syntactic + # elements" + 'indent': 0, + 'separators': (',', ':'), + # "members ordered lexicographically by the Unicode [UNICODE] + # code points of the member names" + 'sort_keys': True, + } + + def thumbprint(self, hash_function=hashes.SHA256): + """Compute JWK Thumbprint. + + https://tools.ietf.org/html/rfc7638 + + """ + digest = hashes.Hash(hash_function(), backend=default_backend()) + digest.update(json.dumps( + dict((k, v) for k, v in six.iteritems(self.to_json()) + if k in self.required), + **self._thumbprint_json_dumps_params).encode()) + return digest.finalize() + @abc.abstractmethod def public_key(self): # pragma: no cover """Generate JWK with public key. @@ -105,6 +133,7 @@ class JWKES(JWK): # pragma: no cover typ = 'ES' cryptography_key_types = ( ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) + required = ('crv', JWK.type_field_name, 'x', 'y') def fields_to_partial_json(self): raise NotImplementedError() @@ -122,6 +151,7 @@ class JWKOct(JWK): """Symmetric JWK.""" typ = 'oct' __slots__ = ('key',) + required = ('k', JWK.type_field_name) def fields_to_partial_json(self): # TODO: An "alg" member SHOULD also be present to identify the @@ -150,6 +180,7 @@ class JWKRSA(JWK): typ = 'RSA' cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) __slots__ = ('key',) + required = ('e', JWK.type_field_name, 'n') def __init__(self, *args, **kwargs): if 'key' in kwargs and not isinstance( diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py index 5462af6b0..d8a7410e8 100644 --- a/acme/acme/jose/jwk_test.py +++ b/acme/acme/jose/jwk_test.py @@ -25,9 +25,24 @@ class JWKTest(unittest.TestCase): self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) -class JWKOctTest(unittest.TestCase): +class JWKTestBaseMixin(object): + """Mixin test for JWK subclass tests.""" + + thumbprint = NotImplemented + + def test_thumbprint_private(self): + self.assertEqual(self.thumbprint, self.jwk.thumbprint()) + + def test_thumbprint_public(self): + self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) + + +class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): """Tests for acme.jose.jwk.JWKOct.""" + thumbprint = (b"=,\xdd;I\x1a+i\x02x\x8a\x12?06IM\xc2\x80" + b"\xe4\xc3\x1a\xfc\x89\xf3)'\xce\xccm\xfd5") + def setUp(self): from acme.jose.jwk import JWKOct self.jwk = JWKOct(key=b'foo') @@ -52,10 +67,13 @@ class JWKOctTest(unittest.TestCase): self.assertTrue(self.jwk.public_key() is self.jwk) -class JWKRSATest(unittest.TestCase): +class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): """Tests for acme.jose.jwk.JWKRSA.""" # pylint: disable=too-many-instance-attributes + thumbprint = (b'\x08\xfa1\x87\x1d\x9b6H/*\x1eW\xc2\xe3\xf6P' + b'\xefs\x0cKB\x87\xcf\x85yO\x045\x0e\x91\x80\x0b') + def setUp(self): from acme.jose.jwk import JWKRSA self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) @@ -87,6 +105,7 @@ class JWKRSATest(unittest.TestCase): 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', 'qi': 'oi45cEkbVoJjAbnQpFY87Q', }) + self.jwk = self.private def test_init_auto_comparable(self): self.assertTrue(isinstance( From 3dac62f20e8dbfacea32df6b1748d5f560b559e3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 20:56:31 +0000 Subject: [PATCH 26/31] json_dumps_pretty: prettier separators. --- acme/acme/jose/interfaces.py | 2 +- acme/acme/jose/interfaces_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py index f841848b3..f85777a30 100644 --- a/acme/acme/jose/interfaces.py +++ b/acme/acme/jose/interfaces.py @@ -194,7 +194,7 @@ class JSONDeSerializable(object): :rtype: str """ - return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) + return self.json_dumps(sort_keys=True, indent=4) @classmethod def json_dump_default(cls, python_object): diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py index 380c3a2a5..91e6f4416 100644 --- a/acme/acme/jose/interfaces_test.py +++ b/acme/acme/jose/interfaces_test.py @@ -91,7 +91,7 @@ class JSONDeSerializableTest(unittest.TestCase): def test_json_dumps_pretty(self): self.assertEqual( - self.seq.json_dumps_pretty(), '[\n "foo1",\n "foo2"\n]') + self.seq.json_dumps_pretty(), '[\n "foo1", \n "foo2"\n]') def test_json_dump_default(self): from acme.jose.interfaces import JSONDeSerializable From c6ebfae15e91a232648e41c4a50ea93d01a15d19 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 6 Oct 2015 20:57:33 +0000 Subject: [PATCH 27/31] Unify quotes --- acme/acme/jose/jwk.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py index 2b8fa0a34..74fa72319 100644 --- a/acme/acme/jose/jwk.py +++ b/acme/acme/jose/jwk.py @@ -88,7 +88,7 @@ class JWK(json_util.TypedJSONObjectWithFields): exceptions[loader] = error # no luck - raise errors.Error("Unable to deserialize key: {0}".format(exceptions)) + raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) @classmethod def load(cls, data, password=None, backend=None): @@ -109,17 +109,17 @@ class JWK(json_util.TypedJSONObjectWithFields): try: key = cls._load_cryptography_key(data, password, backend) except errors.Error as error: - logger.debug("Loading symmetric key, assymentric failed: %s", error) + logger.debug('Loading symmetric key, assymentric failed: %s', error) return JWKOct(key=data) if cls.typ is not NotImplemented and not isinstance( key, cls.cryptography_key_types): - raise errors.Error("Unable to deserialize {0} into {1}".format( + raise errors.Error('Unable to deserialize {0} into {1}'.format( key.__class__, cls.__class__)) for jwk_cls in six.itervalues(cls.TYPES): if isinstance(key, jwk_cls.cryptography_key_types): return jwk_cls(key=key) - raise errors.Error("Unsupported algorithm: {0}".format(key.__class__)) + raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) @JWK.register @@ -235,7 +235,7 @@ class JWKRSA(JWK): jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) if tuple(param for param in all_params if param is None): raise errors.Error( - "Some private parameters are missing: {0}".format( + 'Some private parameters are missing: {0}'.format( all_params)) p, q, dp, dq, qi = tuple( cls._decode_param(x) for x in all_params) From 0d89fa6d88e02257da88403898a35ffcec225947 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 06:21:49 +0000 Subject: [PATCH 28/31] Remove SimpleHTTP TLS from Manual Plugin. --- letsencrypt/plugins/manual.py | 34 +++++------------------------- letsencrypt/plugins/manual_test.py | 8 +++---- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 3f7276725..9d5ef87e9 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -53,7 +53,7 @@ command on the target server (as root): # served and makes it more obvious that Python command will serve # anything recursively under the cwd - HTTP_TEMPLATE = """\ + CMD_TEMPLATE = """\ mkdir -p {root}/public_html/{response.URI_ROOT_PATH} cd {root}/public_html echo -n {validation} > {response.URI_ROOT_PATH}/{encoded_token} @@ -63,33 +63,10 @@ $(command -v python2 || command -v python2.7 || command -v python2.6) -c \\ SimpleHTTPServer.SimpleHTTPRequestHandler.extensions_map = {{'': '{ct}'}}; \\ s = BaseHTTPServer.HTTPServer(('', {port}), SimpleHTTPServer.SimpleHTTPRequestHandler); \\ s.serve_forever()" """ - """Non-TLS command template.""" - - # https://www.piware.de/2011/01/creating-an-https-server-in-python/ - HTTPS_TEMPLATE = """\ -mkdir -p {root}/public_html/{response.URI_ROOT_PATH} -cd {root}/public_html -echo -n {validation} > {response.URI_ROOT_PATH}/{encoded_token} -# run only once per server: -openssl req -new -newkey rsa:4096 -subj "/" -days 1 -nodes -x509 -keyout ../key.pem -out ../cert.pem -$(command -v python2 || command -v python2.7 || command -v python2.6) -c \\ -"import BaseHTTPServer, SimpleHTTPServer, ssl; \\ -SimpleHTTPServer.SimpleHTTPRequestHandler.extensions_map = {{'': '{ct}'}}; \\ -s = BaseHTTPServer.HTTPServer(('', {port}), SimpleHTTPServer.SimpleHTTPRequestHandler); \\ -s.socket = ssl.wrap_socket(s.socket, keyfile='../key.pem', certfile='../cert.pem'); \\ -s.serve_forever()" """ - """TLS command template. - - According to the ACME specification, "the ACME server MUST ignore - the certificate provided by the HTTPS server", so the first command - generates temporary self-signed certificate. - - """ + """Command template.""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.template = (self.HTTP_TEMPLATE if self.config.no_simple_http_tls - else self.HTTPS_TEMPLATE) self._root = (tempfile.mkdtemp() if self.conf("test-mode") else "/tmp/letsencrypt") self._httpd = None @@ -97,8 +74,7 @@ s.serve_forever()" """ @classmethod def add_parser_arguments(cls, add): add("test-mode", action="store_true", - help="Test mode. Executes the manual command in subprocess. " - "Requires openssl to be installed unless --no-simple-http-tls.") + help="Test mode. Executes the manual command in subprocess.") def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -142,11 +118,11 @@ binary for temporary key/certificate generation.""".replace("\n", "") # users, but will not work if multiple domains point at the # same server: default command doesn't support virtual hosts response, validation = achall.gen_response_and_validation( - tls=(not self.config.no_simple_http_tls)) + tls=False) # SimpleHTTP TLS is dead: ietf-wg-acme/acme#7 port = (response.port if self.config.simple_http_port is None else int(self.config.simple_http_port)) - command = self.template.format( + command = self.CMD_TEMPLATE.format( root=self._root, achall=achall, response=response, validation=pipes.quote(validation.json_dumps()), encoded_token=achall.chall.encode("token"), diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index 78bc4ae0e..8cfff1cc5 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -23,15 +23,13 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.manual import Authenticator self.config = mock.MagicMock( - no_simple_http_tls=True, simple_http_port=4430, - manual_test_mode=False) + simple_http_port=8080, manual_test_mode=False) self.auth = Authenticator(config=self.config, name="manual") self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP_P, domain="foo.com", account_key=KEY)] config_test_mode = mock.MagicMock( - no_simple_http_tls=True, simple_http_port=4430, - manual_test_mode=True) + simple_http_port=8080, manual_test_mode=True) self.auth_test_mode = Authenticator( config=config_test_mode, name="manual") @@ -55,7 +53,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertEqual([resp], self.auth.perform(self.achalls)) self.assertEqual(1, mock_raw_input.call_count) mock_verify.assert_called_with( - self.achalls[0].challb.chall, "foo.com", KEY.public_key(), 4430) + self.achalls[0].challb.chall, "foo.com", KEY.public_key(), 8080) message = mock_stdout.write.mock_calls[0][1][0] self.assertTrue(self.achalls[0].chall.encode("token") in message) From 73ee63779c94db8bb270415fb209e016cee7c2c4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 06:23:28 +0000 Subject: [PATCH 29/31] Remove --no-simple-http-tls --- letsencrypt/cli.py | 2 -- letsencrypt/interfaces.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0bd5f537e..64cba508d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -702,8 +702,6 @@ def create_parser(plugins, args): help=config_help("dvsni_port")) helpful.add("testing", "--simple-http-port", type=int, help=config_help("simple_http_port")) - helpful.add("testing", "--no-simple-http-tls", action="store_true", - help=config_help("no_simple_http_tls")) helpful.add_group( "security", description="Security parameters & server settings") diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index 1f51645ab..5e82d61aa 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -223,8 +223,6 @@ class IConfig(zope.interface.Interface): "Port number to perform DVSNI challenge. " "Boulder in testing mode defaults to 5001.") - no_simple_http_tls = zope.interface.Attribute( - "Do not use TLS when solving SimpleHTTP challenges.") simple_http_port = zope.interface.Attribute( "Port used in the SimpleHttp challenge.") From e4e94b20d44dde81e1a9f0a532ef447f04da8c48 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 06:23:40 +0000 Subject: [PATCH 30/31] Remove --no-simple-http-tls from integration tests --- tests/boulder-integration.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ed877d136..25db8ba6d 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -24,7 +24,6 @@ common() { common --domains le1.wtf auth common --domains le2.wtf run common -a manual -d le.wtf auth -common -a manual -d le.wtf --no-simple-http-tls auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf From 0034a8fae45131e46a27128575eff02f51f79f2c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 22:40:02 +0000 Subject: [PATCH 31/31] Add docs to tarballs (fixes #884). --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 80fd8777e..e421e0cd7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,5 @@ include CONTRIBUTING.md include LICENSE.txt include linter_plugin.py include letsencrypt/EULA +recursive-include docs * recursive-include letsencrypt/tests/testdata *