From 03a9a2a89e039c607bb80ec2742d450ac8493165 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 2 Sep 2015 19:52:06 +0000 Subject: [PATCH 01/91] SimpleFS plugin (fixes #742) --- letsencrypt/plugins/common.py | 4 ++ letsencrypt/plugins/simplefs.py | 81 +++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 86 insertions(+) create mode 100644 letsencrypt/plugins/simplefs.py diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index bef8b4d81..3ec1f1f7c 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -49,6 +49,10 @@ class Plugin(object): """ArgumentParser dest namespace (prefix of all destinations).""" return dest_namespace(self.name) + def option_name(self, name): + """Option name (include plugin namespace).""" + return self.option_namespace + name + def dest(self, var): """Find a destination for given variable ``var``.""" # this should do exactly the same what ArgumentParser(arg), diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py new file mode 100644 index 000000000..8bff1946e --- /dev/null +++ b/letsencrypt/plugins/simplefs.py @@ -0,0 +1,81 @@ +"""SimpleFS plugin.""" +import errno +import logging +import os + +import zope.interface + +from acme import challenges + +from letsencrypt import errors +from letsencrypt import interfaces +from letsencrypt.plugins import common + + +logger = logging.getLogger(__name__) + + +class Authenticator(common.Plugin): + """SimpleFS Authenticator.""" + zope.interface.implements(interfaces.IAuthenticator) + zope.interface.classProvides(interfaces.IPluginFactory) + + description = "SimpleFS Authenticator" + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return """\ +Authenticator plugin that performs SimpleHTTP challenge by saving +necessary validation resources to appropriate paths on the file +system. It expects that there is some other HTTP server configured +to serve all files under specified web root ({0}).""".format( + self.option_name("root")) + + @classmethod + def add_parser_arguments(cls, add): + add("root", help="public_html / webroot path") + + def get_chall_pref(self, domain): + # pylint: disable=missing-docstring,no-self-use + return [challenges.SimpleHTTP] + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + + root = self.conf("root") + if root is None: + raise errors.Error("--{0} must be set".format( + self.option_name("root"))) + if not os.path.isdir(root): + raise errors.Error(root + " does not exist or is not a directory") + self.full_root = os.path.join( + root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) + + def prepare(self): # pylint: disable=missing-docstring + logger.debug("Creating root challenges validation dir at %s", + self.full_root) + try: + os.makedirs(self.full_root) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + def perform(self, achalls): # pylint: disable=missing-docstring + return [self._perform_single(achall) for achall in achalls] + + def _path_for_achall(self, achall): + return os.path.join(self.full_root, achall.chall.encode("token")) + + def _perform_single(self, achall): + response, validation = achall.gen_response_and_validation( + tls=(not self.config.no_simple_http_tls)) + path = self._path_for_achall(achall) + logger.debug("Attempting to save validation to %s", path) + with open(path, "w") as validation_file: + validation_file.write(validation.json_dumps()) + return response + + def cleanup(self, achalls): + for achall in achalls: + path = self._path_for_achall(achall) + logger.debug("Removing %s", path) + os.remove(path) diff --git a/setup.py b/setup.py index f816c6c56..bd954a5d6 100644 --- a/setup.py +++ b/setup.py @@ -118,6 +118,7 @@ setup( 'manual = letsencrypt.plugins.manual:ManualAuthenticator', # TODO: null should probably not be presented to the user 'null = letsencrypt.plugins.null:Installer', + 'simplefs = letsencrypt.plugins.simplefs:Authenticator', 'standalone = letsencrypt.plugins.standalone.authenticator' ':StandaloneAuthenticator', ], From 058a85eafdb80d2a442ed07f8a24ed43e84c37f5 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 6 Sep 2015 13:17:29 +0000 Subject: [PATCH 02/91] satisfy lint --- letsencrypt/plugins/simplefs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py index 8bff1946e..67e59983e 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/simplefs.py @@ -22,20 +22,21 @@ class Authenticator(common.Plugin): description = "SimpleFS Authenticator" - def more_info(self): # pylint: disable=missing-docstring,no-self-use - return """\ + MORE_INFO = """\ Authenticator plugin that performs SimpleHTTP challenge by saving necessary validation resources to appropriate paths on the file system. It expects that there is some other HTTP server configured -to serve all files under specified web root ({0}).""".format( - self.option_name("root")) +to serve all files under specified web root ({0}).""" + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return self.MORE_INFO.format(self.conf("root")) @classmethod def add_parser_arguments(cls, add): add("root", help="public_html / webroot path") def get_chall_pref(self, domain): - # pylint: disable=missing-docstring,no-self-use + # pylint: disable=missing-docstring,no-self-use,unused-argument return [challenges.SimpleHTTP] def __init__(self, *args, **kwargs): @@ -74,7 +75,7 @@ to serve all files under specified web root ({0}).""".format( validation_file.write(validation.json_dumps()) return response - def cleanup(self, achalls): + def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: path = self._path_for_achall(achall) logger.debug("Removing %s", path) From 57f6979f67a7f6e5765cb3d02c268853a90dd10d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 6 Sep 2015 14:03:21 +0000 Subject: [PATCH 03/91] Add tests for SimpleFS plugin. --- letsencrypt/plugins/common.py | 8 +-- letsencrypt/plugins/common_test.py | 3 + letsencrypt/plugins/simplefs.py | 15 +++-- letsencrypt/plugins/simplefs_test.py | 82 ++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 letsencrypt/plugins/simplefs_test.py diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 3ec1f1f7c..8c4d618b8 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -44,15 +44,15 @@ class Plugin(object): """ArgumentParser options namespace (prefix of all options).""" return option_namespace(self.name) + def option_name(self, name): + """Option name (include plugin namespace).""" + return self.option_namespace + name + @property def dest_namespace(self): """ArgumentParser dest namespace (prefix of all destinations).""" return dest_namespace(self.name) - def option_name(self, name): - """Option name (include plugin namespace).""" - return self.option_namespace + name - def dest(self, var): """Find a destination for given variable ``var``.""" # this should do exactly the same what ArgumentParser(arg), diff --git a/letsencrypt/plugins/common_test.py b/letsencrypt/plugins/common_test.py index fa761839c..9c6df8c9e 100644 --- a/letsencrypt/plugins/common_test.py +++ b/letsencrypt/plugins/common_test.py @@ -50,6 +50,9 @@ class PluginTest(unittest.TestCase): def test_option_namespace(self): self.assertEqual("mock-", self.plugin.option_namespace) + def test_option_name(self): + self.assertEqual("mock-foo_bar", self.plugin.option_name("foo_bar")) + def test_dest_namespace(self): self.assertEqual("mock_", self.plugin.dest_namespace) diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/simplefs.py index 67e59983e..ad83c13d7 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/simplefs.py @@ -35,32 +35,37 @@ to serve all files under specified web root ({0}).""" def add_parser_arguments(cls, add): add("root", help="public_html / webroot path") - def get_chall_pref(self, domain): + def get_chall_pref(self, domain): # pragma: no cover # pylint: disable=missing-docstring,no-self-use,unused-argument return [challenges.SimpleHTTP] def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) + self.full_root = None + def prepare(self): # pylint: disable=missing-docstring root = self.conf("root") if root is None: - raise errors.Error("--{0} must be set".format( + raise errors.PluginError("--{0} must be set".format( self.option_name("root"))) if not os.path.isdir(root): - raise errors.Error(root + " does not exist or is not a directory") + raise errors.PluginError( + root + " does not exist or is not a directory") self.full_root = os.path.join( root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) - def prepare(self): # pylint: disable=missing-docstring logger.debug("Creating root challenges validation dir at %s", self.full_root) try: os.makedirs(self.full_root) except OSError as exception: if exception.errno != errno.EEXIST: - raise + raise errors.PluginError( + "Couldn't create root for SimpleHTTP " + "challenge responses: {0}", exception) def perform(self, achalls): # pylint: disable=missing-docstring + assert self.full_root is not None return [self._perform_single(achall) for achall in achalls] def _path_for_achall(self, achall): diff --git a/letsencrypt/plugins/simplefs_test.py b/letsencrypt/plugins/simplefs_test.py new file mode 100644 index 000000000..f80e8b29a --- /dev/null +++ b/letsencrypt/plugins/simplefs_test.py @@ -0,0 +1,82 @@ +"""Tests for letsencrypt.plugins.simplefs.""" +import os +import shutil +import tempfile +import unittest + +import mock + +from acme import jose + +from letsencrypt import achallenges +from letsencrypt import errors + +from letsencrypt.tests import acme_util +from letsencrypt.tests import test_util + + +KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + + +class AuthenticatorTest(unittest.TestCase): + """Tests for letsencrypt.plugins.simplefs.Authenticator.""" + + achall = achallenges.SimpleHTTP( + challb=acme_util.SIMPLE_HTTP_P, domain=None, account_key=KEY) + + def setUp(self): + from letsencrypt.plugins.simplefs import Authenticator + self.root = tempfile.mkdtemp() + self.validation_path = os.path.join( + self.root, ".well-known", "acme-challenge", + "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") + self.config = mock.MagicMock(simplefs_root=self.root) + self.auth = Authenticator(self.config, "simplefs") + self.auth.prepare() + + def tearDown(self): + shutil.rmtree(self.root) + + def test_more_info(self): + more_info = self.auth.more_info() + self.assertTrue(isinstance(more_info, str)) + self.assertTrue(self.root in more_info) + + def test_add_parser_arguments(self): + add = mock.MagicMock() + self.auth.add_parser_arguments(add) + self.assertEqual(1, add.call_count) + + def test_prepare_bad_root(self): + self.config.simplefs_root = os.path.join(self.root, "null") + self.assertRaises(errors.PluginError, self.auth.prepare) + + def test_prepare_missing_root(self): + self.config.simplefs_root = None + self.assertRaises(errors.PluginError, self.auth.prepare) + + def test_prepare_full_root_exists(self): + # prepare() has already been called once in setUp() + self.auth.prepare() # shouldn't raise any exceptions + + def test_prepare_reraises_other_errors(self): + self.auth.full_root = os.path.join(self.root, "null") + os.chmod(self.root, 0o000) + self.assertRaises(errors.PluginError, self.auth.prepare) + os.chmod(self.root, 0o700) + + def test_perform_cleanup(self): + responses = self.auth.perform([self.achall]) + self.assertEqual(1, len(responses)) + self.assertTrue(os.path.exists(self.validation_path)) + with open(self.validation_path) as validation_f: + validation = jose.JWS.json_loads(validation_f.read()) + self.assertTrue(responses[0].check_validation( + validation, self.achall.chall, KEY.public_key())) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(self.validation_path)) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From 8c7b8b835117fe5cca92367d9244c4dd480c4ec6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Mon, 7 Sep 2015 06:33:16 +0000 Subject: [PATCH 04/91] Add docs for SimpleFS plugin --- docs/api/plugins/simplefs.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/plugins/simplefs.rst diff --git a/docs/api/plugins/simplefs.rst b/docs/api/plugins/simplefs.rst new file mode 100644 index 000000000..7165b6aca --- /dev/null +++ b/docs/api/plugins/simplefs.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.simplefs` +----------------------------------- + +.. automodule:: letsencrypt.plugins.simplefs + :members: From d73b600eeb4cf4ea602352fd7b786f26cbeb471a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:55:27 +0000 Subject: [PATCH 05/91] acme: _serve_sni -> SSLSocket --- acme/acme/crypto_util.py | 106 ++++++++++++++++++++++++---------- acme/acme/crypto_util_test.py | 44 +++++++------- 2 files changed, 98 insertions(+), 52 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 030946f82..32533630b 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -26,47 +26,95 @@ logger = logging.getLogger(__name__) _DEFAULT_DVSNI_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD -def _serve_sni(certs, sock, reuseaddr=True, method=_DEFAULT_DVSNI_SSL_METHOD, - accept=None): - """Start SNI-enabled server, that drops connection after handshake. +class SSLSocket(object): # pylint: disable=too-few-public-methods + """SSL wrapper for sockets.""" - :param certs: Mapping from SNI name to ``(key, cert)`` `tuple`. - :param sock: Already bound socket. - :param bool reuseaddr: Should `socket.SO_REUSEADDR` be set? - :param method: See `OpenSSL.SSL.Context` for allowed values. - :param accept: Callable that doesn't take any arguments and - returns ``True`` if more connections should be served. + def __init__(self, sock, certs, method=_DEFAULT_DVSNI_SSL_METHOD): + self.sock = sock + self.certs = certs + self.method = method - """ - def _pick_certificate(connection): + def __getattr__(self, name): + return getattr(self.sock, name) + + def _pick_certificate_cb(self, connection): + """SNI certificate callback. + + This method will set a new OpenSSL context object for this + connection when an incoming connection provides an SNI name + (in order to serve the appropriate certificate, if any). + + :param connection: The TLS connection object on which the SNI + extension was received. + :type connection: :class:`OpenSSL.Connection` + + """ + server_name = connection.get_servername() try: - key, cert = certs[connection.get_servername()] + key, cert = self.certs[server_name] except KeyError: + logger.debug("Server name (%s) not recognized, dropping SSL", + server_name) return - new_context = OpenSSL.SSL.Context(method) + new_context = OpenSSL.SSL.Context(self.method) new_context.use_privatekey(key) new_context.use_certificate(cert) connection.set_context(new_context) - if reuseaddr: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.listen(1) # TODO: add func arg? + class FakeConnection(object): + """Fake OpenSSL.SSL.Connection.""" - while accept is None or accept(): - server, addr = sock.accept() - logger.debug('Received connection from %s', addr) + # pylint: disable=missing-docstring - with contextlib.closing(server): - context = OpenSSL.SSL.Context(method) - context.set_tlsext_servername_callback(_pick_certificate) + def __init__(self, connection): + self._wrapped = connection + self._makefile_refs = 0 - server_ssl = OpenSSL.SSL.Connection(context, server) - server_ssl.set_accept_state() - try: - server_ssl.do_handshake() - server_ssl.shutdown() - except OpenSSL.SSL.Error as error: - raise errors.Error(error) + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def shutdown(self, *unused_args): + # OpenSSL.SSL.Connection.shutdown doesn't accept any args + return self._wrapped.shutdown() + + # stuff below ripped off from + # https://hg.python.org/cpython/file/2.7/Lib/ssl.py + # XXX: this uses Python's internal API + + def makefile(self, mode='r', bufsize=-1): + self._makefile_refs += 1 + # SocketServer.StreamRequesthandler.finish will try to + # close the wfile/rfile. close=True causes curl: (56) + # GnuTLS recv error (-110): The TLS connection was + # non-properly terminated. + # TODO: doesn't work in Python3 + # pylint: disable=protected-access + return socket._fileobject(self._wrapped, mode, bufsize, close=False) + + def close(self): + if self._makefile_refs < 1: + self._wrapped.close() + else: + self._makefile_refs -= 1 + + def accept(self): # pylint: disable=missing-docstring + sock, addr = self.sock.accept() + + context = OpenSSL.SSL.Context(self.method) + context.set_tlsext_servername_callback(self._pick_certificate_cb) + + ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock)) + ssl_sock.set_accept_state() + + logger.debug("Performing handshake with %s", addr) + try: + ssl_sock.do_handshake() + except OpenSSL.SSL.Error as error: + # _pick_certificate_cb might have returned without + # creating SSL context (wrong server name) + raise socket.error(error) + + return ssl_sock, addr def probe_sni(name, host, port=443, timeout=300, diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 64c7cb552..bfd16388c 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -4,45 +4,43 @@ import threading import time import unittest -import mock -import OpenSSL +from six.moves import socketserver # pylint: disable=import-error from acme import errors from acme import jose from acme import test_util -class ServeProbeSNITest(unittest.TestCase): - """Tests for acme.crypto_util._serve_sni/probe_sni.""" +class SSLSocketAndProbeSNITest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket/probe_sni.""" def setUp(self): self.cert = test_util.load_cert('cert.pem') - key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, - test_util.load_vector('rsa512_key.pem')) + key = test_util.load_pyopenssl_private_key('rsa512_key.pem') # pylint: disable=protected-access certs = {b'foo': (key, self.cert._wrapped)} - sock = socket.socket() - sock.bind(('', 0)) # pick random port - self.port = sock.getsockname()[1] + from acme.crypto_util import SSLSocket - self.server = threading.Thread(target=self._run_server, args=(certs, sock)) - self.server.start() + class _TestServer(socketserver.TCPServer): + + # pylint: disable=too-few-public-methods + # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init + + def server_bind(self): # pylint: disable=missing-docstring + self.socket = SSLSocket(socket.socket(), certs=certs) + socketserver.TCPServer.server_bind(self) + + self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) + self.port = self.server.socket.getsockname()[1] + self.server_thread = threading.Thread( + # pylint: disable=no-member + target=self.server.handle_request) + self.server_thread.start() time.sleep(1) # TODO: avoid race conditions in other way - @classmethod - def _run_server(cls, certs, sock): - from acme.crypto_util import _serve_sni - # TODO: improve testing of server errors and their conditions - try: - return _serve_sni( - certs, sock, accept=mock.Mock(side_effect=[True, False])) - except errors.Error: - pass - def tearDown(self): - self.server.join() + self.server_thread.join() def _probe(self, name): from acme.crypto_util import probe_sni From 1b24fdae84b7fa17823df4338901b1712950e7ae Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:56:44 +0000 Subject: [PATCH 06/91] acme: challenges helpers --- acme/acme/challenges.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 13186cc4f..a16cc6f89 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -54,6 +54,9 @@ class SimpleHTTP(DVChallenge): TOKEN_SIZE = 128 / 8 # Based on the entropy value from the spec """Minimum size of the :attr:`token` in bytes.""" + URI_ROOT_PATH = ".well-known/acme-challenge" + """URI root path for the server provisioned resource.""" + # TODO: acme-spec doesn't specify token as base64-encoded value token = jose.Field( "token", encoder=jose.encode_b64jose, decoder=functools.partial( @@ -72,6 +75,11 @@ class SimpleHTTP(DVChallenge): # URI_ROOT_PATH! return b'..' not in self.token and b'/' not in self.token + @property + def path(self): + """Path (starting with '/') for provisioned resource.""" + return '/' + self.URI_ROOT_PATH + '/' + self.encode('token') + @ChallengeResponse.register class SimpleHTTPResponse(ChallengeResponse): @@ -83,12 +91,12 @@ class SimpleHTTPResponse(ChallengeResponse): typ = "simpleHttp" tls = jose.Field("tls", default=True, omitempty=True) - URI_ROOT_PATH = ".well-known/acme-challenge" - """URI root path for the server provisioned resource.""" - + URI_ROOT_PATH = SimpleHTTP.URI_ROOT_PATH _URI_TEMPLATE = "{scheme}://{domain}/" + URI_ROOT_PATH + "/{token}" CONTENT_TYPE = "application/jose+json" + PORT = 80 + TLS_PORT = 443 @property def scheme(self): @@ -98,7 +106,7 @@ class SimpleHTTPResponse(ChallengeResponse): @property def port(self): """Port that the ACME client should be listening for validation.""" - return 443 if self.tls else 80 + return self.TLS_PORT if self.tls else self.PORT def uri(self, domain, chall): """Create an URI to the provisioned resource. From daa459f2778816225107a6e6cddc3e66ef17318d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 14:57:07 +0000 Subject: [PATCH 07/91] Add acme.standalone --- acme/acme/standalone.py | 188 ++++++++++++++++++++ acme/acme/standalone_test.py | 128 +++++++++++++ acme/examples/standalone/README | 2 + acme/examples/standalone/localhost/cert.pem | 1 + acme/examples/standalone/localhost/key.pem | 1 + docs/pkgs/acme/index.rst | 7 + 6 files changed, 327 insertions(+) create mode 100644 acme/acme/standalone.py create mode 100644 acme/acme/standalone_test.py create mode 100644 acme/examples/standalone/README create mode 120000 acme/examples/standalone/localhost/cert.pem create mode 120000 acme/examples/standalone/localhost/key.pem diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py new file mode 100644 index 000000000..8f604a439 --- /dev/null +++ b/acme/acme/standalone.py @@ -0,0 +1,188 @@ +"""Support for standalone client challenge solvers. """ +import argparse +import collections +import functools +import logging +import os +import sys + +import six +from six.moves import BaseHTTPServer # pylint: disable=import-error +from six.moves import http_client # pylint: disable=import-error +from six.moves import socketserver # pylint: disable=import-error + +import OpenSSL + +from acme import challenges +from acme import crypto_util + + +logger = logging.getLogger(__name__) + +# six.moves.* | pylint: disable=no-member,attribute-defined-outside-init +# pylint: disable=too-few-public-methods,no-init + + +class TLSServer(socketserver.TCPServer): + """Generic TLS Server.""" + + def __init__(self, *args, **kwargs): + self.certs = kwargs.pop("certs", {}) + self.method = kwargs.pop( + # pylint: disable=protected-access + "method", crypto_util._DEFAULT_DVSNI_SSL_METHOD) + self.allow_reuse_address = kwargs.pop("allow_reuse_address", True) + socketserver.TCPServer.__init__(self, *args, **kwargs) + + def _wrap_sock(self): + self.socket = crypto_util.SSLSocket( + self.socket, certs=self.certs, method=self.method) + + def server_bind(self): # pylint: disable=missing-docstring + self._wrap_sock() + return socketserver.TCPServer.server_bind(self) + + +class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): + """HTTPS Server.""" + + def server_bind(self): + self._wrap_sock() + BaseHTTPServer.HTTPServer.server_bind(self) + + +class ACMEServerMixin: # pylint: disable=old-style-class,no-init + """ACME server common settings mixin.""" + server_version = "ACME standalone client" + allow_reuse_address = True + + +class ACMETLSServer(HTTPSServer, ACMEServerMixin): + """ACME TLS Server.""" + + +class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): + """ACME Server (non-TLS).""" + + +class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """SimpleHTTP challenge handler. + + Adheres to the stdlib"s `socketserver.BaseRequestHandler` interface. + + :ivar set simple_http_resources: A set of `SimpleHTTPResource` + objects. TODO: better name? + + """ + SimpleHTTPResource = collections.namedtuple( + "SimpleHTTPResource", "chall response validation") + + def __init__(self, *args, **kwargs): + self.simple_http_resources = kwargs.pop("simple_http_resources", set()) + socketserver.BaseRequestHandler.__init__(self, *args, **kwargs) + + def do_GET(self): # pylint: disable=invalid-name,missing-docstring + if self.path == "/": + self.handle_index() + elif self.path.startswith("/" + challenges.SimpleHTTP.URI_ROOT_PATH): + self.handle_simple_http_resource() + else: + self.handle_404() + + def handle_index(self): + """Handle index page.""" + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(self.server.server_version) + + def handle_404(self): + """Handler 404 Not Found errors.""" + self.send_response(http_client.NOT_FOUND, message="Not Found") + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write("404") + + def handle_simple_http_resource(self): + """Handle SimpleHTTP provisioned resources.""" + for resource in self.simple_http_resources: + if resource.chall.path == self.path: + logger.debug("Serving SimpleHTTP with token %r", + resource.chall.encode("token")) + self.send_response(http_client.OK) + self.send_header("Content-type", resource.response.CONTENT_TYPE) + self.end_headers() + self.wfile.write(resource.validation.json_dumps().encode()) + return + else: # pylint: disable=useless-else-on-loop + logger.debug("No resources to serve") + logger.debug("%s does not correspond to any resource. ignoring", + self.path) + + @classmethod + def partial_init(cls, simple_http_resources): + """Partially initialize this handler. + + This is useful because `socketserver.BaseServer` takes + uninitialized handler and initializes it with the current + request. + + """ + return functools.partial( + cls, simple_http_resources=simple_http_resources) + + +class ACMERequestHandler(SimpleHTTPRequestHandler): + """ACME request handler.""" + + def handle_one_request(self): + """Handle single request. + + Makes sure that DVSNI probers are ignored. + + """ + try: + return SimpleHTTPRequestHandler.handle_one_request(self) + except OpenSSL.SSL.ZeroReturnError: + logger.debug("Client prematurely closed connection (prober?). " + "Ignoring request.") + + +def simple_server(cli_args, forever=True): + """Run simple standalone client server.""" + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + parser.add_argument( + "-p", "--port", default=0, help="Port to serve at. By default " + "picks random free port.") + args = parser.parse_args(cli_args[1:]) + + certs = {} + resources = {} + + _, hosts, _ = next(os.walk('.')) + for host in hosts: + with open(os.path.join(host, "cert.pem")) as cert_file: + cert_contents = cert_file.read() + with open(os.path.join(host, "key.pem")) as key_file: + key_contents = key_file.read() + certs[host] = ( + OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, key_contents), + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert_contents)) + + handler = ACMERequestHandler.partial_init( + simple_http_resources=resources) + server = ACMETLSServer(('', int(args.port)), handler, certs=certs) + six.print_("Serving at https://localhost:{0}...".format( + server.socket.getsockname()[1])) + if forever: # pragma: no cover + server.serve_forever() + else: + server.handle_request() + + +if __name__ == "__main__": + sys.exit(simple_server(sys.argv)) # pragma: no cover diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py new file mode 100644 index 000000000..794fb1a6e --- /dev/null +++ b/acme/acme/standalone_test.py @@ -0,0 +1,128 @@ +"""Tests for acme.standalone.""" +import os +import threading +import time +import unittest + +from six.moves import http_client # pylint: disable=import-error +from six.moves import socketserver # pylint: disable=import-error + +import requests + +from acme import challenges +from acme import crypto_util +from acme import jose +from acme import test_util + + +class TLSServerTest(unittest.TestCase): + """Tests for acme.standalone.TLSServer.""" + + def test_bind(self): # pylint: disable=no-self-use + from acme.standalone import TLSServer + server = TLSServer( + ('', 0), socketserver.BaseRequestHandler, bind_and_activate=True) + server.server_close() # pylint: disable=no-member + + +class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): + """End-to-end test for ACME TLS server with SimpleHTTP.""" + + def setUp(self): + self.certs = { + b'localhost': (test_util.load_pyopenssl_private_key('rsa512_key.pem'), + # pylint: disable=protected-access + test_util.load_cert('cert.pem')._wrapped), + } + self.account_key = jose.JWK.load( + test_util.load_vector('rsa1024_key.pem')) + + from acme.standalone import ACMETLSServer + from acme.standalone import ACMERequestHandler + self.resources = set() + handler = ACMERequestHandler.partial_init( + simple_http_resources=self.resources) + self.server = ACMETLSServer(('', 0), handler, certs=self.certs) + self.server_thread = threading.Thread( + # pylint: disable=no-member + target=self.server.handle_request) + self.server_thread.start() + + self.port = self.server.socket.getsockname()[1] + + def tearDown(self): + self.server_thread.join() + + def test_index(self): + response = requests.get( + 'https://localhost:{0}'.format(self.port), verify=False) + self.assertEqual(response.text, 'ACME standalone client') + self.assertTrue(response.ok) + + def test_404(self): + response = requests.get( + 'https://localhost:{0}/foo'.format(self.port), verify=False) + self.assertEqual(response.status_code, http_client.NOT_FOUND) + + def test_dvsni(self): + cert = crypto_util.probe_sni( + b'localhost', *self.server.socket.getsockname()) + self.assertEqual(jose.ComparableX509(cert), + jose.ComparableX509(self.certs[b'localhost'][1])) + + def _test_simple_http(self, add): + chall = challenges.SimpleHTTP(token=(b'x' * 16)) + response = challenges.SimpleHTTPResponse(tls=True) + + from acme.standalone import SimpleHTTPRequestHandler + resource = SimpleHTTPRequestHandler.SimpleHTTPResource( + chall=chall, response=response, validation=response.gen_validation( + chall, self.account_key)) + if add: + self.resources.add(resource) + return resource.response.simple_verify( + resource.chall, 'localhost', self.account_key.public_key(), + port=self.port) + + def test_simple_http_found(self): + self.assertTrue(self._test_simple_http(add=True)) + + def test_simple_http_not_found(self): + self.assertFalse(self._test_simple_http(add=False)) + + +class TestSimpleServer(unittest.TestCase): + """Tests for acme.standalone.simple_server.""" + + TEST_CWD = os.path.join(os.path.dirname(__file__), '..', 'examples', 'standalone') + + def setUp(self): + from acme.standalone import simple_server + self.thread = threading.Thread(target=simple_server, kwargs={ + 'cli_args': ('xxx', '--port', '1234'), + 'forever': False, + }) + self.old_cwd = os.getcwd() + os.chdir(self.TEST_CWD) + self.thread.start() + + def tearDown(self): + os.chdir(self.old_cwd) + self.thread.join() + + def test_it(self): + max_attempts = 5 + while max_attempts: + max_attempts -= 1 + try: + response = requests.get('https://localhost:1234', verify=False) + except requests.ConnectionError: + self.assertTrue(max_attempts > 0, "Timeout!") + time.sleep(1) # wait until thread starts + else: + self.assertEqual(response.text, 'ACME standalone client') + break + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/acme/examples/standalone/README b/acme/examples/standalone/README new file mode 100644 index 000000000..89bc5d74e --- /dev/null +++ b/acme/examples/standalone/README @@ -0,0 +1,2 @@ +python -m acme.standalone -p 1234 +curl -k https://localhost:1234 \ No newline at end of file diff --git a/acme/examples/standalone/localhost/cert.pem b/acme/examples/standalone/localhost/cert.pem new file mode 120000 index 000000000..569366af9 --- /dev/null +++ b/acme/examples/standalone/localhost/cert.pem @@ -0,0 +1 @@ +../../../acme/testdata/cert.pem \ No newline at end of file diff --git a/acme/examples/standalone/localhost/key.pem b/acme/examples/standalone/localhost/key.pem new file mode 120000 index 000000000..870f4f876 --- /dev/null +++ b/acme/examples/standalone/localhost/key.pem @@ -0,0 +1 @@ +../../../acme/testdata/rsa512_key.pem \ No newline at end of file diff --git a/docs/pkgs/acme/index.rst b/docs/pkgs/acme/index.rst index 04194c353..23c5a3284 100644 --- a/docs/pkgs/acme/index.rst +++ b/docs/pkgs/acme/index.rst @@ -47,3 +47,10 @@ Errors .. automodule:: acme.errors :members: + + +Standalone +---------- + +.. automodule:: acme.standalone + :members: From a874654e34d18ab57158f117e23693ba8d955d4c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:14:00 +0000 Subject: [PATCH 08/91] NamespaceConfig.simple_http_port. --- letsencrypt/configuration.py | 18 ++++++++++++++++++ letsencrypt/tests/configuration_test.py | 3 ++- letsencrypt/tests/renewer_test.py | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 20774e5cc..5f965cb6d 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -4,6 +4,8 @@ import urlparse import zope.interface +from acme import challenges + from letsencrypt import constants from letsencrypt import interfaces @@ -34,6 +36,13 @@ class NamespaceConfig(object): def __init__(self, namespace): self.namespace = namespace + # XXX: breaks renewer in some bizarre way + #if self.no_simple_http_tls and ( + # self.simple_http_port == self.dvsni_port): + # raise errors.Error( + # "Trying to run SimpleHTTP non-TLS and DVSNI " + # "on the same port ({0})".format(self.dvsni_port)) + def __getattr__(self, name): return getattr(self.namespace, name) @@ -69,6 +78,15 @@ class NamespaceConfig(object): return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + @property + def simple_http_port(self): # pylint: disable=missing-docstring + if self.namespace.simple_http_port is not None: + return self.namespace.simple_http_port + if self.no_simple_http_tls: + return challenges.SimpleHTTPResponse.PORT + else: + return challenges.SimpleHTTPResponse.TLS_PORT + class RenewerConfiguration(object): """Configuration wrapper for renewer.""" diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index 546834c05..91a3dfe37 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -11,7 +11,8 @@ class NamespaceConfigTest(unittest.TestCase): def setUp(self): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', - server='https://acme-server.org:443/new') + server='https://acme-server.org:443/new', + dvsni_port='1234', simple_http_port='4321') from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 293b09537..84d4cf6ec 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -39,7 +39,8 @@ class BaseRenewableCertTest(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.cli_config = configuration.RenewerConfiguration( - namespace=mock.MagicMock(config_dir=self.tempdir)) + namespace=mock.MagicMock( + config_dir=self.tempdir, no_simple_http_tls=False)) # TODO: maybe provide RenewerConfiguration.make_dirs? os.makedirs(os.path.join(self.tempdir, "live", "example.org")) os.makedirs(os.path.join(self.tempdir, "archive", "example.org")) From ef3605730c349073f1f461bf59cbd8d448f7487d Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:26:36 +0000 Subject: [PATCH 09/91] Move already_listening to plugins.util --- docs/api/plugins/util.rst | 5 + .../plugins/standalone/authenticator.py | 44 +------- .../standalone/tests/authenticator_test.py | 103 ------------------ letsencrypt/plugins/util.py | 49 +++++++++ letsencrypt/plugins/util_test.py | 103 ++++++++++++++++++ 5 files changed, 158 insertions(+), 146 deletions(-) create mode 100644 docs/api/plugins/util.rst create mode 100644 letsencrypt/plugins/util.py create mode 100644 letsencrypt/plugins/util_test.py diff --git a/docs/api/plugins/util.rst b/docs/api/plugins/util.rst new file mode 100644 index 000000000..6bc8995db --- /dev/null +++ b/docs/api/plugins/util.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.util` +------------------------------- + +.. automodule:: letsencrypt.plugins.util + :members: diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index 968063781..996f41cdc 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -1,6 +1,5 @@ """Standalone authenticator.""" import os -import psutil import signal import socket import sys @@ -289,47 +288,6 @@ class StandaloneAuthenticator(common.Plugin): # should terminate via sys.exit(). return self.do_child_process(port) - def already_listening(self, port): # pylint: disable=no-self-use - """Check if a process is already listening on the port. - - If so, also tell the user via a display notification. - - .. warning:: - On some operating systems, this function can only usefully be - run as root. - - :param int port: The TCP port in question. - :returns: True or False.""" - - listeners = [conn.pid for conn in psutil.net_connections() - if conn.status == 'LISTEN' and - conn.type == socket.SOCK_STREAM and - conn.laddr[1] == port] - try: - if listeners and listeners[0] is not None: - # conn.pid may be None if the current process doesn't have - # permission to identify the listening process! Additionally, - # listeners may have more than one element if separate - # sockets have bound the same port on separate interfaces. - # We currently only have UI to notify the user about one - # of them at a time. - pid = listeners[0] - name = psutil.Process(pid).name() - display = zope.component.getUtility(interfaces.IDisplay) - display.notification( - "The program {0} (process ID {1}) is already listening " - "on TCP port {2}. This will prevent us from binding to " - "that port. Please stop the {0} program temporarily " - "and then try again.".format(name, pid, port)) - return True - except (psutil.NoSuchProcess, psutil.AccessDenied): - # Perhaps the result of a race where the process could have - # exited or relinquished the port (NoSuchProcess), or the result - # of an OS policy where we're not allowed to look up the process - # name (AccessDenied). - pass - return False - # IAuthenticator method implementations follow def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use @@ -383,7 +341,7 @@ class StandaloneAuthenticator(common.Plugin): if not self.tasks: raise ValueError("nothing for .perform() to do") - if self.already_listening(self.config.dvsni_port): + if util.already_listening(self.config.dvsni_port): # If we know a process is already listening on this port, # tell the user, and don't even attempt to bind it. (This # test is Linux-specific and won't indicate that the port diff --git a/letsencrypt/plugins/standalone/tests/authenticator_test.py b/letsencrypt/plugins/standalone/tests/authenticator_test.py index 7ff2c03e1..955426533 100644 --- a/letsencrypt/plugins/standalone/tests/authenticator_test.py +++ b/letsencrypt/plugins/standalone/tests/authenticator_test.py @@ -187,109 +187,6 @@ class SubprocSignalHandlerTest(unittest.TestCase): mock_exit.assert_called_once_with(0) -class AlreadyListeningTest(unittest.TestCase): - """Tests for already_listening() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_race_condition(self, mock_get_utility, mock_process, mock_net): - # This tests a race condition, or permission problem, or OS - # incompatibility in which, for some reason, no process name can be - # found to match the identified listening PID. - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.side_effect = psutil.NoSuchProcess("No such PID") - # We simulate being unable to find the process name of PID 4416, - # which results in returning False. - self.assertFalse(self.authenticator.already_listening(17)) - self.assertEqual(mock_get_utility.generic_notification.call_count, 0) - mock_process.assert_called_once_with(4416) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_not_listening(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - self.assertFalse(self.authenticator.already_listening(17)) - self.assertEqual(mock_get_utility.generic_notification.call_count, 0) - self.assertEqual(mock_process.call_count, 0) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - result = self.authenticator.already_listening(17) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - mock_process.assert_called_once_with(4416) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil." - "net_connections") - @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil.Process") - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net): - from psutil._common import sconn - conns = [ - sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), - raddr=(), status="LISTEN", pid=None), - sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), - raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), - sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), - raddr=("::1", 111), status="CLOSE_WAIT", pid=None), - sconn(fd=3, family=10, type=1, laddr=("::", 12345), raddr=(), - status="LISTEN", pid=4420), - sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), - raddr=(), status="LISTEN", pid=4416)] - mock_net.return_value = conns - mock_process.name.return_value = "inetd" - result = self.authenticator.already_listening(12345) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - mock_process.assert_called_once_with(4420) - - class PerformTest(unittest.TestCase): """Tests for perform() method.""" def setUp(self): diff --git a/letsencrypt/plugins/util.py b/letsencrypt/plugins/util.py new file mode 100644 index 000000000..42c64b052 --- /dev/null +++ b/letsencrypt/plugins/util.py @@ -0,0 +1,49 @@ +"""Plugin utilities.""" +import socket + +import psutil +import zope.component + +from letsencrypt import interfaces + + +def already_listening(port): + """Check if a process is already listening on the port. + + If so, also tell the user via a display notification. + + .. warning:: + On some operating systems, this function can only usefully be + run as root. + + :param int port: The TCP port in question. + :returns: True or False.""" + + listeners = [conn.pid for conn in psutil.net_connections() + if conn.status == 'LISTEN' and + conn.type == socket.SOCK_STREAM and + conn.laddr[1] == port] + try: + if listeners and listeners[0] is not None: + # conn.pid may be None if the current process doesn't have + # permission to identify the listening process! Additionally, + # listeners may have more than one element if separate + # sockets have bound the same port on separate interfaces. + # We currently only have UI to notify the user about one + # of them at a time. + pid = listeners[0] + name = psutil.Process(pid).name() + display = zope.component.getUtility(interfaces.IDisplay) + display.notification( + "The program {0} (process ID {1}) is already listening " + "on TCP port {2}. This will prevent us from binding to " + "that port. Please stop the {0} program temporarily " + "and then try again.".format(name, pid, port)) + return True + except (psutil.NoSuchProcess, psutil.AccessDenied): + # Perhaps the result of a race where the process could have + # exited or relinquished the port (NoSuchProcess), or the result + # of an OS policy where we're not allowed to look up the process + # name (AccessDenied). + pass + return False diff --git a/letsencrypt/plugins/util_test.py b/letsencrypt/plugins/util_test.py new file mode 100644 index 000000000..14cbcf7a8 --- /dev/null +++ b/letsencrypt/plugins/util_test.py @@ -0,0 +1,103 @@ +"""Tests for letsencrypt.plugins.util.""" +import unittest + +import mock +import psutil + + +class AlreadyListeningTest(unittest.TestCase): + """Tests for letsencrypt.plugins.already_listening.""" + def _call(self, *args, **kwargs): + from letsencrypt.plugins.util import already_listening + return already_listening(*args, **kwargs) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_race_condition(self, mock_get_utility, mock_process, mock_net): + # This tests a race condition, or permission problem, or OS + # incompatibility in which, for some reason, no process name can be + # found to match the identified listening PID. + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.side_effect = psutil.NoSuchProcess("No such PID") + # We simulate being unable to find the process name of PID 4416, + # which results in returning False. + self.assertFalse(self._call(17)) + self.assertEqual(mock_get_utility.generic_notification.call_count, 0) + mock_process.assert_called_once_with(4416) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_not_listening(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + self.assertFalse(self._call(17)) + self.assertEqual(mock_get_utility.generic_notification.call_count, 0) + self.assertEqual(mock_process.call_count, 0) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_listening_ipv4(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + result = self._call(17) + self.assertTrue(result) + self.assertEqual(mock_get_utility.call_count, 1) + mock_process.assert_called_once_with(4416) + + @mock.patch("letsencrypt.plugins.util.psutil.net_connections") + @mock.patch("letsencrypt.plugins.util.psutil.Process") + @mock.patch("letsencrypt.plugins.util.zope.component.getUtility") + def test_listening_ipv6(self, mock_get_utility, mock_process, mock_net): + from psutil._common import sconn + conns = [ + sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30), + raddr=(), status="LISTEN", pid=None), + sconn(fd=3, family=2, type=1, laddr=("192.168.5.10", 32783), + raddr=("20.40.60.80", 22), status="ESTABLISHED", pid=1234), + sconn(fd=-1, family=10, type=1, laddr=("::1", 54321), + raddr=("::1", 111), status="CLOSE_WAIT", pid=None), + sconn(fd=3, family=10, type=1, laddr=("::", 12345), raddr=(), + status="LISTEN", pid=4420), + sconn(fd=3, family=2, type=1, laddr=("0.0.0.0", 17), + raddr=(), status="LISTEN", pid=4416)] + mock_net.return_value = conns + mock_process.name.return_value = "inetd" + result = self._call(12345) + self.assertTrue(result) + self.assertEqual(mock_get_utility.call_count, 1) + mock_process.assert_called_once_with(4420) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From faa6cbdd71ec72616b44a6c2abc8dd9cdff2172e Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 26 Sep 2015 16:47:29 +0000 Subject: [PATCH 10/91] Standalone 2.0 --- letsencrypt/achallenges.py | 22 +- letsencrypt/errors.py | 10 + letsencrypt/plugins/disco_test.py | 12 +- letsencrypt/plugins/standalone.py | 200 ++++++++ letsencrypt/plugins/standalone/__init__.py | 1 - .../plugins/standalone/authenticator.py | 394 -------------- .../plugins/standalone/tests/__init__.py | 1 - .../standalone/tests/authenticator_test.py | 483 ------------------ letsencrypt/tests/achallenges_test.py | 8 +- setup.py | 4 +- 10 files changed, 228 insertions(+), 907 deletions(-) create mode 100644 letsencrypt/plugins/standalone.py delete mode 100644 letsencrypt/plugins/standalone/__init__.py delete mode 100644 letsencrypt/plugins/standalone/authenticator.py delete mode 100644 letsencrypt/plugins/standalone/tests/__init__.py delete mode 100644 letsencrypt/plugins/standalone/tests/authenticator_test.py diff --git a/letsencrypt/achallenges.py b/letsencrypt/achallenges.py index 958a29733..e86f51a3f 100644 --- a/letsencrypt/achallenges.py +++ b/letsencrypt/achallenges.py @@ -19,8 +19,6 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging -import OpenSSL - from acme import challenges from acme import jose @@ -56,10 +54,10 @@ class DVSNI(AnnotatedChallenge): __slots__ = ('challb', 'domain', 'account_key') acme_type = challenges.DVSNI - def gen_cert_and_response(self, key_pem=None, bits=2048, alg=jose.RS256): + def gen_cert_and_response(self, key=None, bits=2048, alg=jose.RS256): """Generate a DVSNI cert and response. - :param bytes key_pem: Private PEM-encoded key used for + :param OpenSSL.crypto.PKey key: Private key used for certificate generation. If none provided, a fresh key will be generated. :param int bits: Number of bits for fresh key generation. @@ -67,23 +65,15 @@ class DVSNI(AnnotatedChallenge): :returns: ``(response, cert_pem, key_pem)`` tuple, where ``response`` is an instance of - `acme.challenges.DVSNIResponse`, ``cert_pem`` is the - PEM-encoded certificate and ``key_pem`` is PEM-encoded - private key. + `acme.challenges.DVSNIResponse`, ``cert`` is a certificate + (`OpenSSL.crypto.X509`) and ``key`` is a private key + (`OpenSSL.crypto.PKey`). :rtype: tuple """ - key = None if key_pem is None else OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, key_pem) response = self.challb.chall.gen_response(self.account_key, alg=alg) cert, key = response.gen_cert(key=key, bits=bits) - - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, cert) - key_pem = OpenSSL.crypto.dump_privatekey( - OpenSSL.crypto.FILETYPE_PEM, key) - - return response, cert_pem, key_pem + return response, cert, key class SimpleHTTP(AnnotatedChallenge): diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index ba0601d29..98c24bf50 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -80,3 +80,13 @@ class NotSupportedError(PluginError): class RevokerError(Error): """Let's Encrypt Revoker error.""" + + +class StandaloneBindError(Error): + """Standalone plugin bind error.""" + + def __init__(self, socket_error, port): + super(StandaloneBindError, self).__init__( + "Problem binding to port {0}: {1}".format(port, socket_error)) + self.socket_error = socket_error + self.port = port diff --git a/letsencrypt/plugins/disco_test.py b/letsencrypt/plugins/disco_test.py index 41699d1ef..8660d94a1 100644 --- a/letsencrypt/plugins/disco_test.py +++ b/letsencrypt/plugins/disco_test.py @@ -8,11 +8,11 @@ import zope.interface from letsencrypt import errors from letsencrypt import interfaces -from letsencrypt.plugins.standalone import authenticator +from letsencrypt.plugins import standalone EP_SA = pkg_resources.EntryPoint( - "sa", "letsencrypt.plugins.standalone.authenticator", - attrs=("StandaloneAuthenticator",), + "sa", "letsencrypt.plugins.standalone", + attrs=("Authenticator",), dist=mock.MagicMock(key="letsencrypt")) @@ -71,8 +71,7 @@ class PluginEntryPointTest(unittest.TestCase): self.assertTrue(self.plugin_ep.entry_point is EP_SA) self.assertEqual("sa", self.plugin_ep.name) - self.assertTrue( - self.plugin_ep.plugin_cls is authenticator.StandaloneAuthenticator) + self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator) def test_init(self): config = mock.MagicMock() @@ -174,8 +173,7 @@ class PluginsRegistryTest(unittest.TestCase): with mock.patch("letsencrypt.plugins.disco.pkg_resources") as mock_pkg: mock_pkg.iter_entry_points.return_value = iter([EP_SA]) plugins = PluginsRegistry.find_all() - self.assertTrue(plugins["sa"].plugin_cls - is authenticator.StandaloneAuthenticator) + self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator) self.assertTrue(plugins["sa"].entry_point is EP_SA) def test_getitem(self): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py new file mode 100644 index 000000000..8e5f1e77d --- /dev/null +++ b/letsencrypt/plugins/standalone.py @@ -0,0 +1,200 @@ +"""Standalone Authenticator.""" +import collections +import functools +import logging +import random +import socket +import threading + +from six.moves import BaseHTTPServer # pylint: disable=import-error + +import OpenSSL +import zope.interface + +from acme import challenges +from acme import crypto_util as acme_crypto_util +from acme import standalone as acme_standalone + +from letsencrypt import achallenges +from letsencrypt import errors +from letsencrypt import interfaces + +from letsencrypt.plugins import common +from letsencrypt.plugins import util + +logger = logging.getLogger(__name__) + + +class ServerManager(object): + """Standalone servers manager.""" + + def __init__(self, certs, simple_http_resources): + self.servers = {} + self.certs = certs + self.simple_http_resources = simple_http_resources + + def run(self, port, tls): + """Run ACME server on specified ``port``.""" + if port in self.servers: + return self.servers[port] + + logger.debug("Starting new server at %s (tls=%s)", port, tls) + handler = acme_standalone.ACMERequestHandler.partial_init( + self.simple_http_resources) + + if tls: + cls = functools.partial( + acme_standalone.HTTPSServer, certs=self.certs) + else: + cls = BaseHTTPServer.HTTPServer + + try: + server = cls(('', port), handler) + except socket.error as error: + errors.StandaloneBindError(error, port) + + stop = threading.Event() + thread = threading.Thread( + target=self._serve, + args=(server, stop), + ) + thread.start() + self.servers[port] = (server, thread, stop) + return self.servers[port] + + def _serve(self, server, stop): + while not stop.is_set(): + server.handle_request() + + def stop(self, port): + """Stop ACME server running on the specified ``port``.""" + server, thread, stop = self.servers[port] + stop.set() + + # dummy request to terminate last handle_request() + sock = socket.socket() + try: + sock.connect(server.socket.getsockname()) + except socket.error: + pass # thread is probably already finished + finally: + sock.close() + + thread.join() + del self.servers[port] + + def items(self): + """Return a list of all port, server tuples.""" + return self.servers.items() + + +class Authenticator(common.Plugin): + """Standalone Authenticator. + + This authenticator creates its own ephemeral TCP listener on the + necessary port in order to respond to incoming DVSNI and SimpleHTTP + challenges from the certificate authority. Therefore, it does not + rely on any existing server program. + + """ + zope.interface.implements(interfaces.IAuthenticator) + zope.interface.classProvides(interfaces.IPluginFactory) + + description = "Standalone Authenticator" + supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + + # one self-signed key for all DVSNI and SimpleHTTP certificates + self.key = OpenSSL.crypto.PKey() + self.key.generate_key(OpenSSL.crypto.TYPE_RSA, bits=2048) + # TODO: generate only when the first SimpleHTTP challenge is solved + self.simple_http_cert = acme_crypto_util.gen_ss_cert( + self.key, domains=["temp server"]) + + self.responses = {} + self.servers = {} + self.served = collections.defaultdict(set) + + # Stuff below is shared across threads (i.e. servers read + # values, main thread writes). Due to the nature of Cython's + # GIL, the operations are safe, c.f. + # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe + self.certs = {} + self.simple_http_resources = set() + + self.servers = ServerManager(self.certs, self.simple_http_resources) + + def more_info(self): # pylint: disable=missing-docstring + return self.__doc__ + + def prepare(self): # pylint: disable=missing-docstring + if any(util.already_listening(port) for port in + (self.config.dvsni_port, self.config.simple_http_port)): + raise errors.MisconfigurationError( + "One of the (possibly) required ports is already taken taken.") + + # TODO: add --chall-pref flag + def get_chall_pref(self, domain): + # pylint: disable=unused-argument,missing-docstring + chall_pref = list(self.supported_challenges) + random.shuffle(chall_pref) # 50% for each challenge + return chall_pref + + def perform(self, achalls): # pylint: disable=missing-docstring + try: + return self.perform2(achalls) + except errors.StandaloneBindError as error: + display = zope.component.getUtility(interfaces.IDisplay) + + if error.socket_error.errno == socket.errno.EACCES: + display.notification( + "Could not bind TCP port {0} because you don't have " + "the appropriate permissions (for example, you " + "aren't running this program as " + "root).".format(error.port)) + elif error.socket_error.errno == socket.errno.EADDRINUSE: + display.notification( + "Could not bind TCP port {0} because it is already in " + "use by another process on this system (such as a web " + "server). Please stop the program in question and then " + "try again.".format(error.port)) + else: + raise # XXX: How to handle unknown errors in binding? + + def perform2(self, achalls): + """Perform achallenges without IDisplay interaction.""" + responses = [] + tls = not self.config.no_simple_http_tls + + for achall in achalls: + if isinstance(achall, achallenges.SimpleHTTP): + server, _, _ = self.servers.run(self.config.simple_http_port, tls=tls) + response, validation = achall.gen_response_and_validation(tls=tls) + self.simple_http_resources.add( + acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( + chall=achall.chall, response=response, + validation=validation)) + cert = self.simple_http_cert + domain = achall.domain + else: # DVSNI + server, _, _ = self.servers.run(self.config.dvsni_port, tls=True) + response, cert, _ = achall.gen_cert_and_response(self.key) + domain = response.z_domain + self.certs[domain] = (self.key, cert) + self.responses[achall] = response + self.served[server].add(achall) + responses.append(response) + + return responses + + def cleanup(self, achalls): # pylint: disable=missing-docstring + # reduce self.served and close servers if none challenges are served + for server, server_achalls in self.served.items(): + for achall in achalls: + if achall in server_achalls: + server_achalls.remove(achall) + for port, (server, _, _) in self.servers.items(): + if not self.served[server]: + self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone/__init__.py b/letsencrypt/plugins/standalone/__init__.py deleted file mode 100644 index 972c484ed..000000000 --- a/letsencrypt/plugins/standalone/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Let's Encrypt Standalone Authenticator plugin.""" diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py deleted file mode 100644 index 996f41cdc..000000000 --- a/letsencrypt/plugins/standalone/authenticator.py +++ /dev/null @@ -1,394 +0,0 @@ -"""Standalone authenticator.""" -import os -import signal -import socket -import sys -import time - -import OpenSSL -import zope.component -import zope.interface - -from acme import challenges - -from letsencrypt import achallenges -from letsencrypt import crypto_util -from letsencrypt import interfaces - -from letsencrypt.plugins import common - - -class StandaloneAuthenticator(common.Plugin): - # pylint: disable=too-many-instance-attributes - """Standalone authenticator. - - This authenticator creates its own ephemeral TCP listener on the - specified port in order to respond to incoming DVSNI challenges from - the certificate authority. Therefore, it does not rely on any - existing server program. - - :param OpenSSL.crypto.PKey private_key: DVSNI challenge certificate - key. - :param sni_names: Mapping from z_domain (`bytes`) to PEM-encoded - certificate (`bytes`). - - """ - zope.interface.implements(interfaces.IAuthenticator) - zope.interface.classProvides(interfaces.IPluginFactory) - - description = "Standalone Authenticator" - - def __init__(self, *args, **kwargs): - super(StandaloneAuthenticator, self).__init__(*args, **kwargs) - self.child_pid = None - self.parent_pid = os.getpid() - self.subproc_state = None - self.tasks = {} - self.sni_names = {} - self.sock = None - self.connection = None - self.key_pem = crypto_util.make_key(bits=2048) - self.private_key = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, self.key_pem) - self.ssl_conn = None - - def prepare(self): - """There is nothing left to setup. - - .. todo:: This should probably do the port check - - """ - - def client_signal_handler(self, sig, unused_frame): - """Signal handler for the parent process. - - This handler receives inter-process communication from the - child process in the form of Unix signals. - - :param int sig: Which signal the process received. - - """ - # subprocess to client READY: SIGIO - # subprocess to client INUSE: SIGUSR1 - # subprocess to client CANTBIND: SIGUSR2 - if sig == signal.SIGIO: - self.subproc_state = "ready" - elif sig == signal.SIGUSR1: - self.subproc_state = "inuse" - elif sig == signal.SIGUSR2: - self.subproc_state = "cantbind" - else: - # NOTREACHED - raise ValueError("Unexpected signal in signal handler") - - def subproc_signal_handler(self, sig, unused_frame): - """Signal handler for the child process. - - This handler receives inter-process communication from the parent - process in the form of Unix signals. - - :param int sig: Which signal the process received. - - """ - # client to subprocess CLEANUP : SIGINT - if sig == signal.SIGINT: - try: - self.ssl_conn.shutdown() - self.ssl_conn.close() - except BaseException: - # There might not even be any currently active SSL connection. - pass - try: - self.connection.close() - except BaseException: - # There might not even be any currently active connection. - pass - try: - self.sock.close() - except BaseException: - # Various things can go wrong in the course of closing these - # connections, but none of them can clearly be usefully - # reported here and none of them should impede us from - # exiting as gracefully as possible. - pass - - os.kill(self.parent_pid, signal.SIGUSR1) - sys.exit(0) - - def sni_callback(self, connection): - """Used internally to respond to incoming SNI names. - - This method will set a new OpenSSL context object for this - connection when an incoming connection provides an SNI name - (in order to serve the appropriate certificate, if any). - - :param connection: The TLS connection object on which the SNI - extension was received. - :type connection: :class:`OpenSSL.Connection` - - """ - sni_name = connection.get_servername() - if sni_name in self.sni_names: - pem_cert = self.sni_names[sni_name] - else: - # TODO: Should we really present a certificate if we get an - # unexpected SNI name? Or should we just disconnect? - pem_cert = next(self.sni_names.itervalues()) - cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, pem_cert) - new_ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) - new_ctx.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda: False) - new_ctx.use_certificate(cert) - new_ctx.use_privatekey(self.private_key) - connection.set_context(new_ctx) - - def do_parent_process(self, port, delay_amount=5): - """Perform the parent process side of the TCP listener task. - - This should only be called by :meth:`start_listener`. We will - wait up to delay_amount seconds to hear from the child process - via a signal. - - :param int port: Which TCP port to bind. - :param float delay_amount: How long in seconds to wait for the - subprocess to notify us whether it succeeded. - - :returns: ``True`` or ``False`` according to whether we were notified - that the child process succeeded or failed in binding the port. - :rtype: bool - - """ - display = zope.component.getUtility(interfaces.IDisplay) - - start_time = time.time() - while time.time() < start_time + delay_amount: - if self.subproc_state == "ready": - return True - elif self.subproc_state == "inuse": - display.notification( - "Could not bind TCP port {0} because it is already in " - "use by another process on this system (such as a web " - "server). Please stop the program in question and then " - "try again.".format(port)) - return False - elif self.subproc_state == "cantbind": - display.notification( - "Could not bind TCP port {0} because you don't have " - "the appropriate permissions (for example, you " - "aren't running this program as " - "root).".format(port)) - return False - time.sleep(0.1) - - display.notification( - "Subprocess unexpectedly timed out while trying to bind TCP " - "port {0}.".format(port)) - - return False - - def do_child_process(self, port): - """Perform the child process side of the TCP listener task. - - This should only be called by :meth:`start_listener`. - - Normally does not return; instead, the child process exits from - within this function or from within the child process signal - handler. - - :param int port: Which TCP port to bind. - - """ - signal.signal(signal.SIGINT, self.subproc_signal_handler) - self.sock = socket.socket() - # SO_REUSEADDR flag tells the kernel to reuse a local socket - # in TIME_WAIT state, without waiting for its natural timeout - # to expire. - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - self.sock.bind(("0.0.0.0", port)) - except socket.error, error: - if error.errno == socket.errno.EACCES: - # Signal permissions denied to bind TCP port - os.kill(self.parent_pid, signal.SIGUSR2) - elif error.errno == socket.errno.EADDRINUSE: - # Signal TCP port is already in use - os.kill(self.parent_pid, signal.SIGUSR1) - else: - # XXX: How to handle unknown errors in binding? - raise error - sys.exit(1) - # XXX: We could use poll mechanism to handle simultaneous - # XXX: rather than sequential inbound TCP connections here - self.sock.listen(1) - # Signal that we've successfully bound TCP port - os.kill(self.parent_pid, signal.SIGIO) - - while True: - self.connection, _ = self.sock.accept() - - # The code below uses the PyOpenSSL bindings to respond to - # the client. This may expose us to bugs and vulnerabilities - # in OpenSSL (and creates additional dependencies). - ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) - ctx.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda: False) - pem_cert = self.tasks.values()[0] - first_cert = OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, pem_cert) - ctx.use_certificate(first_cert) - ctx.use_privatekey(self.private_key) - ctx.set_cipher_list("HIGH") - ctx.set_tlsext_servername_callback(self.sni_callback) - self.ssl_conn = OpenSSL.SSL.Connection(ctx, self.connection) - self.ssl_conn.set_accept_state() - self.ssl_conn.do_handshake() - self.ssl_conn.shutdown() - self.ssl_conn.close() - - def start_listener(self, port): - """Start listener. - - Create a child process which will start a TCP listener on the - specified port to perform the specified DVSNI challenges. - - :param int port: The TCP port to bind. - - :returns: ``True`` or ``False`` to indicate success or failure creating - the subprocess. - :rtype: bool - - """ - # In order to avoid a race condition, we set the signal handler - # that will be needed by the parent process now, and undo this - # action if we turn out to be the child process. (This needs - # to happen before the fork because the child will send one of - # these signals to the parent almost immediately after the - # fork, and the parent must already be ready to receive it.) - signal.signal(signal.SIGIO, self.client_signal_handler) - signal.signal(signal.SIGUSR1, self.client_signal_handler) - signal.signal(signal.SIGUSR2, self.client_signal_handler) - - sys.stdout.flush() - fork_result = os.fork() - if fork_result: - # PARENT process (still the Let's Encrypt client process) - self.child_pid = fork_result - # do_parent_process() can return True or False to indicate - # reported success or failure creating the listener. - return self.do_parent_process(port) - else: - # CHILD process (the TCP listener subprocess) - # Undo the parent's signal handler settings, which aren't - # applicable to us. - signal.signal(signal.SIGIO, signal.SIG_DFL) - signal.signal(signal.SIGUSR1, signal.SIG_DFL) - signal.signal(signal.SIGUSR2, signal.SIG_DFL) - - self.child_pid = os.getpid() - # do_child_process() is normally not expected to return but - # should terminate via sys.exit(). - return self.do_child_process(port) - - # IAuthenticator method implementations follow - - def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use - """Get challenge preferences. - - IAuthenticator interface method get_chall_pref. - Return a list of challenge types that this authenticator - can perform for this domain. In the case of the - StandaloneAuthenticator, the only challenge type that can ever - be performed is dvsni. - - :returns: A list containing only 'dvsni'. - - """ - return [challenges.DVSNI] - - def perform(self, achalls): - """Perform the challenge. - - .. warning:: - For the StandaloneAuthenticator, because there is no convenient - way to add additional requests, this should only be invoked - once; subsequent invocations are an error. To perform - validations for multiple independent sets of domains, a separate - StandaloneAuthenticator should be instantiated. - - """ - if self.child_pid or self.tasks: - # We should not be willing to continue with perform - # if there were existing pending challenges. - raise ValueError(".perform() was called with pending tasks!") - results_if_success = [] - results_if_failure = [] - if not achalls or not isinstance(achalls, list): - raise ValueError(".perform() was called without challenge list") - # TODO: "bits" should be user-configurable - for achall in achalls: - if isinstance(achall, achallenges.DVSNI): - # We will attempt to do it - response, cert_pem, _ = achall.gen_cert_and_response( - key_pem=self.key_pem) - self.sni_names[response.z_domain] = cert_pem - self.tasks[achall.token] = cert_pem - results_if_success.append(response) - results_if_failure.append(None) - else: - # We will not attempt to do this challenge because it - # is not a type we can handle - results_if_success.append(False) - results_if_failure.append(False) - if not self.tasks: - raise ValueError("nothing for .perform() to do") - - if util.already_listening(self.config.dvsni_port): - # If we know a process is already listening on this port, - # tell the user, and don't even attempt to bind it. (This - # test is Linux-specific and won't indicate that the port - # is bound if invoked on a different operating system.) - return results_if_failure - # Try to do the authentication; note that this creates - # the listener subprocess via os.fork() - if self.start_listener(self.config.dvsni_port): - return results_if_success - else: - # TODO: This should probably raise a DVAuthError exception - # rather than returning a list of None objects. - return results_if_failure - - def cleanup(self, achalls): - """Clean up. - - If some challenges are removed from the list, the authenticator - socket will still respond to those challenges. Once all - challenges have been removed from the list, the listener is - deactivated and stops listening. - - """ - # Remove this from pending tasks list - for achall in achalls: - assert isinstance(achall, achallenges.DVSNI) - if achall.token in self.tasks: - del self.tasks[achall.token] - else: - # Could not find the challenge to remove! - raise ValueError("could not find the challenge to remove") - if self.child_pid and not self.tasks: - # There are no remaining challenges, so - # try to shutdown self.child_pid cleanly. - # TODO: ignore any signals from child during this process - os.kill(self.child_pid, signal.SIGINT) - time.sleep(1) - # TODO: restore original signal handlers in parent process - # by resetting their actions to SIG_DFL - # print "TCP listener subprocess has been told to shut down" - - def more_info(self): # pylint: disable=no-self-use - """Human-readable string that describes the Authenticator.""" - return ("The Standalone Authenticator uses PyOpenSSL to listen " - "on port {port} and perform DVSNI challenges. Once a " - "certificate is attained, it will be saved in the " - "(TODO) current working directory.{linesep}{linesep}" - "TCP port {port} must be available in order to use the " - "Standalone Authenticator.".format( - linesep=os.linesep, port=self.config.dvsni_port)) diff --git a/letsencrypt/plugins/standalone/tests/__init__.py b/letsencrypt/plugins/standalone/tests/__init__.py deleted file mode 100644 index 059cd2780..000000000 --- a/letsencrypt/plugins/standalone/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Let's Encrypt Standalone Tests""" diff --git a/letsencrypt/plugins/standalone/tests/authenticator_test.py b/letsencrypt/plugins/standalone/tests/authenticator_test.py deleted file mode 100644 index 955426533..000000000 --- a/letsencrypt/plugins/standalone/tests/authenticator_test.py +++ /dev/null @@ -1,483 +0,0 @@ -"""Tests for letsencrypt.plugins.standalone.authenticator.""" -import os -import psutil -import signal -import socket -import unittest - -import mock -import OpenSSL - -from acme import challenges -from acme import jose - -from letsencrypt import achallenges - -from letsencrypt.tests import acme_util -from letsencrypt.tests import test_util - - -ACCOUNT_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -CHALL_KEY_PEM = test_util.load_vector("rsa512_key_2.pem") -CHALL_KEY = OpenSSL.crypto.load_privatekey( - OpenSSL.crypto.FILETYPE_PEM, CHALL_KEY_PEM) -CONFIG = mock.Mock(dvsni_port=5001) - - -# Classes based on to allow interrupting infinite loop under test -# after one iteration, based on. -# http://igorsobreira.com/2013/03/17/testing-infinite-loops.html - -class _SocketAcceptOnlyNTimes(object): - # pylint: disable=too-few-public-methods - """ - Callable that will raise `CallableExhausted` - exception after `limit` calls, modified to also return - a tuple simulating the return values of a socket.accept() - call - """ - def __init__(self, limit): - self.limit = limit - self.calls = 0 - - def __call__(self): - self.calls += 1 - if self.calls > self.limit: - raise CallableExhausted - # Modified here for a single use as socket.accept() - return (mock.MagicMock(), "ignored") - - -class CallableExhausted(Exception): - # pylint: disable=too-few-public-methods - """Exception raised when a method is called more than the - specified number of times.""" - - -class ChallPrefTest(unittest.TestCase): - """Tests for chall_pref() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_chall_pref(self): - self.assertEqual(self.authenticator.get_chall_pref("example.com"), - [challenges.DVSNI]) - - -class SNICallbackTest(unittest.TestCase): - """Tests for sni_callback() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.cert = achallenges.DVSNI( - challb=acme_util.DVSNI_P, - domain="example.com", - account_key=ACCOUNT_KEY - ).gen_cert_and_response(key_pem=CHALL_KEY_PEM)[1] - self.authenticator.private_key = CHALL_KEY - self.authenticator.sni_names = {"abcdef.acme.invalid": self.cert} - self.authenticator.child_pid = 12345 - - def test_real_servername(self): - connection = mock.MagicMock() - connection.get_servername.return_value = "abcdef.acme.invalid" - self.authenticator.sni_callback(connection) - self.assertEqual(connection.set_context.call_count, 1) - called_ctx = connection.set_context.call_args[0][0] - self.assertTrue(isinstance(called_ctx, OpenSSL.SSL.Context)) - - def test_fake_servername(self): - """Test behavior of SNI callback when an unexpected name is received. - - (Currently the expected behavior in this case is to return the - "first" certificate with which the listener was configured, - although they are stored in an unordered data structure so - this might not be the one that was first in the challenge list - passed to the perform method. In the future, this might result - in dropping the connection instead.)""" - connection = mock.MagicMock() - connection.get_servername.return_value = "example.com" - self.authenticator.sni_callback(connection) - self.assertEqual(connection.set_context.call_count, 1) - called_ctx = connection.set_context.call_args[0][0] - self.assertTrue(isinstance(called_ctx, OpenSSL.SSL.Context)) - - -class ClientSignalHandlerTest(unittest.TestCase): - """Tests for client_signal_handler() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.authenticator.tasks = {"footoken.acme.invalid": "stuff"} - self.authenticator.child_pid = 12345 - - def test_client_signal_handler(self): - self.assertTrue(self.authenticator.subproc_state is None) - self.authenticator.client_signal_handler(signal.SIGIO, None) - self.assertEqual(self.authenticator.subproc_state, "ready") - - self.authenticator.client_signal_handler(signal.SIGUSR1, None) - self.assertEqual(self.authenticator.subproc_state, "inuse") - - self.authenticator.client_signal_handler(signal.SIGUSR2, None) - self.assertEqual(self.authenticator.subproc_state, "cantbind") - - # Testing the unreached path for a signal other than these - # specified (which can't occur in normal use because this - # function is only set as a signal handler for the above three - # signals). - self.assertRaises( - ValueError, self.authenticator.client_signal_handler, - signal.SIGPIPE, None) - - -class SubprocSignalHandlerTest(unittest.TestCase): - """Tests for subproc_signal_handler() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.authenticator.tasks = {"footoken.acme.invalid": "stuff"} - self.authenticator.child_pid = 12345 - self.authenticator.parent_pid = 23456 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_subproc_signal_handler(self, mock_exit, mock_kill): - self.authenticator.ssl_conn = mock.MagicMock() - self.authenticator.connection = mock.MagicMock() - self.authenticator.sock = mock.MagicMock() - self.authenticator.subproc_signal_handler(signal.SIGINT, None) - self.assertEquals(self.authenticator.ssl_conn.shutdown.call_count, 1) - self.assertEquals(self.authenticator.ssl_conn.close.call_count, 1) - self.assertEquals(self.authenticator.connection.close.call_count, 1) - self.assertEquals(self.authenticator.sock.close.call_count, 1) - mock_kill.assert_called_once_with( - self.authenticator.parent_pid, signal.SIGUSR1) - mock_exit.assert_called_once_with(0) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_subproc_signal_handler_trouble(self, mock_exit, mock_kill): - """Test attempting to shut down a non-existent connection. - - (This could occur because none was established or active at the - time the signal handler tried to perform the cleanup).""" - self.authenticator.ssl_conn = mock.MagicMock() - self.authenticator.connection = mock.MagicMock() - self.authenticator.sock = mock.MagicMock() - # AttributeError simulates the case where one of these properties - # is None because no connection exists. We raise it for - # ssl_conn.close() instead of ssl_conn.shutdown() for better code - # coverage. - self.authenticator.ssl_conn.close.side_effect = AttributeError("!") - self.authenticator.connection.close.side_effect = AttributeError("!") - self.authenticator.sock.close.side_effect = AttributeError("!") - self.authenticator.subproc_signal_handler(signal.SIGINT, None) - self.assertEquals(self.authenticator.ssl_conn.shutdown.call_count, 1) - self.assertEquals(self.authenticator.ssl_conn.close.call_count, 1) - self.assertEquals(self.authenticator.connection.close.call_count, 1) - self.assertEquals(self.authenticator.sock.close.call_count, 1) - mock_kill.assert_called_once_with( - self.authenticator.parent_pid, signal.SIGUSR1) - mock_exit.assert_called_once_with(0) - - -class PerformTest(unittest.TestCase): - """Tests for perform() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - self.achall1 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"foo"), "pending"), - domain="foo.example.com", account_key=ACCOUNT_KEY) - self.achall2 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"bar"), "pending"), - domain="bar.example.com", account_key=ACCOUNT_KEY) - bad_achall = ("This", "Represents", "A Non-DVSNI", "Challenge") - self.achalls = [self.achall1, self.achall2, bad_achall] - - def test_perform_when_already_listening(self): - self.authenticator.already_listening = mock.Mock() - self.authenticator.already_listening.return_value = True - result = self.authenticator.perform([self.achall1]) - self.assertEqual(result, [None]) - - def test_can_perform(self): - """What happens if start_listener() returns True.""" - self.authenticator.start_listener = mock.Mock() - self.authenticator.start_listener.return_value = True - self.authenticator.already_listening = mock.Mock(return_value=False) - result = self.authenticator.perform(self.achalls) - self.assertEqual(len(self.authenticator.tasks), 2) - self.assertTrue(self.achall1.token in self.authenticator.tasks) - self.assertTrue(self.achall2.token in self.authenticator.tasks) - self.assertTrue(isinstance(result, list)) - self.assertEqual(len(result), 3) - self.assertTrue(isinstance(result[0], challenges.ChallengeResponse)) - self.assertTrue(isinstance(result[1], challenges.ChallengeResponse)) - self.assertFalse(result[2]) - self.authenticator.start_listener.assert_called_once_with( - CONFIG.dvsni_port) - - def test_cannot_perform(self): - """What happens if start_listener() returns False.""" - self.authenticator.start_listener = mock.Mock() - self.authenticator.start_listener.return_value = False - self.authenticator.already_listening = mock.Mock(return_value=False) - result = self.authenticator.perform(self.achalls) - self.assertEqual(len(self.authenticator.tasks), 2) - self.assertTrue(self.achall1.token in self.authenticator.tasks) - self.assertTrue(self.achall2.token in self.authenticator.tasks) - self.assertTrue(isinstance(result, list)) - self.assertEqual(len(result), 3) - self.assertEqual(result, [None, None, False]) - self.authenticator.start_listener.assert_called_once_with( - CONFIG.dvsni_port) - - def test_perform_with_pending_tasks(self): - self.authenticator.tasks = {"footoken.acme.invalid": "cert_data"} - extra_achall = acme_util.DVSNI_P - self.assertRaises( - ValueError, self.authenticator.perform, [extra_achall]) - - def test_perform_without_challenge_list(self): - extra_achall = acme_util.DVSNI_P - # This is wrong because a challenge must be specified. - self.assertRaises(ValueError, self.authenticator.perform, []) - # This is wrong because it must be a list, not a bare challenge. - self.assertRaises( - ValueError, self.authenticator.perform, extra_achall) - # This is wrong because the list must contain at least one challenge. - self.assertRaises( - ValueError, self.authenticator.perform, range(20)) - - -class StartListenerTest(unittest.TestCase): - """Tests for start_listener() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.fork") - def test_start_listener_fork_parent(self, mock_fork): - self.authenticator.do_parent_process = mock.Mock() - self.authenticator.do_parent_process.return_value = True - mock_fork.return_value = 22222 - result = self.authenticator.start_listener(1717) - # start_listener is expected to return the True or False return - # value from do_parent_process. - self.assertTrue(result) - self.assertEqual(self.authenticator.child_pid, 22222) - self.authenticator.do_parent_process.assert_called_once_with(1717) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.fork") - def test_start_listener_fork_child(self, mock_fork): - self.authenticator.do_parent_process = mock.Mock() - self.authenticator.do_child_process = mock.Mock() - mock_fork.return_value = 0 - self.authenticator.start_listener(1717) - self.assertEqual(self.authenticator.child_pid, os.getpid()) - self.authenticator.do_child_process.assert_called_once_with(1717) - - -class DoParentProcessTest(unittest.TestCase): - """Tests for do_parent_process() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_ok(self, mock_get_utility): - self.authenticator.subproc_state = "ready" - result = self.authenticator.do_parent_process(1717) - self.assertTrue(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_inuse(self, mock_get_utility): - self.authenticator.subproc_state = "inuse" - result = self.authenticator.do_parent_process(1717) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_cantbind(self, mock_get_utility): - self.authenticator.subproc_state = "cantbind" - result = self.authenticator.do_parent_process(1717) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "zope.component.getUtility") - def test_do_parent_process_timeout(self, mock_get_utility): - # Normally times out in 5 seconds and returns False. We can - # now set delay_amount to a lower value so that it times out - # faster than it would under normal use. - result = self.authenticator.do_parent_process(1717, delay_amount=1) - self.assertFalse(result) - self.assertEqual(mock_get_utility.call_count, 1) - - -class DoChildProcessTest(unittest.TestCase): - """Tests for do_child_process() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.cert = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"abcdef"), "pending"), - domain="example.com", account_key=ACCOUNT_KEY).gen_cert_and_response( - key_pem=CHALL_KEY_PEM)[1] - self.authenticator.private_key = CHALL_KEY - self.authenticator.tasks = {"abcdef.acme.invalid": self.cert} - self.authenticator.parent_pid = 12345 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_do_child_process_cantbind1( - self, mock_exit, mock_kill, mock_socket): - mock_exit.side_effect = IndentationError("subprocess would exit here") - eaccess = socket.error(socket.errno.EACCES, "Permission denied") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eaccess - mock_socket.return_value = sample_socket - # Using the IndentationError as an error that cannot easily be - # generated at runtime, to indicate the behavior of sys.exit has - # taken effect without actually causing the test process to exit. - # (Just replacing it with a no-op causes logic errors because the - # do_child_process code assumes that calling sys.exit() will - # cause subsequent code not to be executed.) - self.assertRaises( - IndentationError, self.authenticator.do_child_process, 1717) - mock_exit.assert_called_once_with(1) - mock_kill.assert_called_once_with(12345, signal.SIGUSR2) - - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.sys.exit") - def test_do_child_process_cantbind2(self, mock_exit, mock_kill, - mock_socket): - mock_exit.side_effect = IndentationError("subprocess would exit here") - eaccess = socket.error(socket.errno.EADDRINUSE, "Port already in use") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eaccess - mock_socket.return_value = sample_socket - self.assertRaises( - IndentationError, self.authenticator.do_child_process, 1717) - mock_exit.assert_called_once_with(1) - mock_kill.assert_called_once_with(12345, signal.SIGUSR1) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "socket.socket") - def test_do_child_process_cantbind3(self, mock_socket): - """Test case where attempt to bind socket results in an unhandled - socket error. (The expected behavior is arguably wrong because it - will crash the program; the reason for the expected behavior is - that we don't have a way to report arbitrary socket errors.)""" - eio = socket.error(socket.errno.EIO, "Imaginary unhandled error") - sample_socket = mock.MagicMock() - sample_socket.bind.side_effect = eio - mock_socket.return_value = sample_socket - self.assertRaises( - socket.error, self.authenticator.do_child_process, 1717) - - @mock.patch("letsencrypt.plugins.standalone.authenticator." - "OpenSSL.SSL.Connection") - @mock.patch("letsencrypt.plugins.standalone.authenticator.socket.socket") - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - def test_do_child_process_success( - self, mock_kill, mock_socket, mock_connection): - sample_socket = mock.MagicMock() - sample_socket.accept.side_effect = _SocketAcceptOnlyNTimes(2) - mock_socket.return_value = sample_socket - mock_connection.return_value = mock.MagicMock() - self.assertRaises( - CallableExhausted, self.authenticator.do_child_process, 1717) - mock_socket.assert_called_once_with() - sample_socket.bind.assert_called_once_with(("0.0.0.0", 1717)) - sample_socket.listen.assert_called_once_with(1) - self.assertEqual(sample_socket.accept.call_count, 3) - mock_kill.assert_called_once_with(12345, signal.SIGIO) - # TODO: We could have some tests about the fact that the listener - # asks OpenSSL to negotiate a TLS connection (and correctly - # sets the SNI callback function). - - -class CleanupTest(unittest.TestCase): - """Tests for cleanup() method.""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import \ - StandaloneAuthenticator - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - self.achall = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"footoken"), "pending"), - domain="foo.example.com", account_key="key") - self.authenticator.tasks = {self.achall.token: "stuff"} - self.authenticator.child_pid = 12345 - - @mock.patch("letsencrypt.plugins.standalone.authenticator.os.kill") - @mock.patch("letsencrypt.plugins.standalone.authenticator.time.sleep") - def test_cleanup(self, mock_sleep, mock_kill): - mock_sleep.return_value = None - mock_kill.return_value = None - - self.authenticator.cleanup([self.achall]) - - mock_kill.assert_called_once_with(12345, signal.SIGINT) - mock_sleep.assert_called_once_with(1) - - def test_bad_cleanup(self): - self.assertRaises( - ValueError, self.authenticator.cleanup, [achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI(token=b"badtoken"), "pending"), - domain="bad.example.com", account_key="key")]) - - -class MoreInfoTest(unittest.TestCase): - """Tests for more_info() method. (trivially)""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import ( - StandaloneAuthenticator) - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_more_info(self): - """Make sure exceptions aren't raised.""" - self.authenticator.more_info() - - -class InitTest(unittest.TestCase): - """Tests for more_info() method. (trivially)""" - def setUp(self): - from letsencrypt.plugins.standalone.authenticator import ( - StandaloneAuthenticator) - self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None) - - def test_prepare(self): - """Make sure exceptions aren't raised. - - .. todo:: Add on more once things are setup appropriately. - - """ - self.authenticator.prepare() - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/letsencrypt/tests/achallenges_test.py b/letsencrypt/tests/achallenges_test.py index 1cd2f1677..66b1a7ca7 100644 --- a/letsencrypt/tests/achallenges_test.py +++ b/letsencrypt/tests/achallenges_test.py @@ -1,6 +1,8 @@ """Tests for letsencrypt.achallenges.""" import unittest +import OpenSSL + from acme import challenges from acme import jose @@ -22,10 +24,10 @@ class DVSNITest(unittest.TestCase): self.assertEqual(self.challb.token, self.achall.token) def test_gen_cert_and_response(self): - response, cert_pem, key_pem = self.achall.gen_cert_and_response() + response, cert, key = self.achall.gen_cert_and_response() self.assertTrue(isinstance(response, challenges.DVSNIResponse)) - self.assertTrue(isinstance(cert_pem, bytes)) - self.assertTrue(isinstance(key_pem, bytes)) + self.assertTrue(isinstance(cert, OpenSSL.crypto.X509)) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 6e1640e3e..5853c8ac0 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ install_requires = [ 'python2-pythondialog>=3.2.2rc1', # Debian squeeze support, cf. #280 'pytz', 'requests', + 'six', 'zope.component', 'zope.interface', ] @@ -119,8 +120,7 @@ setup( 'manual = letsencrypt.plugins.manual:ManualAuthenticator', # TODO: null should probably not be presented to the user 'null = letsencrypt.plugins.null:Installer', - 'standalone = letsencrypt.plugins.standalone.authenticator' - ':StandaloneAuthenticator', + 'standalone = letsencrypt.plugins.standalone:Authenticator', ], }, From d88455a1b9529545318d113e82e6ab31fc7c72d7 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 09:11:37 +0000 Subject: [PATCH 11/91] Rename simplefs to webroot --- docs/api/plugins/simplefs.rst | 5 ----- docs/api/plugins/webroot.rst | 5 +++++ letsencrypt/plugins/{simplefs.py => webroot.py} | 6 +++--- .../plugins/{simplefs_test.py => webroot_test.py} | 14 +++++++------- setup.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 docs/api/plugins/simplefs.rst create mode 100644 docs/api/plugins/webroot.rst rename letsencrypt/plugins/{simplefs.py => webroot.py} (96%) rename letsencrypt/plugins/{simplefs_test.py => webroot_test.py} (85%) diff --git a/docs/api/plugins/simplefs.rst b/docs/api/plugins/simplefs.rst deleted file mode 100644 index 7165b6aca..000000000 --- a/docs/api/plugins/simplefs.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`letsencrypt.plugins.simplefs` ------------------------------------ - -.. automodule:: letsencrypt.plugins.simplefs - :members: diff --git a/docs/api/plugins/webroot.rst b/docs/api/plugins/webroot.rst new file mode 100644 index 000000000..339d546a5 --- /dev/null +++ b/docs/api/plugins/webroot.rst @@ -0,0 +1,5 @@ +:mod:`letsencrypt.plugins.webroot` +---------------------------------- + +.. automodule:: letsencrypt.plugins.webroot + :members: diff --git a/letsencrypt/plugins/simplefs.py b/letsencrypt/plugins/webroot.py similarity index 96% rename from letsencrypt/plugins/simplefs.py rename to letsencrypt/plugins/webroot.py index ad83c13d7..d641855ae 100644 --- a/letsencrypt/plugins/simplefs.py +++ b/letsencrypt/plugins/webroot.py @@ -1,4 +1,4 @@ -"""SimpleFS plugin.""" +"""Webroot plugin.""" import errno import logging import os @@ -16,11 +16,11 @@ logger = logging.getLogger(__name__) class Authenticator(common.Plugin): - """SimpleFS Authenticator.""" + """Webroot Authenticator.""" zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) - description = "SimpleFS Authenticator" + description = "Webroot Authenticator" MORE_INFO = """\ Authenticator plugin that performs SimpleHTTP challenge by saving diff --git a/letsencrypt/plugins/simplefs_test.py b/letsencrypt/plugins/webroot_test.py similarity index 85% rename from letsencrypt/plugins/simplefs_test.py rename to letsencrypt/plugins/webroot_test.py index f80e8b29a..abd12e152 100644 --- a/letsencrypt/plugins/simplefs_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -1,4 +1,4 @@ -"""Tests for letsencrypt.plugins.simplefs.""" +"""Tests for letsencrypt.plugins.webroot.""" import os import shutil import tempfile @@ -19,19 +19,19 @@ KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) class AuthenticatorTest(unittest.TestCase): - """Tests for letsencrypt.plugins.simplefs.Authenticator.""" + """Tests for letsencrypt.plugins.webroot.Authenticator.""" achall = achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP_P, domain=None, account_key=KEY) def setUp(self): - from letsencrypt.plugins.simplefs import Authenticator + from letsencrypt.plugins.webroot import Authenticator self.root = tempfile.mkdtemp() self.validation_path = os.path.join( self.root, ".well-known", "acme-challenge", "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") - self.config = mock.MagicMock(simplefs_root=self.root) - self.auth = Authenticator(self.config, "simplefs") + self.config = mock.MagicMock(webroot_root=self.root) + self.auth = Authenticator(self.config, "webroot") self.auth.prepare() def tearDown(self): @@ -48,11 +48,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertEqual(1, add.call_count) def test_prepare_bad_root(self): - self.config.simplefs_root = os.path.join(self.root, "null") + self.config.webroot_root = os.path.join(self.root, "null") self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_missing_root(self): - self.config.simplefs_root = None + self.config.webroot_root = None self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_full_root_exists(self): diff --git a/setup.py b/setup.py index a4897aadb..92dd39d46 100644 --- a/setup.py +++ b/setup.py @@ -119,7 +119,7 @@ setup( 'letsencrypt.plugins': [ 'manual = letsencrypt.plugins.manual:Authenticator', 'null = letsencrypt.plugins.null:Installer', - 'simplefs = letsencrypt.plugins.simplefs:Authenticator', + 'webroot = letsencrypt.plugins.webroot:Authenticator', 'standalone = letsencrypt.plugins.standalone.authenticator' ':StandaloneAuthenticator', ], From 63c080b05f2040d3507526efa1cbe10a01bfa565 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 09:15:17 +0000 Subject: [PATCH 12/91] --webroot-root -> --webroot-path --- letsencrypt/plugins/webroot.py | 16 ++++++++-------- letsencrypt/plugins/webroot_test.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index d641855ae..ed8991bc5 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -29,11 +29,11 @@ system. It expects that there is some other HTTP server configured to serve all files under specified web root ({0}).""" def more_info(self): # pylint: disable=missing-docstring,no-self-use - return self.MORE_INFO.format(self.conf("root")) + return self.MORE_INFO.format(self.conf("path")) @classmethod def add_parser_arguments(cls, add): - add("root", help="public_html / webroot path") + add("path", help="public_html / webroot path") def get_chall_pref(self, domain): # pragma: no cover # pylint: disable=missing-docstring,no-self-use,unused-argument @@ -44,15 +44,15 @@ to serve all files under specified web root ({0}).""" self.full_root = None def prepare(self): # pylint: disable=missing-docstring - root = self.conf("root") - if root is None: + path = self.conf("path") + if path is None: raise errors.PluginError("--{0} must be set".format( - self.option_name("root"))) - if not os.path.isdir(root): + self.option_name("path"))) + if not os.path.isdir(path): raise errors.PluginError( - root + " does not exist or is not a directory") + path + " does not exist or is not a directory") self.full_root = os.path.join( - root, challenges.SimpleHTTPResponse.URI_ROOT_PATH) + path, challenges.SimpleHTTPResponse.URI_ROOT_PATH) logger.debug("Creating root challenges validation dir at %s", self.full_root) diff --git a/letsencrypt/plugins/webroot_test.py b/letsencrypt/plugins/webroot_test.py index abd12e152..d8c0e2aa2 100644 --- a/letsencrypt/plugins/webroot_test.py +++ b/letsencrypt/plugins/webroot_test.py @@ -26,21 +26,21 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.webroot import Authenticator - self.root = tempfile.mkdtemp() + self.path = tempfile.mkdtemp() self.validation_path = os.path.join( - self.root, ".well-known", "acme-challenge", + self.path, ".well-known", "acme-challenge", "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ") - self.config = mock.MagicMock(webroot_root=self.root) + self.config = mock.MagicMock(webroot_path=self.path) self.auth = Authenticator(self.config, "webroot") self.auth.prepare() def tearDown(self): - shutil.rmtree(self.root) + shutil.rmtree(self.path) def test_more_info(self): more_info = self.auth.more_info() self.assertTrue(isinstance(more_info, str)) - self.assertTrue(self.root in more_info) + self.assertTrue(self.path in more_info) def test_add_parser_arguments(self): add = mock.MagicMock() @@ -48,11 +48,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertEqual(1, add.call_count) def test_prepare_bad_root(self): - self.config.webroot_root = os.path.join(self.root, "null") + self.config.webroot_path = os.path.join(self.path, "null") self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_missing_root(self): - self.config.webroot_root = None + self.config.webroot_path = None self.assertRaises(errors.PluginError, self.auth.prepare) def test_prepare_full_root_exists(self): @@ -60,10 +60,10 @@ class AuthenticatorTest(unittest.TestCase): self.auth.prepare() # shouldn't raise any exceptions def test_prepare_reraises_other_errors(self): - self.auth.full_root = os.path.join(self.root, "null") - os.chmod(self.root, 0o000) + self.auth.full_path = os.path.join(self.path, "null") + os.chmod(self.path, 0o000) self.assertRaises(errors.PluginError, self.auth.prepare) - os.chmod(self.root, 0o700) + os.chmod(self.path, 0o700) def test_perform_cleanup(self): responses = self.auth.perform([self.achall]) From b4b7b020a28bca4618dc0be438054bc18e17c8c8 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:32:29 +0000 Subject: [PATCH 13/91] Add NamespaceConfigTest.test_simple_http_port --- letsencrypt/tests/configuration_test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index 24a84d888..acf9273d0 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -12,7 +12,7 @@ class NamespaceConfigTest(unittest.TestCase): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', server='https://acme-server.org:443/new', - dvsni_port='1234', simple_http_port='4321') + dvsni_port='1234', simple_http_port=4321) from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) @@ -47,6 +47,16 @@ class NamespaceConfigTest(unittest.TestCase): self.assertEqual(self.config.key_dir, '/tmp/config/keys') self.assertEqual(self.config.temp_checkpoint_dir, '/tmp/foo/t') + def test_simple_http_port(self): + self.assertEqual(4321, self.config.simple_http_port) + + self.namespace.simple_http_port = None + self.namespace.no_simple_http_tls = True + self.assertEqual(80, self.config.simple_http_port) + + self.namespace.no_simple_http_tls = False + self.assertEqual(443, self.config.simple_http_port) + class RenewerConfigurationTest(unittest.TestCase): """Test for letsencrypt.configuration.RenewerConfiguration.""" From b660650a903343c72014969cf2a7d58d6ce1a3b0 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:37:47 +0000 Subject: [PATCH 14/91] Add StandaloneBindErrorTest --- letsencrypt/tests/errors_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/letsencrypt/tests/errors_test.py b/letsencrypt/tests/errors_test.py index a99d84719..5da7c0b7a 100644 --- a/letsencrypt/tests/errors_test.py +++ b/letsencrypt/tests/errors_test.py @@ -1,6 +1,8 @@ """Tests for letsencrypt.errors.""" import unittest +import mock + from acme import messages from letsencrypt import achallenges @@ -22,5 +24,21 @@ class FaiiledChallengesTest(unittest.TestCase): "Failed authorization procedure. example.com (dns): tls")) +class StandaloneBindErrorTest(unittest.TestCase): + """Tests for letsencrypt.errors.StandaloneBindError.""" + + def setUp(self): + from letsencrypt.errors import StandaloneBindError + self.error = StandaloneBindError(mock.sentinel.error, 1234) + + def test_instance_args(self): + self.assertEqual(mock.sentinel.error, self.error.socket_error) + self.assertEqual(1234, self.error.port) + + def test_str(self): + self.assertTrue(str(self.error).startswith( + "Problem binding to port 1234: ")) + + if __name__ == "__main__": unittest.main() # pragma: no cover From b0efc61f97e2a29adc1ac5c4c5439ec4cc70d77b Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 10:58:31 +0000 Subject: [PATCH 15/91] Add ServerManagerTest stub. --- letsencrypt/plugins/standalone.py | 2 +- letsencrypt/plugins/standalone_test.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 letsencrypt/plugins/standalone_test.py diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 8e5f1e77d..4444ed2c1 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -49,7 +49,7 @@ class ServerManager(object): cls = BaseHTTPServer.HTTPServer try: - server = cls(('', port), handler) + server = cls(("", port), handler) except socket.error as error: errors.StandaloneBindError(error, port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py new file mode 100644 index 000000000..8c0dd6f97 --- /dev/null +++ b/letsencrypt/plugins/standalone_test.py @@ -0,0 +1,21 @@ +"""Tests for letsencrypt.plugins.standalone.""" +import unittest + + +class ServerManagerTest(unittest.TestCase): + """Tests for letsencrypt.plugins.standalone.ServerManager.""" + + def setUp(self): + from letsencrypt.plugins.standalone import ServerManager + self.certs = {} + self.simple_http_resources = {} + self.mgr = ServerManager(self.certs, self.simple_http_resources) + + def test_init(self): + self.assertTrue(self.mgr.certs is self.certs) + self.assertTrue( + self.mgr.simple_http_resources is self.simple_http_resources) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover From d1fcc422e0aa218d579028f0536c7214fe68d2ba Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:09:42 +0000 Subject: [PATCH 16/91] Use ACME(TLS)Server in plugins.standalone --- letsencrypt/plugins/standalone.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 4444ed2c1..5f2bc9292 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -44,9 +44,9 @@ class ServerManager(object): if tls: cls = functools.partial( - acme_standalone.HTTPSServer, certs=self.certs) + acme_standalone.ACMETLSServer, certs=self.certs) else: - cls = BaseHTTPServer.HTTPServer + cls = acme_standalone.ACMEServer try: server = cls(("", port), handler) From 22b1514f516b2404a7db748a7f0adbd40cf55fc4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:01:20 +0000 Subject: [PATCH 17/91] server_forever2/shutdown2 --- acme/acme/standalone.py | 37 ++++++++++++++++++++++++++++++- acme/acme/standalone_test.py | 23 ++++++++++++++++++- letsencrypt/plugins/standalone.py | 32 ++++++-------------------- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 8f604a439..501d239cb 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -4,6 +4,7 @@ import collections import functools import logging import os +import socket import sys import six @@ -52,18 +53,52 @@ class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): class ACMEServerMixin: # pylint: disable=old-style-class,no-init - """ACME server common settings mixin.""" + """ACME server common settings mixin. + + .. warning:: + Subclasses have to init ``_stopped = False`` (it's not done here, + because of old-style classes madness). + + """ server_version = "ACME standalone client" allow_reuse_address = True + def serve_forever2(self): + """Serve forever, until other thread calls `shutdown2`.""" + while not self._stopped: + self.handle_request() + + def shutdown2(self): + """Shutdown server loop from `serve_forever2`.""" + self._stopped = True + + # dummy request to terminate last server_forever2.handle_request() + sock = socket.socket() + try: + sock.connect(self.socket.getsockname()) + except socket.error: + pass # thread is probably already finished + finally: + sock.close() + + self.server_close() + class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ACME TLS Server.""" + def __init__(self, *args, **kwargs): + self._stopped = False + HTTPSServer.__init__(self, *args, **kwargs) + class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): """ACME Server (non-TLS).""" + def __init__(self, *args, **kwargs): + self._stopped = False + BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) + class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """SimpleHTTP challenge handler. diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 794fb1a6e..9ff99f5ff 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -25,6 +25,26 @@ class TLSServerTest(unittest.TestCase): server.server_close() # pylint: disable=no-member +class ACMEServerMixinTest(unittest.TestCase): + """Tests for acme.standalone.ACMEServerMixin.""" + + def test_shutdown2_not_running(self): + from acme.standalone import ACMEServer + server = ACMEServer(("", 0), socketserver.BaseRequestHandler) + server.shutdown2() + server.shutdown2() + + +class ACMEServerTest(unittest.TestCase): + """Test for acme.standalone.ACMEServer.""" + + def test_init(self): + from acme.standalone import ACMEServer + server = ACMEServer(("", 0), socketserver.BaseRequestHandler) + # pylint: disable=protected-access + self.assertFalse(server._stopped) + + class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): """End-to-end test for ACME TLS server with SimpleHTTP.""" @@ -45,12 +65,13 @@ class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): self.server = ACMETLSServer(('', 0), handler, certs=self.certs) self.server_thread = threading.Thread( # pylint: disable=no-member - target=self.server.handle_request) + target=self.server.serve_forever2) self.server_thread.start() self.port = self.server.socket.getsockname()[1] def tearDown(self): + self.server.shutdown2() self.server_thread.join() def test_index(self): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 5f2bc9292..071fdbf42 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -53,33 +53,15 @@ class ServerManager(object): except socket.error as error: errors.StandaloneBindError(error, port) - stop = threading.Event() - thread = threading.Thread( - target=self._serve, - args=(server, stop), - ) + thread = threading.Thread(target=server.serve_forever2) thread.start() - self.servers[port] = (server, thread, stop) + self.servers[port] = (server, thread) return self.servers[port] - def _serve(self, server, stop): - while not stop.is_set(): - server.handle_request() - def stop(self, port): """Stop ACME server running on the specified ``port``.""" - server, thread, stop = self.servers[port] - stop.set() - - # dummy request to terminate last handle_request() - sock = socket.socket() - try: - sock.connect(server.socket.getsockname()) - except socket.error: - pass # thread is probably already finished - finally: - sock.close() - + server, thread = self.servers[port] + server.shutdown2() thread.join() del self.servers[port] @@ -170,7 +152,7 @@ class Authenticator(common.Plugin): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server, _, _ = self.servers.run(self.config.simple_http_port, tls=tls) + server, _ = self.servers.run(self.config.simple_http_port, tls=tls) response, validation = achall.gen_response_and_validation(tls=tls) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -179,7 +161,7 @@ class Authenticator(common.Plugin): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server, _, _ = self.servers.run(self.config.dvsni_port, tls=True) + server, _ = self.servers.run(self.config.dvsni_port, tls=True) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) @@ -195,6 +177,6 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _, _) in self.servers.items(): + for port, (server, _) in self.servers.items(): if not self.served[server]: self.servers.stop(port) From 7687ecd6e36dbb15fcc633bc3c98123cf9605859 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 11:41:19 +0000 Subject: [PATCH 18/91] 100% coverage for standalone.ServerManager --- letsencrypt/plugins/standalone.py | 9 ++++--- letsencrypt/plugins/standalone_test.py | 34 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 071fdbf42..b3fb92ad1 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -51,12 +51,15 @@ class ServerManager(object): try: server = cls(("", port), handler) except socket.error as error: - errors.StandaloneBindError(error, port) + raise errors.StandaloneBindError(error, port) + + # if port == 0, then random free port on OS is taken + real_port = server.socket.getsockname() thread = threading.Thread(target=server.serve_forever2) thread.start() - self.servers[port] = (server, thread) - return self.servers[port] + self.servers[real_port] = (server, thread) + return self.servers[real_port] def stop(self, port): """Stop ACME server running on the specified ``port``.""" diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 8c0dd6f97..ad8122c78 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -1,6 +1,9 @@ """Tests for letsencrypt.plugins.standalone.""" +import socket import unittest +from letsencrypt import errors + class ServerManagerTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.ServerManager.""" @@ -16,6 +19,37 @@ class ServerManagerTest(unittest.TestCase): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) + def test_run_stop_non_tls(self): + server, thread = self.mgr.run(port=0, tls=False) + self.mgr.stop(port=server.socket.getsockname()) + + def test_run_stop_tls(self): + server, thread = self.mgr.run(port=0, tls=True) + self.mgr.stop(port=server.socket.getsockname()) + + def test_run_idempotent(self): + server, thread = self.mgr.run(port=0, tls=False) + port = server.socket.getsockname() + server2, thread2 = self.mgr.run(port=port, tls=False) + self.assertTrue(server is server2) + self.assertTrue(thread2 is thread2) + self.mgr.stop(port) + + def test_run_bind_error(self): + some_server = socket.socket() + some_server.bind(("", 0)) + port = some_server.getsockname()[1] + self.assertRaises( + errors.StandaloneBindError, self.mgr.run, port, tls=False) + + def test_items(self): + server, thread = self.mgr.run(port=0, tls=True) + port = server.socket.getsockname() + self.assertEqual(port, self.mgr.items()[0][0]) + self.assertTrue(self.mgr.items()[0][1][0] is server) + self.assertTrue(self.mgr.items()[0][1][1] is thread) + self.mgr.stop(port=port) + if __name__ == "__main__": unittest.main() # pragma: no cover From bba0560c0ea0a368aa46a80ba173a3095011c991 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 13:24:30 +0000 Subject: [PATCH 19/91] Almost full coverage for plugins.standalone (perform2 left). --- letsencrypt/plugins/standalone.py | 3 +- letsencrypt/plugins/standalone_test.py | 85 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index b3fb92ad1..bfe4a6606 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -99,7 +99,6 @@ class Authenticator(common.Plugin): self.key, domains=["temp server"]) self.responses = {} - self.servers = {} self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read @@ -118,7 +117,7 @@ class Authenticator(common.Plugin): if any(util.already_listening(port) for port in (self.config.dvsni_port, self.config.simple_http_port)): raise errors.MisconfigurationError( - "One of the (possibly) required ports is already taken taken.") + "One of the (possibly) required ports is already taken.") # TODO: add --chall-pref flag def get_chall_pref(self, domain): diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index ad8122c78..8b76633c9 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -2,7 +2,13 @@ import socket import unittest +import mock +import six + +from acme import challenges + from letsencrypt import errors +from letsencrypt import interfaces class ServerManagerTest(unittest.TestCase): @@ -51,5 +57,84 @@ class ServerManagerTest(unittest.TestCase): self.mgr.stop(port=port) +class AuthenticatorTest(unittest.TestCase): + """Tests for letsencrypt.plugins.standalone.Authenticator.""" + + def setUp(self): + from letsencrypt.plugins.standalone import Authenticator + config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.auth = Authenticator(config, name="standalone") + + def test_more_info(self): + self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + + @mock.patch("letsencrypt.plugins.standalone.util") + def test_prepare_misconfiguration(self, mock_util): + mock_util.already_listening.return_value = True + self.assertRaises(errors.MisconfigurationError, self.auth.prepare) + mock_util.already_listening.assert_called_once_with(1234) + + def test_get_chall_pref(self): + self.assertEqual(set(self.auth.get_chall_pref(domain=None)), + set([challenges.SimpleHTTP, challenges.DVSNI])) + + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") + def test_perform(self, mock_get_utility): + achalls = [1, 2, 3] + self.auth.perform2 = mock.Mock(return_value=mock.sentinel.responses) + self.assertEqual(mock.sentinel.responses, self.auth.perform(achalls)) + self.auth.perform2.assert_called_once_with(achalls) + + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") + def _test_perform_bind_errors(self, errno, achalls, mock_get_utility): + def _perform2(achalls): + raise errors.StandaloneBindError(mock.Mock(errno=errno), 1234) + + self.auth.perform2 = mock.MagicMock(side_effect=_perform2) + self.auth.perform(achalls) + mock_get_utility.assert_called_once_with(interfaces.IDisplay) + notification = mock_get_utility.return_value.notification + self.assertEqual(1, notification.call_count) + self.assertTrue("1234" in notification.call_args[0][0]) + + def test_perform_eacces(self): + self._test_perform_bind_errors(socket.errno.EACCES, []) + + def test_perform_eaddrinuse(self): + self._test_perform_bind_errors(socket.errno.EADDRINUSE, []) + + def test_perfom_unknown_bind_error(self): + self.assertRaises( + errors.StandaloneBindError, self._test_perform_bind_errors, + socket.errno.ENOTCONN, []) + + def test_cleanup(self): + servers = {1: "server1", 2: "server2"} + self.auth.servers = mock.Mock() + self.auth.servers.items.return_value = [ + (1, ("server1", "thread1")), + (2, ("server2", "thread2")), + ] + self.auth.served["server1"].add("chall1") + self.auth.served["server2"].update(["chall2", "chall3"]) + + self.auth.cleanup(["chall1"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set(["chall2", "chall3"])}) + self.auth.servers.stop.assert_called_once_with(1) + + self.auth.servers.items.return_value = [ + (2, ("server2", "thread2")), + ] + self.auth.cleanup(["chall2"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set(["chall3"])}) + self.assertEqual(1, self.auth.servers.stop.call_count) + + self.auth.cleanup(["chall3"]) + self.assertEqual(self.auth.served, { + "server1": set(), "server2": set([])}) + self.auth.servers.stop.assert_called_with(2) + if __name__ == "__main__": unittest.main() # pragma: no cover From 517a74f432758b27615b1cc4361968dcbe77bd9a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:21:13 +0000 Subject: [PATCH 20/91] standalone 2.0: lint, docs, cleanup. --- acme/acme/crypto_util.py | 8 ++- letsencrypt/plugins/standalone.py | 74 +++++++++++++++++++------- letsencrypt/plugins/standalone_test.py | 33 ++++++------ 3 files changed, 78 insertions(+), 37 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 32533630b..9ea5812b3 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,8 +27,14 @@ _DEFAULT_DVSNI_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD class SSLSocket(object): # pylint: disable=too-few-public-methods - """SSL wrapper for sockets.""" + """SSL wrapper for sockets. + :ivar socket sock: Original wrapped socket. + :ivar dict certs: Mapping from domain names (`bytes`) to + `OpenSSL.crypto.X509`. + :ivar method: See `OpenSSL.SSL.Context` for allowed values. + + """ def __init__(self, sock, certs, method=_DEFAULT_DVSNI_SSL_METHOD): self.sock = sock self.certs = certs diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index bfe4a6606..cde673b2c 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -6,8 +6,6 @@ import random import socket import threading -from six.moves import BaseHTTPServer # pylint: disable=import-error - import OpenSSL import zope.interface @@ -26,17 +24,39 @@ logger = logging.getLogger(__name__) class ServerManager(object): - """Standalone servers manager.""" + """Standalone servers manager. + Manager for `ACMEServer` and `ACMETLSServer` instances. + + `certs` and `simple_http_resources` correspond to + `acme.crypto_util.SSLSocket.certs` and + `acme.crypto_util.SSLSocket.simple_http_resources` respectively. All + created servers share the same certificates and resources, so if + you're running both TLS and non-TLS instances, SimpleHTTP handlers + will serve the same URLs! + + """ def __init__(self, certs, simple_http_resources): - self.servers = {} + self._servers = {} self.certs = certs self.simple_http_resources = simple_http_resources def run(self, port, tls): - """Run ACME server on specified ``port``.""" - if port in self.servers: - return self.servers[port] + """Run ACME server on specified ``port``. + + This method is idempotent, i.e. all calls with the same pair of + ``(port, tls)`` will reuse the same server. + + :param int port: Port to run the server on. + :param bool tls: TLS or non-TLS? + + :returns: Server instance (`ACMEServerMixin`) and the + corresponding (already started) thread (`threading.Thread`). + :rtype: tuple + + """ + if port in self._servers: + return self._servers[port] logger.debug("Starting new server at %s (tls=%s)", port, tls) handler = acme_standalone.ACMERequestHandler.partial_init( @@ -54,23 +74,38 @@ class ServerManager(object): raise errors.StandaloneBindError(error, port) # if port == 0, then random free port on OS is taken - real_port = server.socket.getsockname() + # pylint: disable=no-member + host, real_port = server.socket.getsockname() thread = threading.Thread(target=server.serve_forever2) + logger.debug("Starting server at %s:%d", host, real_port) thread.start() - self.servers[real_port] = (server, thread) - return self.servers[real_port] + + self._servers[real_port] = (server, thread) + return self._servers[real_port] def stop(self, port): - """Stop ACME server running on the specified ``port``.""" - server, thread = self.servers[port] + """Stop ACME server running on the specified ``port``. + + :param int port: + + """ + server, thread = self._servers[port] server.shutdown2() thread.join() - del self.servers[port] + del self._servers[port] - def items(self): - """Return a list of all port, server tuples.""" - return self.servers.items() + def running(self): + """Return all running instances. + + Once the server is stopped using `stop`, it will not be + returned. + + :returns: ``(port, (server, thread))`` + :rtype: tuple + + """ + return self._servers.items() class Authenticator(common.Plugin): @@ -98,7 +133,6 @@ class Authenticator(common.Plugin): self.simple_http_cert = acme_crypto_util.gen_ss_cert( self.key, domains=["temp server"]) - self.responses = {} self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read @@ -117,7 +151,8 @@ class Authenticator(common.Plugin): if any(util.already_listening(port) for port in (self.config.dvsni_port, self.config.simple_http_port)): raise errors.MisconfigurationError( - "One of the (possibly) required ports is already taken.") + "At least one of the (possibly) required ports is " + "already taken.") # TODO: add --chall-pref flag def get_chall_pref(self, domain): @@ -167,7 +202,6 @@ class Authenticator(common.Plugin): response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) - self.responses[achall] = response self.served[server].add(achall) responses.append(response) @@ -179,6 +213,6 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in self.servers.items(): + for port, (server, _) in self.servers.running(): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 8b76633c9..2cde623ac 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -26,19 +26,19 @@ class ServerManagerTest(unittest.TestCase): self.mgr.simple_http_resources is self.simple_http_resources) def test_run_stop_non_tls(self): - server, thread = self.mgr.run(port=0, tls=False) - self.mgr.stop(port=server.socket.getsockname()) + server, _ = self.mgr.run(port=0, tls=False) + self.mgr.stop(port=server.socket.getsockname()[1]) def test_run_stop_tls(self): - server, thread = self.mgr.run(port=0, tls=True) - self.mgr.stop(port=server.socket.getsockname()) + server, _ = self.mgr.run(port=0, tls=True) + self.mgr.stop(port=server.socket.getsockname()[1]) def test_run_idempotent(self): server, thread = self.mgr.run(port=0, tls=False) - port = server.socket.getsockname() + port = server.socket.getsockname()[1] server2, thread2 = self.mgr.run(port=port, tls=False) self.assertTrue(server is server2) - self.assertTrue(thread2 is thread2) + self.assertTrue(thread is thread2) self.mgr.stop(port) def test_run_bind_error(self): @@ -48,12 +48,12 @@ class ServerManagerTest(unittest.TestCase): self.assertRaises( errors.StandaloneBindError, self.mgr.run, port, tls=False) - def test_items(self): + def test_running(self): server, thread = self.mgr.run(port=0, tls=True) - port = server.socket.getsockname() - self.assertEqual(port, self.mgr.items()[0][0]) - self.assertTrue(self.mgr.items()[0][1][0] is server) - self.assertTrue(self.mgr.items()[0][1][1] is thread) + port = server.socket.getsockname()[1] + self.assertEqual(port, self.mgr.running()[0][0]) + self.assertTrue(self.mgr.running()[0][1][0] is server) + self.assertTrue(self.mgr.running()[0][1][1] is thread) self.mgr.stop(port=port) @@ -79,7 +79,7 @@ class AuthenticatorTest(unittest.TestCase): set([challenges.SimpleHTTP, challenges.DVSNI])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") - def test_perform(self, mock_get_utility): + def test_perform(self, unused_mock_get_utility): achalls = [1, 2, 3] self.auth.perform2 = mock.Mock(return_value=mock.sentinel.responses) self.assertEqual(mock.sentinel.responses, self.auth.perform(achalls)) @@ -87,7 +87,7 @@ class AuthenticatorTest(unittest.TestCase): @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def _test_perform_bind_errors(self, errno, achalls, mock_get_utility): - def _perform2(achalls): + def _perform2(unused_achalls): raise errors.StandaloneBindError(mock.Mock(errno=errno), 1234) self.auth.perform2 = mock.MagicMock(side_effect=_perform2) @@ -98,9 +98,11 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue("1234" in notification.call_args[0][0]) def test_perform_eacces(self): + # pylint: disable=no-value-for-parameter self._test_perform_bind_errors(socket.errno.EACCES, []) def test_perform_eaddrinuse(self): + # pylint: disable=no-value-for-parameter self._test_perform_bind_errors(socket.errno.EADDRINUSE, []) def test_perfom_unknown_bind_error(self): @@ -109,9 +111,8 @@ class AuthenticatorTest(unittest.TestCase): socket.errno.ENOTCONN, []) def test_cleanup(self): - servers = {1: "server1", 2: "server2"} self.auth.servers = mock.Mock() - self.auth.servers.items.return_value = [ + self.auth.servers.running.return_value = [ (1, ("server1", "thread1")), (2, ("server2", "thread2")), ] @@ -123,7 +124,7 @@ class AuthenticatorTest(unittest.TestCase): "server1": set(), "server2": set(["chall2", "chall3"])}) self.auth.servers.stop.assert_called_once_with(1) - self.auth.servers.items.return_value = [ + self.auth.servers.running.return_value = [ (2, ("server2", "thread2")), ] self.auth.cleanup(["chall2"]) From df04938f6ae628dfb32d98730dd7f51849ab59d9 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:45:49 +0000 Subject: [PATCH 21/91] Standalone 2.0: add detection for unsupported SimpleHTTP TLS platform. --- acme/acme/crypto_util.py | 12 +++++++++++- acme/acme/standalone.py | 7 +++++++ letsencrypt/plugins/standalone.py | 11 ++++++++--- letsencrypt/plugins/standalone_test.py | 19 +++++++++++++++---- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 9ea5812b3..5829a511f 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -70,6 +70,15 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" + MAKEFILE_SUPPORT = hasattr(socket, "_fileobject") + """Is `makefile` supported on your platform? + + .. warning:: `makefile`, as currently implemented, is supported + on select platforms only, as it uses CPython's internal API. + You've been warned! + + """ + # pylint: disable=missing-docstring def __init__(self, connection): @@ -85,9 +94,10 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods # stuff below ripped off from # https://hg.python.org/cpython/file/2.7/Lib/ssl.py - # XXX: this uses Python's internal API def makefile(self, mode='r', bufsize=-1): + assert self.MAKEFILE_SUPPORT, ( + "You need compatible version for makefile support") self._makefile_refs += 1 # SocketServer.StreamRequesthandler.finish will try to # close the wfile/rfile. close=True causes curl: (56) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 501d239cb..ce10407e6 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -87,6 +87,13 @@ class ACMEServerMixin: # pylint: disable=old-style-class,no-init class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ACME TLS Server.""" + SIMPLE_HTTP_SUPPORT = crypto_util.SSLSocket.FakeConnection.MAKEFILE_SUPPORT + """Is SimpleHTTP supported on your platform. + + Please see a warning for `acme.crypto_util.SSLSocket.FakeConnection`. + + """ + def __init__(self, *args, **kwargs): self._stopped = False HTTPSServer.__init__(self, *args, **kwargs) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cde673b2c..08d1e5c63 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -4,6 +4,7 @@ import functools import logging import random import socket +import sys import threading import OpenSSL @@ -121,7 +122,6 @@ class Authenticator(common.Plugin): zope.interface.classProvides(interfaces.IPluginFactory) description = "Standalone Authenticator" - supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -136,7 +136,7 @@ class Authenticator(common.Plugin): self.served = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read - # values, main thread writes). Due to the nature of Cython's + # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe self.certs = {} @@ -157,7 +157,12 @@ class Authenticator(common.Plugin): # TODO: add --chall-pref flag def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - chall_pref = list(self.supported_challenges) + supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + if not self.config.no_simple_http_tls and not ( + acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): + logger.debug("SimpleHTTPS not supported: %s", sys.version) + supported_challenges.discard(challenges.SimpleHTTP) + chall_pref = list(supported_challenges) random.shuffle(chall_pref) # 50% for each challenge return chall_pref diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 2cde623ac..7fff7b972 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -62,8 +62,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.standalone import Authenticator - config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) - self.auth = Authenticator(config, name="standalone") + self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.auth = Authenticator(self.config, name="standalone") def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) @@ -74,9 +74,20 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.MisconfigurationError, self.auth.prepare) mock_util.already_listening.assert_called_once_with(1234) - def test_get_chall_pref(self): + @mock.patch("letsencrypt.plugins.standalone.acme_standalone") + def test_get_chall_pref_tls_supported(self, mock_astandalone): + mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = True + for no_simple_http_tls in True, False: + self.config.no_simple_http_tls = no_simple_http_tls + self.assertEqual(set(self.auth.get_chall_pref(domain=None)), + set([challenges.DVSNI, challenges.SimpleHTTP])) + + @mock.patch("letsencrypt.plugins.standalone.acme_standalone") + def test_get_chall_pref_simple_tls_not_supported(self, mock_astandalone): + mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = False + self.config.no_simple_http_tls = False self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.SimpleHTTP, challenges.DVSNI])) + set([challenges.DVSNI])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): From 1a0f8889ad741f98e7669070f70c8339fc53f38a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 17:54:58 +0000 Subject: [PATCH 22/91] ServerManager.running returns dict --- letsencrypt/plugins/standalone.py | 7 +++-- letsencrypt/plugins/standalone_test.py | 40 +++++++++++++------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 08d1e5c63..33eabbfab 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -8,6 +8,7 @@ import sys import threading import OpenSSL +import six import zope.interface from acme import challenges @@ -102,11 +103,11 @@ class ServerManager(object): Once the server is stopped using `stop`, it will not be returned. - :returns: ``(port, (server, thread))`` + :returns: Mapping from port to ``(server, thread)``. :rtype: tuple """ - return self._servers.items() + return self._servers.copy() class Authenticator(common.Plugin): @@ -218,6 +219,6 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in self.servers.running(): + for port, (server, _) in six.iteritems(self.servers.running()): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 7fff7b972..a9f321cf3 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -25,21 +25,28 @@ class ServerManagerTest(unittest.TestCase): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) - def test_run_stop_non_tls(self): - server, _ = self.mgr.run(port=0, tls=False) - self.mgr.stop(port=server.socket.getsockname()[1]) + def _test_run_stop(self, tls): + server, _ = self.mgr.run(port=0, tls=tls) + port = server.socket.getsockname()[1] + self.assertEqual(self.mgr.running(), {port: (server, mock.ANY)}) + self.mgr.stop(port=port) + self.assertEqual(self.mgr.running(), {}) def test_run_stop_tls(self): - server, _ = self.mgr.run(port=0, tls=True) - self.mgr.stop(port=server.socket.getsockname()[1]) + self._test_run_stop(tls=True) + + def test_run_stop_non_tls(self): + self._test_run_stop(tls=False) def test_run_idempotent(self): server, thread = self.mgr.run(port=0, tls=False) port = server.socket.getsockname()[1] server2, thread2 = self.mgr.run(port=port, tls=False) + self.assertEqual(self.mgr.running(), {port: (server, thread)}) self.assertTrue(server is server2) self.assertTrue(thread is thread2) self.mgr.stop(port) + self.assertEqual(self.mgr.running(), {}) def test_run_bind_error(self): some_server = socket.socket() @@ -47,14 +54,7 @@ class ServerManagerTest(unittest.TestCase): port = some_server.getsockname()[1] self.assertRaises( errors.StandaloneBindError, self.mgr.run, port, tls=False) - - def test_running(self): - server, thread = self.mgr.run(port=0, tls=True) - port = server.socket.getsockname()[1] - self.assertEqual(port, self.mgr.running()[0][0]) - self.assertTrue(self.mgr.running()[0][1][0] is server) - self.assertTrue(self.mgr.running()[0][1][1] is thread) - self.mgr.stop(port=port) + self.assertEqual(self.mgr.running(), {}) class AuthenticatorTest(unittest.TestCase): @@ -123,10 +123,10 @@ class AuthenticatorTest(unittest.TestCase): def test_cleanup(self): self.auth.servers = mock.Mock() - self.auth.servers.running.return_value = [ - (1, ("server1", "thread1")), - (2, ("server2", "thread2")), - ] + self.auth.servers.running.return_value = { + 1: ("server1", "thread1"), + 2: ("server2", "thread2"), + } self.auth.served["server1"].add("chall1") self.auth.served["server2"].update(["chall2", "chall3"]) @@ -135,9 +135,9 @@ class AuthenticatorTest(unittest.TestCase): "server1": set(), "server2": set(["chall2", "chall3"])}) self.auth.servers.stop.assert_called_once_with(1) - self.auth.servers.running.return_value = [ - (2, ("server2", "thread2")), - ] + self.auth.servers.running.return_value = { + 2: ("server2", "thread2"), + } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { "server1": set(), "server2": set(["chall3"])}) From 774dc7db9aaeacd9562429a98bdc4fc791ba7f2f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:23:26 +0000 Subject: [PATCH 23/91] 100% coverage for Standalone 2.0 --- letsencrypt/plugins/standalone_test.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index a9f321cf3..d5836eab4 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -6,10 +6,16 @@ import mock import six from acme import challenges +from acme import jose +from acme import standalone as acme_standalone +from letsencrypt import achallenges from letsencrypt import errors from letsencrypt import interfaces +from letsencrypt.tests import acme_util +from letsencrypt.tests import test_util + class ServerManagerTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.ServerManager.""" @@ -121,6 +127,39 @@ class AuthenticatorTest(unittest.TestCase): errors.StandaloneBindError, self._test_perform_bind_errors, socket.errno.ENOTCONN, []) + def test_perform2(self): + domain = b'localhost' + key = jose.JWK.load(test_util.load_vector('rsa512_key.pem')) + simple_http = achallenges.SimpleHTTP( + challb=acme_util.SIMPLE_HTTP_P, domain=domain, account_key=key) + dvsni = achallenges.DVSNI( + challb=acme_util.DVSNI_P, domain=domain, account_key=key) + + self.auth.servers = mock.MagicMock() + + def _run(port, tls): # pylint: disable=unused-argument + return "server{0}".format(port), "thread{0}".format(port) + + self.auth.servers.run.side_effect = _run + responses = self.auth.perform2([simple_http, dvsni]) + + self.assertTrue(isinstance(responses, list)) + self.assertEqual(2, len(responses)) + self.assertTrue(isinstance(responses[0], challenges.SimpleHTTPResponse)) + self.assertTrue(isinstance(responses[1], challenges.DVSNIResponse)) + + self.assertEqual(self.auth.servers.run.mock_calls, [ + mock.call(4321, tls=False), mock.call(1234, tls=True)]) + self.assertEqual(self.auth.served, { + "server1234": set([dvsni]), + "server4321": set([simple_http]), + }) + self.assertEqual(1, len(self.auth.simple_http_resources)) + self.assertEqual(2, len(self.auth.certs)) + self.assertEqual(list(self.auth.simple_http_resources), [ + acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( + acme_util.SIMPLE_HTTP, responses[0], mock.ANY)]) + def test_cleanup(self): self.auth.servers = mock.Mock() self.auth.servers.running.return_value = { From ea45fc6504af1c25e3faeab72f457839da51a5ea Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:32:49 +0000 Subject: [PATCH 24/91] TestSimpleServer: don't rely on symlinks --- acme/acme/standalone_test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 9ff99f5ff..a6f7502b6 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -1,6 +1,8 @@ """Tests for acme.standalone.""" import os +import shutil import threading +import tempfile import time import unittest @@ -115,21 +117,28 @@ class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): class TestSimpleServer(unittest.TestCase): """Tests for acme.standalone.simple_server.""" - TEST_CWD = os.path.join(os.path.dirname(__file__), '..', 'examples', 'standalone') - def setUp(self): + # mirror ../examples/standalone + self.test_cwd = tempfile.mkdtemp() + localhost_dir = os.path.join(self.test_cwd, 'localhost') + os.makedirs(localhost_dir) + shutil.copy(test_util.vector_path('cert.pem'), localhost_dir) + shutil.copy(test_util.vector_path('rsa512_key.pem'), + os.path.join(localhost_dir, 'key.pem')) + from acme.standalone import simple_server self.thread = threading.Thread(target=simple_server, kwargs={ 'cli_args': ('xxx', '--port', '1234'), 'forever': False, }) self.old_cwd = os.getcwd() - os.chdir(self.TEST_CWD) + os.chdir(self.test_cwd) self.thread.start() def tearDown(self): os.chdir(self.old_cwd) self.thread.join() + shutil.rmtree(self.test_cwd) def test_it(self): max_attempts = 5 From 509af11a92ec84a94e03c75d6ffb86ac8f068940 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 18:53:36 +0000 Subject: [PATCH 25/91] --standalone-supported-chalenges --- letsencrypt/plugins/standalone.py | 37 ++++++++++++++++++++++++-- letsencrypt/plugins/standalone_test.py | 3 ++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 33eabbfab..b76095f89 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -1,4 +1,5 @@ """Standalone Authenticator.""" +import argparse import collections import functools import logging @@ -110,6 +111,27 @@ class ServerManager(object): return self._servers.copy() +SUPPORTED_CHALLENGES = set([challenges.DVSNI, challenges.SimpleHTTP]) + + +def supported_challenges_validator(data): + """Supported challenges validator.""" + challs = data.split(",") + unrecognized = [name for name in challs + if name not in challenges.Challenge.TYPES] + if unrecognized: + raise argparse.ArgumentTypeError( + "Unrecognized challenges: {0}".format(", ".join(unrecognized))) + + choices = set(chall.typ for chall in SUPPORTED_CHALLENGES) + if not set(challs).issubset(choices): + raise argparse.ArgumentTypeError( + "Plugin does not support the following (valid) " + "challenges: {0}".format(", ".join(challs - choices))) + + return data + + class Authenticator(common.Plugin): """Standalone Authenticator. @@ -145,6 +167,18 @@ class Authenticator(common.Plugin): self.servers = ServerManager(self.certs, self.simple_http_resources) + @classmethod + def add_parser_arguments(cls, add): + add("supported-challenges", help="Supported challenges, " + "order preferences are randomly chosen.", + type=supported_challenges_validator, default=",".join( + sorted(chall.typ for chall in SUPPORTED_CHALLENGES))) + + @property + def supported_challenges(self): + return set(challenges.Challenge.TYPES[name] for name in + self.conf("supported-challenges").split(",")) + def more_info(self): # pylint: disable=missing-docstring return self.__doc__ @@ -155,10 +189,9 @@ class Authenticator(common.Plugin): "At least one of the (possibly) required ports is " "already taken.") - # TODO: add --chall-pref flag def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - supported_challenges = set([challenges.SimpleHTTP, challenges.DVSNI]) + supported_challenges = self.supported_challenges if not self.config.no_simple_http_tls and not ( acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): logger.debug("SimpleHTTPS not supported: %s", sys.version) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index d5836eab4..4a077a36f 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -68,7 +68,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.standalone import Authenticator - self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321) + self.config = mock.MagicMock(dvsni_port=1234, simple_http_port=4321, + standalone_supported_challenges="dvsni,simpleHttp") self.auth = Authenticator(self.config, name="standalone") def test_more_info(self): From 827c66c6674b3a98beaaf31362de496333efd1ea Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:19:36 +0000 Subject: [PATCH 26/91] Integration tests: standalone dvsni and simpleHttp. --- tests/boulder-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ed877d136..210f13c23 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -21,8 +21,8 @@ common() { "$@" } -common --domains le1.wtf auth -common --domains le2.wtf run +common --domains le1.wtf --standalone-supported-challenges dvsni auth +common --domains le2.wtf --standalone-supported-challenges simpleHttp run common -a manual -d le.wtf auth common -a manual -d le.wtf --no-simple-http-tls auth From 4d25cabfe2ae2a0e23324b9c53331cbfe658c51a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:27:47 +0000 Subject: [PATCH 27/91] Add missing docstring --- letsencrypt/plugins/standalone.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index b76095f89..17e99073f 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -176,6 +176,7 @@ class Authenticator(common.Plugin): @property def supported_challenges(self): + """Challenges supported by this plugin.""" return set(challenges.Challenge.TYPES[name] for name in self.conf("supported-challenges").split(",")) From ed70c948aaad5d56835eecd7e7a77fd4ba87ca88 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 19:50:27 +0000 Subject: [PATCH 28/91] Renewer: restore IConfig.simple_http_port data type. --- letsencrypt/renewer.py | 1 + letsencrypt/tests/renewer_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 1c9cddc95..e00fe5532 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -70,6 +70,7 @@ def renew(cert, old_version): # was an int, not a str) config.rsa_key_size = int(config.rsa_key_size) config.dvsni_port = int(config.dvsni_port) + config.namespace.simple_http_port = int(config.namespace.simple_http_port) zope.component.provideUtility(config) try: authenticator = plugins[renewalparams["authenticator"]] diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 6941e084c..518332a94 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -625,6 +625,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.configfile["renewalparams"]["server"] = "acme.example.com" self.test_rc.configfile["renewalparams"]["authenticator"] = "fake" self.test_rc.configfile["renewalparams"]["dvsni_port"] = "4430" + self.test_rc.configfile["renewalparams"]["simple_http_port"] = "1234" self.test_rc.configfile["renewalparams"]["account"] = "abcde" mock_auth = mock.MagicMock() mock_pd.PluginsRegistry.find_all.return_value = {"apache": mock_auth} From 94c6e307c960c1abc5d7f25a72b0d4fd1821d359 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 20:13:00 +0000 Subject: [PATCH 29/91] Fix plugins.common.Dvsni._setup_challenge_cert. --- letsencrypt/plugins/common.py | 7 ++++++- letsencrypt/plugins/common_test.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 95ad56a0a..88394f565 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -5,6 +5,7 @@ import re import shutil import tempfile +import OpenSSL import zope.interface from acme.jose import util as jose_util @@ -181,7 +182,11 @@ class Dvsni(object): self.configurator.reverter.register_file_creation(True, key_path) self.configurator.reverter.register_file_creation(True, cert_path) - response, cert_pem, key_pem = achall.gen_cert_and_response(s) + response, cert, key = achall.gen_cert_and_response(s) + cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert) + key_pem = OpenSSL.crypto.dump_privatekey( + OpenSSL.crypto.FILETYPE_PEM, key) # Write out challenge cert and key with open(cert_path, "wb") as cert_chall_fd: diff --git a/letsencrypt/plugins/common_test.py b/letsencrypt/plugins/common_test.py index fa761839c..f1eb19094 100644 --- a/letsencrypt/plugins/common_test.py +++ b/letsencrypt/plugins/common_test.py @@ -2,6 +2,7 @@ import unittest import mock +import OpenSSL from acme import challenges from acme import jose @@ -144,7 +145,9 @@ class DvsniTest(unittest.TestCase): response = challenges.DVSNIResponse(validation=mock.Mock()) achall = mock.MagicMock() - achall.gen_cert_and_response.return_value = (response, "cert", "key") + key = test_util.load_pyopenssl_private_key("rsa512_key.pem") + achall.gen_cert_and_response.return_value = ( + response, test_util.load_cert("cert.pem"), key) with mock.patch("letsencrypt.plugins.common.open", mock_open, create=True): @@ -156,10 +159,12 @@ class DvsniTest(unittest.TestCase): # pylint: disable=no-member mock_open.assert_called_once_with(self.sni.get_cert_path(achall), "wb") - mock_open.return_value.write.assert_called_once_with("cert") + mock_open.return_value.write.assert_called_once_with( + test_util.load_vector("cert.pem")) mock_safe_open.assert_called_once_with( self.sni.get_key_path(achall), "wb", chmod=0o400) - mock_safe_open.return_value.write.assert_called_once_with("key") + mock_safe_open.return_value.write.assert_called_once_with( + OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) if __name__ == "__main__": From 1774ab64c4590014f425ebfaa913ede3d586ad06 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 4 Oct 2015 20:24:53 +0000 Subject: [PATCH 30/91] Add SupportedChallengesValidatorTest. --- letsencrypt/plugins/standalone.py | 8 ++++++-- letsencrypt/plugins/standalone_test.py | 27 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 17e99073f..05a9f1655 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -115,7 +115,11 @@ SUPPORTED_CHALLENGES = set([challenges.DVSNI, challenges.SimpleHTTP]) def supported_challenges_validator(data): - """Supported challenges validator.""" + """Supported challenges validator for the `argparse`. + + It should be passed as `type` argument to `add_argument`. + + """ challs = data.split(",") unrecognized = [name for name in challs if name not in challenges.Challenge.TYPES] @@ -127,7 +131,7 @@ def supported_challenges_validator(data): if not set(challs).issubset(choices): raise argparse.ArgumentTypeError( "Plugin does not support the following (valid) " - "challenges: {0}".format(", ".join(challs - choices))) + "challenges: {0}".format(", ".join(set(challs) - choices))) return data diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 4a077a36f..1cd310107 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -1,4 +1,5 @@ """Tests for letsencrypt.plugins.standalone.""" +import argparse import socket import unittest @@ -63,6 +64,28 @@ class ServerManagerTest(unittest.TestCase): self.assertEqual(self.mgr.running(), {}) +class SupportedChallengesValidatorTest(unittest.TestCase): + """Tests for plugins.standalone.supported_challenges_validator.""" + + def _call(self, data): + from letsencrypt.plugins.standalone import ( + supported_challenges_validator) + return supported_challenges_validator(data) + + def test_correct(self): + self.assertEqual("dvsni", self._call("dvsni")) + self.assertEqual("simpleHttp", self._call("simpleHttp")) + self.assertEqual("dvsni,simpleHttp", self._call("dvsni,simpleHttp")) + self.assertEqual("simpleHttp,dvsni", self._call("simpleHttp,dvsni")) + + def test_unrecognized(self): + assert "foo" not in challenges.Challenge.TYPES + self.assertRaises(argparse.ArgumentTypeError, self._call, "foo") + + def test_not_subset(self): + self.assertRaises(argparse.ArgumentTypeError, self._call, "dns") + + class AuthenticatorTest(unittest.TestCase): """Tests for letsencrypt.plugins.standalone.Authenticator.""" @@ -72,6 +95,10 @@ class AuthenticatorTest(unittest.TestCase): standalone_supported_challenges="dvsni,simpleHttp") self.auth = Authenticator(self.config, name="standalone") + def test_supported_challenges(self): + self.assertEqual(self.auth.supported_challenges, + set([challenges.DVSNI, challenges.SimpleHTTP])) + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) From f0c11152d2eb39e67e4f8a5f43fffa9140e72bd0 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:00:47 +0000 Subject: [PATCH 31/91] ACMEServerMixin.__init__ --- acme/acme/standalone.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index ce10407e6..d4d6f0ea4 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -53,16 +53,13 @@ class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): class ACMEServerMixin: # pylint: disable=old-style-class,no-init - """ACME server common settings mixin. - - .. warning:: - Subclasses have to init ``_stopped = False`` (it's not done here, - because of old-style classes madness). - - """ + """ACME server common settings mixin.""" server_version = "ACME standalone client" allow_reuse_address = True + def __init__(self): + self._stopped = False + def serve_forever2(self): """Serve forever, until other thread calls `shutdown2`.""" while not self._stopped: @@ -95,7 +92,7 @@ class ACMETLSServer(HTTPSServer, ACMEServerMixin): """ def __init__(self, *args, **kwargs): - self._stopped = False + ACMEServerMixin.__init__(self) HTTPSServer.__init__(self, *args, **kwargs) @@ -103,7 +100,7 @@ class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): """ACME Server (non-TLS).""" def __init__(self, *args, **kwargs): - self._stopped = False + ACMEServerMixin.__init__(self) BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) From 7102f9ef4b1bc4eaaa53e4e0eb03e98afe21ee36 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:20:47 +0000 Subject: [PATCH 32/91] Don't expose threads from ServerManager. --- letsencrypt/plugins/standalone.py | 36 ++++++++++++++------------ letsencrypt/plugins/standalone_test.py | 23 ++++++++-------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 05a9f1655..551707263 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -39,8 +39,10 @@ class ServerManager(object): will serve the same URLs! """ + _Instance = collections.namedtuple("_Instance", "server thread") + def __init__(self, certs, simple_http_resources): - self._servers = {} + self._instances = {} self.certs = certs self.simple_http_resources = simple_http_resources @@ -53,13 +55,12 @@ class ServerManager(object): :param int port: Port to run the server on. :param bool tls: TLS or non-TLS? - :returns: Server instance (`ACMEServerMixin`) and the - corresponding (already started) thread (`threading.Thread`). - :rtype: tuple + :returns: Server instance. + :rtype: ACMEServerMixin """ - if port in self._servers: - return self._servers[port] + if port in self._instances: + return self._instances[port].server logger.debug("Starting new server at %s (tls=%s)", port, tls) handler = acme_standalone.ACMERequestHandler.partial_init( @@ -84,8 +85,8 @@ class ServerManager(object): logger.debug("Starting server at %s:%d", host, real_port) thread.start() - self._servers[real_port] = (server, thread) - return self._servers[real_port] + self._instances[real_port] = self._Instance(server, thread) + return server def stop(self, port): """Stop ACME server running on the specified ``port``. @@ -93,10 +94,10 @@ class ServerManager(object): :param int port: """ - server, thread = self._servers[port] - server.shutdown2() - thread.join() - del self._servers[port] + instance = self._instances[port] + instance.server.shutdown2() + instance.thread.join() + del self._instances[port] def running(self): """Return all running instances. @@ -104,11 +105,12 @@ class ServerManager(object): Once the server is stopped using `stop`, it will not be returned. - :returns: Mapping from port to ``(server, thread)``. + :returns: Mapping from ``port`` to ``server``. :rtype: tuple """ - return self._servers.copy() + return dict((port, instance.server) for port, instance + in six.iteritems(self._instances)) SUPPORTED_CHALLENGES = set([challenges.DVSNI, challenges.SimpleHTTP]) @@ -233,7 +235,7 @@ class Authenticator(common.Plugin): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server, _ = self.servers.run(self.config.simple_http_port, tls=tls) + server = self.servers.run(self.config.simple_http_port, tls=tls) response, validation = achall.gen_response_and_validation(tls=tls) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -242,7 +244,7 @@ class Authenticator(common.Plugin): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server, _ = self.servers.run(self.config.dvsni_port, tls=True) + server = self.servers.run(self.config.dvsni_port, tls=True) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) @@ -257,6 +259,6 @@ class Authenticator(common.Plugin): for achall in achalls: if achall in server_achalls: server_achalls.remove(achall) - for port, (server, _) in six.iteritems(self.servers.running()): + for port, server in six.iteritems(self.servers.running()): if not self.served[server]: self.servers.stop(port) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 1cd310107..a45935d2b 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -33,9 +33,9 @@ class ServerManagerTest(unittest.TestCase): self.mgr.simple_http_resources is self.simple_http_resources) def _test_run_stop(self, tls): - server, _ = self.mgr.run(port=0, tls=tls) - port = server.socket.getsockname()[1] - self.assertEqual(self.mgr.running(), {port: (server, mock.ANY)}) + server = self.mgr.run(port=0, tls=tls) + port = server.socket.getsockname()[1] # pylint: disable=no-member + self.assertEqual(self.mgr.running(), {port: server}) self.mgr.stop(port=port) self.assertEqual(self.mgr.running(), {}) @@ -46,12 +46,11 @@ class ServerManagerTest(unittest.TestCase): self._test_run_stop(tls=False) def test_run_idempotent(self): - server, thread = self.mgr.run(port=0, tls=False) - port = server.socket.getsockname()[1] - server2, thread2 = self.mgr.run(port=port, tls=False) - self.assertEqual(self.mgr.running(), {port: (server, thread)}) + server = self.mgr.run(port=0, tls=False) + port = server.socket.getsockname()[1] # pylint: disable=no-member + server2 = self.mgr.run(port=port, tls=False) + self.assertEqual(self.mgr.running(), {port: server}) self.assertTrue(server is server2) - self.assertTrue(thread is thread2) self.mgr.stop(port) self.assertEqual(self.mgr.running(), {}) @@ -166,7 +165,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def _run(port, tls): # pylint: disable=unused-argument - return "server{0}".format(port), "thread{0}".format(port) + return "server{0}".format(port) self.auth.servers.run.side_effect = _run responses = self.auth.perform2([simple_http, dvsni]) @@ -191,8 +190,8 @@ class AuthenticatorTest(unittest.TestCase): def test_cleanup(self): self.auth.servers = mock.Mock() self.auth.servers.running.return_value = { - 1: ("server1", "thread1"), - 2: ("server2", "thread2"), + 1: "server1", + 2: "server2", } self.auth.served["server1"].add("chall1") self.auth.served["server2"].update(["chall2", "chall3"]) @@ -203,7 +202,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers.stop.assert_called_once_with(1) self.auth.servers.running.return_value = { - 2: ("server2", "thread2"), + 2: "server2", } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { From 5afb0ebd1c16442a2f94469ee3db4f27fc3a9852 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 7 Oct 2015 19:42:32 +0000 Subject: [PATCH 33/91] Remove SimpleHTTP TLS from standalone 2.0 --- letsencrypt/configuration.py | 7 ++----- letsencrypt/plugins/standalone.py | 15 +++++---------- letsencrypt/plugins/standalone_test.py | 15 ++------------- letsencrypt/tests/configuration_test.py | 5 ----- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 5f965cb6d..a2eab5ecb 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -37,8 +37,7 @@ class NamespaceConfig(object): self.namespace = namespace # XXX: breaks renewer in some bizarre way - #if self.no_simple_http_tls and ( - # self.simple_http_port == self.dvsni_port): + #if self.simple_http_port == self.dvsni_port: # raise errors.Error( # "Trying to run SimpleHTTP non-TLS and DVSNI " # "on the same port ({0})".format(self.dvsni_port)) @@ -82,10 +81,8 @@ class NamespaceConfig(object): def simple_http_port(self): # pylint: disable=missing-docstring if self.namespace.simple_http_port is not None: return self.namespace.simple_http_port - if self.no_simple_http_tls: - return challenges.SimpleHTTPResponse.PORT else: - return challenges.SimpleHTTPResponse.TLS_PORT + return challenges.SimpleHTTPResponse.PORT class RenewerConfiguration(object): diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 551707263..97964e8cc 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -5,7 +5,6 @@ import functools import logging import random import socket -import sys import threading import OpenSSL @@ -198,12 +197,7 @@ class Authenticator(common.Plugin): def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring - supported_challenges = self.supported_challenges - if not self.config.no_simple_http_tls and not ( - acme_standalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT): - logger.debug("SimpleHTTPS not supported: %s", sys.version) - supported_challenges.discard(challenges.SimpleHTTP) - chall_pref = list(supported_challenges) + chall_pref = list(self.supported_challenges) random.shuffle(chall_pref) # 50% for each challenge return chall_pref @@ -231,12 +225,13 @@ class Authenticator(common.Plugin): def perform2(self, achalls): """Perform achallenges without IDisplay interaction.""" responses = [] - tls = not self.config.no_simple_http_tls for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): - server = self.servers.run(self.config.simple_http_port, tls=tls) - response, validation = achall.gen_response_and_validation(tls=tls) + server = self.servers.run( + self.config.simple_http_port, tls=False) + response, validation = achall.gen_response_and_validation( + tls=False) self.simple_http_resources.add( acme_standalone.SimpleHTTPRequestHandler.SimpleHTTPResource( chall=achall.chall, response=response, diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index a45935d2b..e99bd473a 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -107,20 +107,9 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.MisconfigurationError, self.auth.prepare) mock_util.already_listening.assert_called_once_with(1234) - @mock.patch("letsencrypt.plugins.standalone.acme_standalone") - def test_get_chall_pref_tls_supported(self, mock_astandalone): - mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = True - for no_simple_http_tls in True, False: - self.config.no_simple_http_tls = no_simple_http_tls - self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.DVSNI, challenges.SimpleHTTP])) - - @mock.patch("letsencrypt.plugins.standalone.acme_standalone") - def test_get_chall_pref_simple_tls_not_supported(self, mock_astandalone): - mock_astandalone.ACMETLSServer.SIMPLE_HTTP_SUPPORT = False - self.config.no_simple_http_tls = False + def test_get_chall_pref(self): self.assertEqual(set(self.auth.get_chall_pref(domain=None)), - set([challenges.DVSNI])) + set([challenges.DVSNI, challenges.SimpleHTTP])) @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index acf9273d0..c1eba8570 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -49,14 +49,9 @@ class NamespaceConfigTest(unittest.TestCase): def test_simple_http_port(self): self.assertEqual(4321, self.config.simple_http_port) - self.namespace.simple_http_port = None - self.namespace.no_simple_http_tls = True self.assertEqual(80, self.config.simple_http_port) - self.namespace.no_simple_http_tls = False - self.assertEqual(443, self.config.simple_http_port) - class RenewerConfigurationTest(unittest.TestCase): """Test for letsencrypt.configuration.RenewerConfiguration.""" From 304414a214730593e11520affcd99684f1bddd74 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 8 Oct 2015 21:10:12 +0000 Subject: [PATCH 34/91] Remove SimpleHTTP TLS from acme. --- acme/acme/crypto_util.py | 31 -------- acme/acme/standalone.py | 69 +++++------------ acme/acme/standalone_test.py | 122 +++++++++++++++++------------- letsencrypt/plugins/standalone.py | 19 ++--- 4 files changed, 96 insertions(+), 145 deletions(-) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 5829a511f..5f24e9d9e 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -70,20 +70,10 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods class FakeConnection(object): """Fake OpenSSL.SSL.Connection.""" - MAKEFILE_SUPPORT = hasattr(socket, "_fileobject") - """Is `makefile` supported on your platform? - - .. warning:: `makefile`, as currently implemented, is supported - on select platforms only, as it uses CPython's internal API. - You've been warned! - - """ - # pylint: disable=missing-docstring def __init__(self, connection): self._wrapped = connection - self._makefile_refs = 0 def __getattr__(self, name): return getattr(self._wrapped, name) @@ -92,27 +82,6 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods # OpenSSL.SSL.Connection.shutdown doesn't accept any args return self._wrapped.shutdown() - # stuff below ripped off from - # https://hg.python.org/cpython/file/2.7/Lib/ssl.py - - def makefile(self, mode='r', bufsize=-1): - assert self.MAKEFILE_SUPPORT, ( - "You need compatible version for makefile support") - self._makefile_refs += 1 - # SocketServer.StreamRequesthandler.finish will try to - # close the wfile/rfile. close=True causes curl: (56) - # GnuTLS recv error (-110): The TLS connection was - # non-properly terminated. - # TODO: doesn't work in Python3 - # pylint: disable=protected-access - return socket._fileobject(self._wrapped, mode, bufsize, close=False) - - def close(self): - if self._makefile_refs < 1: - self._wrapped.close() - else: - self._makefile_refs -= 1 - def accept(self): # pylint: disable=missing-docstring sock, addr = self.sock.accept() diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index d4d6f0ea4..089c2ff18 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -44,15 +44,7 @@ class TLSServer(socketserver.TCPServer): return socketserver.TCPServer.server_bind(self) -class HTTPSServer(TLSServer, BaseHTTPServer.HTTPServer): - """HTTPS Server.""" - - def server_bind(self): - self._wrap_sock() - BaseHTTPServer.HTTPServer.server_bind(self) - - -class ACMEServerMixin: # pylint: disable=old-style-class,no-init +class ACMEServerMixin: # pylint: disable=old-style-class """ACME server common settings mixin.""" server_version = "ACME standalone client" allow_reuse_address = True @@ -81,27 +73,23 @@ class ACMEServerMixin: # pylint: disable=old-style-class,no-init self.server_close() -class ACMETLSServer(HTTPSServer, ACMEServerMixin): - """ACME TLS Server.""" +class DVSNIServer(TLSServer, ACMEServerMixin): + """DVSNI Server.""" - SIMPLE_HTTP_SUPPORT = crypto_util.SSLSocket.FakeConnection.MAKEFILE_SUPPORT - """Is SimpleHTTP supported on your platform. - - Please see a warning for `acme.crypto_util.SSLSocket.FakeConnection`. - - """ - - def __init__(self, *args, **kwargs): + def __init__(self, server_address, certs): ACMEServerMixin.__init__(self) - HTTPSServer.__init__(self, *args, **kwargs) + TLSServer.__init__( + self, server_address, socketserver.BaseRequestHandler, certs=certs) -class ACMEServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): - """ACME Server (non-TLS).""" +class SimpleHTTPServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): + """SimpleHTTP Server.""" - def __init__(self, *args, **kwargs): + def __init__(self, server_address, resources): ACMEServerMixin.__init__(self) - BaseHTTPServer.HTTPServer.__init__(self, *args, **kwargs) + BaseHTTPServer.HTTPServer.__init__( + self, server_address, SimpleHTTPRequestHandler.partial_init( + simple_http_resources=resources)) class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): @@ -133,14 +121,14 @@ class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write(self.server.server_version) + self.wfile.write(self.server.server_version.encode()) def handle_404(self): """Handler 404 Not Found errors.""" self.send_response(http_client.NOT_FOUND, message="Not Found") self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write("404") + self.wfile.write(b"404") def handle_simple_http_resource(self): """Handle SimpleHTTP provisioned resources.""" @@ -171,24 +159,8 @@ class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): cls, simple_http_resources=simple_http_resources) -class ACMERequestHandler(SimpleHTTPRequestHandler): - """ACME request handler.""" - - def handle_one_request(self): - """Handle single request. - - Makes sure that DVSNI probers are ignored. - - """ - try: - return SimpleHTTPRequestHandler.handle_one_request(self) - except OpenSSL.SSL.ZeroReturnError: - logger.debug("Client prematurely closed connection (prober?). " - "Ignoring request.") - - -def simple_server(cli_args, forever=True): - """Run simple standalone client server.""" +def simple_dvsni_server(cli_args, forever=True): + """Run simple standalone DVSNI server.""" logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() @@ -198,7 +170,6 @@ def simple_server(cli_args, forever=True): args = parser.parse_args(cli_args[1:]) certs = {} - resources = {} _, hosts, _ = next(os.walk('.')) for host in hosts: @@ -206,15 +177,13 @@ def simple_server(cli_args, forever=True): cert_contents = cert_file.read() with open(os.path.join(host, "key.pem")) as key_file: key_contents = key_file.read() - certs[host] = ( + certs[host.encode()] = ( OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_contents), OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, cert_contents)) - handler = ACMERequestHandler.partial_init( - simple_http_resources=resources) - server = ACMETLSServer(('', int(args.port)), handler, certs=certs) + server = DVSNIServer(('', int(args.port)), certs=certs) six.print_("Serving at https://localhost:{0}...".format( server.socket.getsockname()[1])) if forever: # pragma: no cover @@ -224,4 +193,4 @@ def simple_server(cli_args, forever=True): if __name__ == "__main__": - sys.exit(simple_server(sys.argv)) # pragma: no cover + sys.exit(simple_dvsni_server(sys.argv)) # pragma: no cover diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index a6f7502b6..9eb192c74 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -13,6 +13,7 @@ import requests from acme import challenges from acme import crypto_util +from acme import errors from acme import jose from acme import test_util @@ -30,25 +31,27 @@ class TLSServerTest(unittest.TestCase): class ACMEServerMixinTest(unittest.TestCase): """Tests for acme.standalone.ACMEServerMixin.""" + def setUp(self): + from acme.standalone import ACMEServerMixin + + class _MockServer(socketserver.TCPServer, ACMEServerMixin): + def __init__(self, *args, **kwargs): + socketserver.TCPServer.__init__(self, *args, **kwargs) + ACMEServerMixin.__init__(self) + self.server = _MockServer(("", 0), socketserver.BaseRequestHandler) + + def test_serve_shutdown(self): + thread = threading.Thread(target=self.server.serve_forever2) + thread.start() + self.server.shutdown2() + def test_shutdown2_not_running(self): - from acme.standalone import ACMEServer - server = ACMEServer(("", 0), socketserver.BaseRequestHandler) - server.shutdown2() - server.shutdown2() + self.server.shutdown2() + self.server.shutdown2() -class ACMEServerTest(unittest.TestCase): - """Test for acme.standalone.ACMEServer.""" - - def test_init(self): - from acme.standalone import ACMEServer - server = ACMEServer(("", 0), socketserver.BaseRequestHandler) - # pylint: disable=protected-access - self.assertFalse(server._stopped) - - -class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): - """End-to-end test for ACME TLS server with SimpleHTTP.""" +class DVSNIServerTest(unittest.TestCase): + """Test for acme.standalone.DVSNIServer.""" def setUp(self): self.certs = { @@ -56,36 +59,19 @@ class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): # pylint: disable=protected-access test_util.load_cert('cert.pem')._wrapped), } - self.account_key = jose.JWK.load( - test_util.load_vector('rsa1024_key.pem')) - - from acme.standalone import ACMETLSServer - from acme.standalone import ACMERequestHandler - self.resources = set() - handler = ACMERequestHandler.partial_init( - simple_http_resources=self.resources) - self.server = ACMETLSServer(('', 0), handler, certs=self.certs) - self.server_thread = threading.Thread( - # pylint: disable=no-member - target=self.server.serve_forever2) - self.server_thread.start() - - self.port = self.server.socket.getsockname()[1] + from acme.standalone import DVSNIServer + self.server = DVSNIServer(("", 0), certs=self.certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.handle_request) + self.thread.start() def tearDown(self): self.server.shutdown2() - self.server_thread.join() + self.thread.join() - def test_index(self): - response = requests.get( - 'https://localhost:{0}'.format(self.port), verify=False) - self.assertEqual(response.text, 'ACME standalone client') - self.assertTrue(response.ok) - - def test_404(self): - response = requests.get( - 'https://localhost:{0}/foo'.format(self.port), verify=False) - self.assertEqual(response.status_code, http_client.NOT_FOUND) + def test_init(self): + # pylint: disable=protected-access + self.assertFalse(self.server._stopped) def test_dvsni(self): cert = crypto_util.probe_sni( @@ -93,9 +79,41 @@ class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): self.assertEqual(jose.ComparableX509(cert), jose.ComparableX509(self.certs[b'localhost'][1])) + +class SimpleHTTPServerTest(unittest.TestCase): + """Tests for acme.standalone.SimpleHTTPServer.""" + + def setUp(self): + self.account_key = jose.JWK.load( + test_util.load_vector('rsa1024_key.pem')) + self.resources = set() + + from acme.standalone import SimpleHTTPServer + self.server = SimpleHTTPServer(('', 0), resources=self.resources) + + # pylint: disable=no-member + self.port = self.server.socket.getsockname()[1] + self.thread = threading.Thread(target=self.server.handle_request) + self.thread.start() + + def tearDown(self): + self.server.shutdown2() + self.thread.join() + + def test_index(self): + response = requests.get( + 'http://localhost:{0}'.format(self.port), verify=False) + self.assertEqual(response.text, 'ACME standalone client') + self.assertTrue(response.ok) + + def test_404(self): + response = requests.get( + 'http://localhost:{0}/foo'.format(self.port), verify=False) + self.assertEqual(response.status_code, http_client.NOT_FOUND) + def _test_simple_http(self, add): chall = challenges.SimpleHTTP(token=(b'x' * 16)) - response = challenges.SimpleHTTPResponse(tls=True) + response = challenges.SimpleHTTPResponse(tls=False) from acme.standalone import SimpleHTTPRequestHandler resource = SimpleHTTPRequestHandler.SimpleHTTPResource( @@ -114,8 +132,8 @@ class ACMESimpleHTTPTLSServerTestEndToEnd(unittest.TestCase): self.assertFalse(self._test_simple_http(add=False)) -class TestSimpleServer(unittest.TestCase): - """Tests for acme.standalone.simple_server.""" +class TestSimpleDVSNIServer(unittest.TestCase): + """Tests for acme.standalone.simple_dvsni_server.""" def setUp(self): # mirror ../examples/standalone @@ -126,9 +144,10 @@ class TestSimpleServer(unittest.TestCase): shutil.copy(test_util.vector_path('rsa512_key.pem'), os.path.join(localhost_dir, 'key.pem')) - from acme.standalone import simple_server - self.thread = threading.Thread(target=simple_server, kwargs={ - 'cli_args': ('xxx', '--port', '1234'), + from acme.standalone import simple_dvsni_server + self.port = 1234 + self.thread = threading.Thread(target=simple_dvsni_server, kwargs={ + 'cli_args': ('xxx', '--port', str(self.port)), 'forever': False, }) self.old_cwd = os.getcwd() @@ -145,12 +164,13 @@ class TestSimpleServer(unittest.TestCase): while max_attempts: max_attempts -= 1 try: - response = requests.get('https://localhost:1234', verify=False) - except requests.ConnectionError: + cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', self.port) + except errors.Error: self.assertTrue(max_attempts > 0, "Timeout!") time.sleep(1) # wait until thread starts else: - self.assertEqual(response.text, 'ACME standalone client') + self.assertEqual(jose.ComparableX509(cert), + test_util.load_cert('cert.pem')) break diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index 97964e8cc..cb95ec408 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -1,7 +1,6 @@ """Standalone Authenticator.""" import argparse import collections -import functools import logging import random import socket @@ -61,25 +60,19 @@ class ServerManager(object): if port in self._instances: return self._instances[port].server - logger.debug("Starting new server at %s (tls=%s)", port, tls) - handler = acme_standalone.ACMERequestHandler.partial_init( - self.simple_http_resources) - - if tls: - cls = functools.partial( - acme_standalone.ACMETLSServer, certs=self.certs) - else: - cls = acme_standalone.ACMEServer - + address = ("", port) try: - server = cls(("", port), handler) + if tls: + server = acme_standalone.DVSNIServer(address, self.certs) + else: + server = acme_standalone.SimpleHTTPServer( + address, self.simple_http_resources) except socket.error as error: raise errors.StandaloneBindError(error, port) # if port == 0, then random free port on OS is taken # pylint: disable=no-member host, real_port = server.socket.getsockname() - thread = threading.Thread(target=server.serve_forever2) logger.debug("Starting server at %s:%d", host, real_port) thread.start() From 5edd809161c659d82592d53bc6e48af4f49f2e1a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 11 Oct 2015 10:51:24 +0000 Subject: [PATCH 35/91] ApacheConfigurator.is_enabled using filecmp (fixes #838). --- letsencrypt-apache/letsencrypt_apache/configurator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f3d2b5f9a..cb81366b3 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import filecmp import itertools import logging import os @@ -945,9 +946,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ enabled_dir = os.path.join(self.parser.root, "sites-enabled") for entry in os.listdir(enabled_dir): - if os.path.realpath(os.path.join(enabled_dir, entry)) == avail_fp: - return True - + try: + if filecmp.cmp(avail_fp, os.path.join(enabled_dir, entry)): + return True + except OSError: + pass return False def enable_site(self, vhost): From ce4120186128136376dab24715af4ed0865d56e1 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 11 Oct 2015 10:52:08 +0000 Subject: [PATCH 36/91] Require tests pass in dev release. --- tools/dev-release.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index d93a6d21f..cebe5001c 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -88,8 +88,7 @@ mkdir ../kgs kgs="../kgs/$version" pip freeze | tee $kgs pip install nose -# TODO: letsencrypt_apache fails due to symlink, c.f. #838 -nosetests letsencrypt $SUBPKGS || true +nosetests letsencrypt $SUBPKGS echo "New root: $root" echo "KGS is at $root/kgs" From 6ae3f4c973a5326edc696c224dff286c438c84ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Douglas=20Jose=CC=81?= Date: Mon, 12 Oct 2015 21:25:52 -0400 Subject: [PATCH 37/91] Added pip and virtualenv installation steps to Mac's bootstrap script. --- bootstrap/mac.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh index 6779188a7..06defa853 100755 --- a/bootstrap/mac.sh +++ b/bootstrap/mac.sh @@ -6,3 +6,13 @@ fi brew install augeas brew install dialog + +if ! hash pip 2>/dev/null; then + echo "pip Not Installed\nInstalling python from Homebrew..." + brew install python +fi + +if ! hash virtualenv 2>/dev/null; then + echo "virtualenv Not Installed\nInstalling with pip" + pip install virtualenv +fi From c4042e6ce8c74e85854343b20350fb497d0232d6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 13 Oct 2015 07:09:14 +0000 Subject: [PATCH 38/91] Busy wait loop for testing serve_forever2 This fixes race conditions, such as those in https://travis-ci.org/letsencrypt/letsencrypt/jobs/84990239: + nosetests -c /dev/null --with-cover --cover-tests --cover-package acme --cover-min-percentage=100 acme .......................................................................................................................................................................................................................................................................................................................................................Exception in thread Thread-5: Traceback (most recent call last): File "/opt/python/2.7.9/lib/python2.7/threading.py", line 810, in __bootstrap_inner self.run() File "/opt/python/2.7.9/lib/python2.7/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs) File "/opt/python/2.7.9/lib/python2.7/SocketServer.py", line 271, in handle_request timeout = self.socket.gettimeout() File "/opt/python/2.7.9/lib/python2.7/socket.py", line 224, in meth return getattr(self._sock,name)(*args) File "/opt/python/2.7.9/lib/python2.7/socket.py", line 170, in _dummy raise error(EBADF, 'Bad file descriptor') error: [Errno 9] Bad file descriptor .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET /foo HTTP/1.1" 404 - .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET / HTTP/1.1" 200 - .127.0.0.1 - - [12/Oct/2015 20:08:23] "GET /.well-known/acme-challenge/eHh4eHh4eHh4eHh4eHh4eA HTTP/1.1" 200 - ..... Name Stmts Miss Cover Missing ------------------------------------------------------------ acme.py 0 0 100% acme/challenges.py 215 0 100% acme/challenges_test.py 366 0 100% acme/client.py 215 0 100% acme/client_test.py 308 0 100% acme/crypto_util.py 92 0 100% acme/crypto_util_test.py 53 0 100% acme/errors.py 19 0 100% acme/errors_test.py 18 0 100% acme/fields.py 32 0 100% acme/fields_test.py 41 0 100% acme/jose.py 8 0 100% acme/jose/b64.py 15 0 100% acme/jose/b64_test.py 38 0 100% acme/jose/errors.py 12 0 100% acme/jose/errors_test.py 8 0 100% acme/jose/interfaces.py 39 0 100% acme/jose/interfaces_test.py 73 0 100% acme/jose/json_util.py 170 0 100% acme/jose/json_util_test.py 214 0 100% acme/jose/jwa.py 105 0 100% acme/jose/jwa_test.py 58 0 100% acme/jose/jwk.py 114 0 100% acme/jose/jwk_test.py 96 0 100% acme/jose/jws.py 205 0 100% acme/jose/jws_test.py 145 0 100% acme/jose/util.py 114 0 100% acme/jose/util_test.py 126 0 100% acme/jws.py 17 0 100% acme/jws_test.py 27 0 100% acme/messages.py 184 0 100% acme/messages_test.py 198 0 100% acme/other.py 21 0 100% acme/other_test.py 48 0 100% acme/standalone.py 102 1 99% 58 acme/standalone_test.py 109 0 100% acme/test_util.py 28 0 100% acme/util.py 3 0 100% acme/util_test.py 7 0 100% ------------------------------------------------------------ TOTAL 3643 1 99% nose.plugins.cover: ERROR: TOTAL Coverage did not reach minimum required: 100% --- acme/acme/standalone_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 9eb192c74..7eda2fce3 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -1,6 +1,7 @@ """Tests for acme.standalone.""" import os import shutil +import socket import threading import tempfile import time @@ -40,9 +41,27 @@ class ACMEServerMixinTest(unittest.TestCase): ACMEServerMixin.__init__(self) self.server = _MockServer(("", 0), socketserver.BaseRequestHandler) + def _busy_wait(self): # pragma: no cover + # This function is used to avoid race coditions in tests, but + # not all of the functionality is always used, hence "no + # cover" + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # pylint: disable=no-member + sock.connect(self.server.socket.getsockname()) + except socket.error: + pass + else: + break + finally: + sock.close() + time.sleep(1) + def test_serve_shutdown(self): thread = threading.Thread(target=self.server.serve_forever2) thread.start() + self._busy_wait() self.server.shutdown2() def test_shutdown2_not_running(self): From c7732114cbedc915d14526f5c96a1a17e2becadb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 13 Oct 2015 14:50:23 -0700 Subject: [PATCH 39/91] Only test CLI for nginx plugin if it is present - Fixes 919 --- letsencrypt/tests/cli_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index d0fae370d..f690e77f9 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -75,7 +75,10 @@ class CLITest(unittest.TestCase): output.truncate(0) self.assertRaises(SystemExit, self._call_stdout, ['-h', 'nginx']) out = output.getvalue() - self.assertTrue("--nginx-ctl" in out) + from letsencrypt.plugins import disco + if "nginx" in disco.PluginsRegistry.find_all(): + # may be false while building distributions without plugins + self.assertTrue("--nginx-ctl" in out) self.assertTrue("--manual-test-mode" not in out) self.assertTrue("--checkpoints" not in out) output.truncate(0) From d5fd9986de701a2c0036a845c3443ec935a8e273 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 14 Oct 2015 09:26:59 -0700 Subject: [PATCH 40/91] Add rateLimited error type. --- acme/acme/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 02ae24c8f..9d4dcbf30 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -24,6 +24,7 @@ class Error(jose.JSONObjectWithFields, Exception): 'connection': 'The server could not connect to the client for DV', 'dnssec': 'The server could not validate a DNSSEC signed domain', 'malformed': 'The request message was malformed', + 'rateLimited': 'There were too many requests of a given type', 'serverInternal': 'The server experienced an internal error', 'tls': 'The server experienced a TLS error during DV', 'unauthorized': 'The client lacks sufficient authorization', From 9b77c9aecb27ed95d7392cc26614e86bfa479579 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 17:30:03 +0000 Subject: [PATCH 41/91] Uncomment simplehttp/dvsni port check --- letsencrypt/configuration.py | 10 +++++----- letsencrypt/tests/configuration_test.py | 9 ++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index a2eab5ecb..f72005233 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -7,6 +7,7 @@ import zope.interface from acme import challenges from letsencrypt import constants +from letsencrypt import errors from letsencrypt import interfaces @@ -36,11 +37,10 @@ class NamespaceConfig(object): def __init__(self, namespace): self.namespace = namespace - # XXX: breaks renewer in some bizarre way - #if self.simple_http_port == self.dvsni_port: - # raise errors.Error( - # "Trying to run SimpleHTTP non-TLS and DVSNI " - # "on the same port ({0})".format(self.dvsni_port)) + if self.simple_http_port == self.dvsni_port: + raise errors.Error( + "Trying to run SimpleHTTP and DVSNI " + "on the same port ({0})".format(self.dvsni_port)) def __getattr__(self, name): return getattr(self.namespace, name) diff --git a/letsencrypt/tests/configuration_test.py b/letsencrypt/tests/configuration_test.py index c1eba8570..44bccb577 100644 --- a/letsencrypt/tests/configuration_test.py +++ b/letsencrypt/tests/configuration_test.py @@ -4,6 +4,8 @@ import unittest import mock +from letsencrypt import errors + class NamespaceConfigTest(unittest.TestCase): """Tests for letsencrypt.configuration.NamespaceConfig.""" @@ -12,10 +14,15 @@ class NamespaceConfigTest(unittest.TestCase): self.namespace = mock.MagicMock( config_dir='/tmp/config', work_dir='/tmp/foo', foo='bar', server='https://acme-server.org:443/new', - dvsni_port='1234', simple_http_port=4321) + dvsni_port=1234, simple_http_port=4321) from letsencrypt.configuration import NamespaceConfig self.config = NamespaceConfig(self.namespace) + def test_init_same_ports(self): + self.namespace.dvsni_port = 4321 + from letsencrypt.configuration import NamespaceConfig + self.assertRaises(errors.Error, NamespaceConfig, self.namespace) + def test_proxy_getattr(self): self.assertEqual(self.config.foo, 'bar') self.assertEqual(self.config.work_dir, '/tmp/foo') From 3f942d742608cb97b897f0ea6167ac8fe3aa04dd Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 18:19:10 +0000 Subject: [PATCH 42/91] Basic virtualenv bootstrap script for end users. --- bootstrap/venv.sh | 35 +++++++++++++++++++++++++++++++++++ docs/using.rst | 12 +++--------- 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100755 bootstrap/venv.sh diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh new file mode 100755 index 000000000..619135d0d --- /dev/null +++ b/bootstrap/venv.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e +# +# Installs and updates letencrypt virtualenv +# +# USAGE: source ./dev/venv.sh + + +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -d $VENV_PATH ] +then + virtualenv --no-site-packages --python python2 $VENV_PATH +fi + +. $VENV_PATH/bin/activate +pip install -U setuptools +pip install -U pip + +# TODO: install apache and nginx plugins by default? +# --pre is not necessary for dev releases in more recent pip versions +pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx + +echo +echo "Congratulations, Let's Encrypt has been successfully installed/updated!" +echo +echo -n "Your prompt should now be prepended with ($VENV_NAME). Next " +echo -n "time, if the prompt is different, 'source' this script again " +echo -n "before running 'letsencrypt'." +echo +echo +echo "You can now run 'letsencrypt --help'." diff --git a/docs/using.rst b/docs/using.rst index 9611f37c0..b2d2db42a 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -122,15 +122,9 @@ bash``. Installation ============ -.. "pip install acme" doesn't search for "acme" in cwd, just like "pip - install -e acme" does; `-U setuptools pip` necessary for #722 - .. code-block:: shell - virtualenv --no-site-packages -p python2 venv - ./venv/bin/pip install -U setuptools - ./venv/bin/pip install -U pip - ./venv/bin/pip install -r requirements.txt acme/ . letsencrypt-apache/ letsencrypt-nginx/ + source ./bootstrap/venv.sh .. warning:: Please do **not** use ``python setup.py install``. Please do **not** attempt the installation commands as @@ -148,13 +142,13 @@ To get a new certificate run: .. code-block:: shell - sudo ./venv/bin/letsencrypt auth + (letsencrypt)$ letsencrypt auth The ``letsencrypt`` commandline tool has a builtin help: .. code-block:: shell - ./venv/bin/letsencrypt --help + (letsencrypt)$ letsencrypt --help Configuration file From 8a8dfd4bc3697e322669abc663020a1d3f5054c6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 18:48:43 +0000 Subject: [PATCH 43/91] More verbose tox python env tests --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index b10558077..69fce6615 100644 --- a/tox.ini +++ b/tox.ini @@ -12,11 +12,11 @@ envlist = py27,cover,lint commands = pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt # -q does not suppress errors - python setup.py test -q - python setup.py test -q -s acme - python setup.py test -q -s letsencrypt_apache - python setup.py test -q -s letsencrypt_nginx - python setup.py test -q -s letshelp_letsencrypt + python setup.py test + python setup.py test -s acme + python setup.py test -s letsencrypt_apache + python setup.py test -s letsencrypt_nginx + python setup.py test -s letshelp_letsencrypt setenv = PYTHONPATH = {toxinidir} From 371daa42cac800c0031b251c02e8cc3e511b4b14 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:16:30 +0000 Subject: [PATCH 44/91] Quickfix for boulder#985 --- tests/boulder-integration.sh | 9 +++++++-- tests/integration/_common.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 1de46eda1..273fa6ef6 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -27,9 +27,14 @@ common() { "$@" } +# TODO: boulder#985 +common_http() { + common --dvsni-port 0 --simple-http-port 5001 "$@" +} + common --domains le1.wtf --standalone-supported-challenges dvsni auth -common --domains le2.wtf --standalone-supported-challenges simpleHttp run -common -a manual -d le.wtf auth +common_http --domains le2.wtf --standalone-supported-challenges simpleHttp run +common_http -a manual -d le.wtf auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index fd60b9258..b4386daa8 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5001 \ + --simple-http-porta 5002 \ --manual-test-mode \ $store_flags \ --text \ From 99a31463b0b8bf5f63a9522d133284d9a0fbc313 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:23:33 +0000 Subject: [PATCH 45/91] Fix typo: porta -> port --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index b4386daa8..ab645f6d6 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-porta 5002 \ + --simple-http-port 5002 \ --manual-test-mode \ $store_flags \ --text \ From 128147af3b540979a51a4a4074b57f7cb4a065da Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 12:27:18 -0700 Subject: [PATCH 46/91] Removed misleading renewal messages --- letsencrypt/client.py | 26 -------------------------- letsencrypt/tests/client_test.py | 31 ------------------------------- 2 files changed, 57 deletions(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 123bab121..bb04f4f5a 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -5,7 +5,6 @@ import os from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa import OpenSSL -import zope.component from acme import client as acme_client from acme import jose @@ -19,7 +18,6 @@ from letsencrypt import continuity_auth from letsencrypt import crypto_util from letsencrypt import errors from letsencrypt import error_handler -from letsencrypt import interfaces from letsencrypt import le_util from letsencrypt import reverter from letsencrypt import storage @@ -258,32 +256,8 @@ class Client(object): OpenSSL.crypto.FILETYPE_PEM, certr.body), key.pem, crypto_util.dump_pyopenssl_chain(chain), params, config, cli_config) - self._report_renewal_status(lineage) return lineage - def _report_renewal_status(self, cert): - # pylint: disable=no-self-use - """Informs the user about automatic renewal and deployment. - - :param .RenewableCert cert: Newly issued certificate - - """ - if cert.autorenewal_is_enabled(): - if cert.autodeployment_is_enabled(): - msg = "Automatic renewal and deployment has " - else: - msg = "Automatic renewal but not automatic deployment has " - elif cert.autodeployment_is_enabled(): - msg = "Automatic deployment but not automatic renewal has " - else: - msg = "Automatic renewal and deployment has not " - - msg += ("been enabled for your certificate. These settings can be " - "configured in the directories under {0}.").format( - cert.cli_config.renewal_configs_dir) - reporter = zope.component.getUtility(interfaces.IReporter) - reporter.add_message(msg, reporter.LOW_PRIORITY) - def save_certificate(self, certr, chain_cert, cert_path, chain_path): # pylint: disable=no-self-use """Saves the certificate received from the ACME server. diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index fddb86607..9a5a2bbe1 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -114,37 +114,6 @@ class ClientTest(unittest.TestCase): mock.sentinel.key, domains, self.config.csr_dir) self._check_obtain_certificate() - @mock.patch("letsencrypt.client.zope.component.getUtility") - def test_report_renewal_status(self, mock_zope): - # pylint: disable=protected-access - cert = mock.MagicMock() - cert.cli_config.renewal_configs_dir = "/foo/bar/baz" - - cert.autorenewal_is_enabled.return_value = True - cert.autodeployment_is_enabled.return_value = True - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal and deployment has been" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autorenewal_is_enabled.return_value = False - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("deployment but not automatic renewal" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autodeployment_is_enabled.return_value = False - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal and deployment has not" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - - cert.autorenewal_is_enabled.return_value = True - self.client._report_renewal_status(cert) - msg = mock_zope().add_message.call_args[0][0] - self.assertTrue("renewal but not automatic deployment" in msg) - self.assertTrue(cert.cli_config.renewal_configs_dir in msg) - def test_save_certificate(self): certs = ["matching_cert.pem", "cert.pem", "cert-san.pem"] tmp_path = tempfile.mkdtemp() From 18ddcc72f67f07bb0cdb1bbe5d64e336769ec8ff Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 14 Oct 2015 19:40:07 +0000 Subject: [PATCH 47/91] More quickfix for boulder#985 --- tests/boulder-integration.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 273fa6ef6..1edae2765 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -28,18 +28,21 @@ common() { } # TODO: boulder#985 +common_dvsni() { + common --dvsni-port 5001 --simple-http-port 0 "$@" +} common_http() { common --dvsni-port 0 --simple-http-port 5001 "$@" } -common --domains le1.wtf --standalone-supported-challenges dvsni auth +common_dvsni --domains le1.wtf --standalone-supported-challenges dvsni auth common_http --domains le2.wtf --standalone-supported-challenges simpleHttp run common_http -a manual -d le.wtf auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf ./examples/generate-csr.sh le3.wtf -common auth --csr "$CSR_PATH" \ +common_dvsni auth --csr "$CSR_PATH" \ --cert-path "${root}/csr/cert.pem" \ --chain-path "${root}/csr/chain.pem" openssl x509 -in "${root}/csr/0000_cert.pem" -text From 7cdcea4dd557d59dba2860cf6854a628115015ba Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 12:43:06 -0700 Subject: [PATCH 48/91] Mock account not client --- letsencrypt/tests/account_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/tests/account_test.py b/letsencrypt/tests/account_test.py index cd98e1e20..4e2fdb122 100644 --- a/letsencrypt/tests/account_test.py +++ b/letsencrypt/tests/account_test.py @@ -70,12 +70,12 @@ class ReportNewAccountTest(unittest.TestCase): from letsencrypt.account import report_new_account report_new_account(self.acc, self.config) - @mock.patch("letsencrypt.client.zope.component.queryUtility") + @mock.patch("letsencrypt.account.zope.component.queryUtility") def test_no_reporter(self, mock_zope): mock_zope.return_value = None self._call() - @mock.patch("letsencrypt.client.zope.component.queryUtility") + @mock.patch("letsencrypt.account.zope.component.queryUtility") def test_it(self, mock_zope): self._call() call_list = mock_zope().add_message.call_args_list From 4083cd42addaee2ccf115b1620ad07965ce72a61 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 15:09:55 -0700 Subject: [PATCH 49/91] Explain email recovery better --- letsencrypt/display/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index cb424a81b..37ce66b62 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -124,7 +124,7 @@ def get_email(): """ while True: code, email = zope.component.getUtility(interfaces.IDisplay).input( - "Enter email address") + "Enter email address (used for urgent notices and lost key recovery)") if code == display_util.OK: if le_util.safe_email(email): From 47eaf297fba19668d7c5e483ec7abaa6085eafad Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 15:57:09 -0700 Subject: [PATCH 50/91] Start a letsencrypt-auto script Which handles all venv-related tasks for installing from pip, and gives us auto-updating. --- letsencrypt-auto | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 letsencrypt-auto diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 100755 index 000000000..82720a3fc --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1,45 @@ +#!/bin/bash -e +# +# Installs and updates the letencrypt virtualenv, and runs letsencrypt +# using that virtual environment. This allows the client to function decently +# without requiring specific versions of its dependencies from the operating +# system. + +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN=${VENV_PATH}/bin + +for arg in "$@" ; do + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] ; then + VERBOSE=1 + fi +done + +# virtualenv call is not idempotent: it overwrites pip upgraded in +# later steps, causing "ImportError: cannot import name unpack_url" +if [ ! -d $VENV_PATH ] +then + echo "Creating virtual environment..." + if [ "$VERBOSE" = 1 ] ; then + virtualenv --no-site-packages --python python2 $VENV_PATH + else + virtualenv --no-site-packages --python python2 $VENV_PATH > /dev/null + fi +fi + +echo "Updating letsencrypt and virtual environment dependencies..." +if [ "$VERBOSE" = 1 ] ; then + $VENV_BIN/pip install -U setuptools + $VENV_BIN/pip install -U pip + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx +else + $VENV_BIN/pip install -U setuptools > /dev/null + $VENV_BIN/pip install -U pip > /dev/null + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null +fi + +# TODO: install apache and nginx plugins by default? +# --pre is not necessary for dev releases in more recent pip versions + +sudo $VENV_BIN/letsencrypt $@ From b85e13de3f126fca8612bf3ce01c777d1a03615e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 16:07:12 -0700 Subject: [PATCH 51/91] Add bootstrapping to letsencrypt-auto --- letsencrypt-auto | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/letsencrypt-auto b/letsencrypt-auto index 82720a3fc..77a545fc6 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -20,6 +20,34 @@ done # later steps, causing "ImportError: cannot import name unpack_url" if [ ! -d $VENV_PATH ] then + BOOTSTRAP=`dirname $0`/bootstrap + if [ ! -f $BOOTSTRAP/debian.sh ] ; then + echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" + exit 1 + fi + if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + sudo $BOOTSTRAP/_deb_common.sh + elif [ -f /etc/arch-release ] ; then + echo "Bootstrapping dependencies for Archlinux..." + sudo $BOOTSTRAP/archlinux.sh + elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + sudo $BOOTSTRAP/_rpm_common.sh + elif uname | grep -iq FreeBSD ; then + echo "Bootstrapping dependencies for FreeBSD..." + sudo $BOOTSTRAP/freebsd.sh + elif uname | grep -iq Darwin ; then + echo "Bootstrapping dependencies for Mac OS X..." + sudo $BOOTSTRAP/mac.sh + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/using.html#prerequisites" + echo "for more info" + fi + echo "Creating virtual environment..." if [ "$VERBOSE" = 1 ] ; then virtualenv --no-site-packages --python python2 $VENV_PATH From 99793c54d6a0ca7a41ca9c1ab9ec9104e82dbc44 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 16:25:53 -0700 Subject: [PATCH 52/91] More portability --- bootstrap/mac.sh | 2 +- letsencrypt-auto | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh index 6779188a7..84e87cf5b 100755 --- a/bootstrap/mac.sh +++ b/bootstrap/mac.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e if ! hash brew 2>/dev/null; then echo "Homebrew Not Installed\nDownloading..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" diff --git a/letsencrypt-auto b/letsencrypt-auto index 77a545fc6..7f7533f39 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/sh -e # # Installs and updates the letencrypt virtualenv, and runs letsencrypt # using that virtual environment. This allows the client to function decently @@ -39,7 +39,8 @@ then sudo $BOOTSTRAP/freebsd.sh elif uname | grep -iq Darwin ; then echo "Bootstrapping dependencies for Mac OS X..." - sudo $BOOTSTRAP/mac.sh + echo "WARNING: Mac support is very experimental at present..." + $BOOTSTRAP/mac.sh else echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" echo From 7a1c37968cd35f9f7087cfaac24460afdf75bfeb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 17:21:36 -0700 Subject: [PATCH 53/91] Make the UX clearer before sudo --- letsencrypt-auto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 7f7533f39..7ee715bf4 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -68,7 +68,7 @@ else $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null fi -# TODO: install apache and nginx plugins by default? -# --pre is not necessary for dev releases in more recent pip versions - +# Explain what's about to happen, for the benefit of those getting sudo +# password prompts... +echo "Running with virtualenv:" sudo $VENV_BIN/letsencrypt "$@" sudo $VENV_BIN/letsencrypt $@ From 2e0dc4fc5061864f73e9742f85c5ebefd1198f5f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 18:25:14 -0700 Subject: [PATCH 54/91] Remove misleading Docker suggestion from the quick-install docs But keep it around in case anyone needs this? --- docs/using.rst | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index b2d2db42a..3e3973d2f 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -2,26 +2,6 @@ Using the Let's Encrypt client ============================== -Quick start -=========== - -Using Docker_ you can quickly get yourself a testing cert. From the -server that the domain your requesting a cert for resolves to, -`install Docker`_, issue the following command: - -.. code-block:: shell - - sudo docker run -it --rm -p 443:443 --name letsencrypt \ - -v "/etc/letsencrypt:/etc/letsencrypt" \ - -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest - -and follow the instructions. Your new cert will be available in -``/etc/letsencrypt/certs``. - -.. _Docker: https://docker.com -.. _`install Docker`: https://docs.docker.com/docker/userguide/ - Getting the code ================ @@ -174,3 +154,25 @@ By default, the following locations are searched: .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io + +Running with Docker +=================== + +Docker_ is another way to quickly obtaintesting certs. From the +server that the domain your requesting a cert for resolves to, +`install Docker`_, issue the following command: + +.. code-block:: shell + + sudo docker auth -it --rm -p 443:443 --name letsencrypt \ + -v "/etc/letsencrypt:/etc/letsencrypt" \ + -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ + quay.io/letsencrypt/letsencrypt:latest + +and follow the instructions. Your new cert will be available in +``/etc/letsencrypt/certs``. + +.. _Docker: https://docker.com +.. _`install Docker`: https://docs.docker.com/docker/userguide/ + + From 8a5d199ddffebc0b000b26989a9451f1d12f5639 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 18:35:34 -0700 Subject: [PATCH 55/91] Try to make the "using" instructions as simple and accurate as possible Some of the verbose stuff we had before is really of historical/developer interest only --- docs/contributing.rst | 96 +++++++++++++++++++++++++++++++++++++ docs/using.rst | 109 +++++++++++++++++++----------------------- 2 files changed, 146 insertions(+), 59 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 614f6f2aa..ed74bbb60 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -296,3 +296,99 @@ commands: This should generate documentation in the ``docs/_build/html`` directory. + +Notes on OS depedencies +======================= + +OS level dependencies are managed by scripts in ``bootstrap``. Some notes +are provided here mainly for the :ref:`developers ` reference. + +In general: + +* ``sudo`` is required as a suggested way of running privileged process +* `Augeas`_ is required for the Python bindings +* ``virtualenv`` and ``pip`` are used for managing other python library + dependencies + + +Ubuntu +------ + +.. code-block:: shell + + sudo ./bootstrap/ubuntu.sh + + +Debian +------ + +.. code-block:: shell + + sudo ./bootstrap/debian.sh + +For squeeze you will need to: + +- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. + + +.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 + + +Mac OSX +------- + +.. code-block:: shell + + ./bootstrap/mac.sh + + +Fedora +------ + +.. code-block:: shell + + sudo ./bootstrap/fedora.sh + + +Centos 7 +-------- + +.. code-block:: shell + + sudo ./bootstrap/centos.sh + + +FreeBSD +------- + +.. code-block:: shell + + sudo ./bootstrap/freebsd.sh + +Bootstrap script for FreeBSD uses ``pkg`` for package installation, +i.e. it does not use ports. + +FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see +below), you will need a compatbile shell, e.g. ``pkg install bash && +bash``. + + +Running with Docker +=================== + +Docker_ is another way to quickly obtaintesting certs. From the +server that the domain your requesting a cert for resolves to, +`install Docker`_, issue the following command: + +.. code-block:: shell + + sudo docker auth -it --rm -p 443:443 --name letsencrypt \ + -v "/etc/letsencrypt:/etc/letsencrypt" \ + -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ + quay.io/letsencrypt/letsencrypt:latest + +and follow the instructions. Your new cert will be available in +``/etc/letsencrypt/certs``. + +.. _Docker: https://docker.com +.. _`install Docker`: https://docs.docker.com/docker/userguide/ diff --git a/docs/using.rst b/docs/using.rst index 3e3973d2f..1920efe38 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -24,17 +24,62 @@ above method instead. .. _prerequisites: -Prerequisites -============= +Installation and Usage +====================== -The demo code is supported and known to work on **Ubuntu and -Debian**. Therefore, prerequisites for other platforms listed below -are provided mainly for the :ref:`developers ` reference. +To install and run the client you just need to type: + +.. code-block:: shell + + ./letsencrypt-auto + +.. warning:: Please do **not** use ``python setup.py install``. That mode of + operation might corrupt your operating system and is **not supported** + by the Let's Encrypt team! + +The ``letsencrypt`` commandline tool has a builtin help: + +.. code-block:: shell + + ./letsencrypt-auto --help + + +Configuration file +------------------ + +It is possible to specify configuration file with +``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For +instance, if you are a contributor, you might find the following +handy: + +.. include:: ../examples/dev-cli.ini + :code: ini + +By default, the following locations are searched: + +- ``/etc/letsencrypt/cli.ini`` +- ``$XDG_CONFIG_HOME/letsencrypt/cli.ini`` (or + ``~/.config/letsencrypt/cli.ini`` if ``$XDG_CONFIG_HOME`` is not + set). + +.. keep it up to date with constants.py + + +.. _Augeas: http://augeas.net/ +.. _Virtualenv: https://virtualenv.pypa.io + +Notes on OS depedencies +======================= + +OS level dependencies are managed by scripts in ``bootstrap``. Some notes +are provided here mainly for the :ref:`developers ` reference. In general: * ``sudo`` is required as a suggested way of running privileged process * `Augeas`_ is required for the Python bindings +* ``virtualenv`` and ``pip`` are used for managing other python library + dependencies Ubuntu @@ -99,61 +144,7 @@ below), you will need a compatbile shell, e.g. ``pkg install bash && bash``. -Installation -============ -.. code-block:: shell - - source ./bootstrap/venv.sh - -.. warning:: Please do **not** use ``python setup.py install``. Please - do **not** attempt the installation commands as - superuser/root and/or without Virtualenv_, e.g. ``sudo - python setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt - your operating system and are **not supported** by the - Let's Encrypt team! - - -Usage -===== - -To get a new certificate run: - -.. code-block:: shell - - (letsencrypt)$ letsencrypt auth - -The ``letsencrypt`` commandline tool has a builtin help: - -.. code-block:: shell - - (letsencrypt)$ letsencrypt --help - - -Configuration file ------------------- - -It is possible to specify configuration file with -``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For -instance, if you are a contributor, you might find the following -handy: - -.. include:: ../examples/dev-cli.ini - :code: ini - -By default, the following locations are searched: - -- ``/etc/letsencrypt/cli.ini`` -- ``$XDG_CONFIG_HOME/letsencrypt/cli.ini`` (or - ``~/.config/letsencrypt/cli.ini`` if ``$XDG_CONFIG_HOME`` is not - set). - -.. keep it up to date with constants.py - - -.. _Augeas: http://augeas.net/ -.. _Virtualenv: https://virtualenv.pypa.io Running with Docker =================== From 68ed333fc0ce55bfe54945a39d7f2a1190b69776 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 21:07:40 -0700 Subject: [PATCH 56/91] Quick fix --- letsencrypt/cli.py | 6 ++++++ letsencrypt/tests/cli_test.py | 10 ++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 07ccd38fd..92d845538 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -301,6 +301,12 @@ def _auth_from_domains(le_client, config, domains, plugins): raise errors.Error("Certificate could not be obtained") _report_new_cert(lineage.cert) + reporter_util = zope.component.getUtility(interfaces.IReporter) + reporter_util.add_message( + "Your certificate will expire at {0}. To obtain a new version of the " + "certificate in the future, simply run this client again.".format( + lineage.notafter().ctime()), + reporter_util.MEDIUM_PRIORITY) return lineage diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f690e77f9..a3efd9d40 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -128,8 +128,9 @@ class CLITest(unittest.TestCase): self._auth_new_request_common(mock_client) self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) - self.assertTrue( - cert_path in mock_get_utility().add_message.call_args[0][0]) + msg = mock_get_utility().add_message.call_args_list[0][0][0] + self.assertTrue(cert_path in msg) + self.assertEqual(mock_get_utility().add_message.call_count, 2) def test_auth_new_request_failure(self): mock_client = mock.MagicMock() @@ -164,8 +165,9 @@ class CLITest(unittest.TestCase): self.assertEqual(mock_lineage.save_successor.call_count, 1) mock_lineage.update_all_links_to.assert_called_once_with( mock_lineage.latest_common_version()) - self.assertTrue( - cert_path in mock_get_utility().add_message.call_args[0][0]) + msg = mock_get_utility().add_message.call_args_list[0][0][0] + self.assertTrue(cert_path in msg) + self.assertEqual(mock_get_utility().add_message.call_count, 2) @mock.patch('letsencrypt.cli.display_ops.pick_installer') @mock.patch('letsencrypt.cli.zope.component.getUtility') From 2cd0e64537a2c774f549386640f8f9be12b1fe73 Mon Sep 17 00:00:00 2001 From: lf Date: Wed, 14 Oct 2015 22:35:52 -0600 Subject: [PATCH 57/91] Change as_string to a __str__ in nginxparser.py This change would make the RawNginxDumper more in line with other Python libraries and the standard library. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index 2926a43d0..84b54b4db 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -84,9 +84,13 @@ class RawNginxDumper(object): else: yield spacer * current_indent + key + spacer + values + ';' - def as_string(self): + def __str__(self): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' + + def as_string(self): + """Return the parsed block as a string.""" + return str(self) # Shortcut functions to respect Python's serialization interface @@ -122,7 +126,7 @@ def dumps(blocks, indentation=4): :rtype: str """ - return RawNginxDumper(blocks, indentation).as_string() + return str(RawNginxDumper(blocks, indentation)) def dump(blocks, _file, indentation=4): From 87000ac5c60eda6b8f7e4c291c53bc97a006c3e7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 22:24:41 -0700 Subject: [PATCH 58/91] Move docker back into "using" It seems the changes removing bootstrap from "using" weren't commited earlier, either --- docs/contributing.rst | 21 ----------- docs/using.rst | 81 +------------------------------------------ 2 files changed, 1 insertion(+), 101 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ed74bbb60..8661b195e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -371,24 +371,3 @@ i.e. it does not use ports. FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see below), you will need a compatbile shell, e.g. ``pkg install bash && bash``. - - -Running with Docker -=================== - -Docker_ is another way to quickly obtaintesting certs. From the -server that the domain your requesting a cert for resolves to, -`install Docker`_, issue the following command: - -.. code-block:: shell - - sudo docker auth -it --rm -p 443:443 --name letsencrypt \ - -v "/etc/letsencrypt:/etc/letsencrypt" \ - -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest - -and follow the instructions. Your new cert will be available in -``/etc/letsencrypt/certs``. - -.. _Docker: https://docker.com -.. _`install Docker`: https://docs.docker.com/docker/userguide/ diff --git a/docs/using.rst b/docs/using.rst index 1920efe38..125aa29b3 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -68,83 +68,6 @@ By default, the following locations are searched: .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Notes on OS depedencies -======================= - -OS level dependencies are managed by scripts in ``bootstrap``. Some notes -are provided here mainly for the :ref:`developers ` reference. - -In general: - -* ``sudo`` is required as a suggested way of running privileged process -* `Augeas`_ is required for the Python bindings -* ``virtualenv`` and ``pip`` are used for managing other python library - dependencies - - -Ubuntu ------- - -.. code-block:: shell - - sudo ./bootstrap/ubuntu.sh - - -Debian ------- - -.. code-block:: shell - - sudo ./bootstrap/debian.sh - -For squeeze you will need to: - -- Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. - - -.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 - - -Mac OSX -------- - -.. code-block:: shell - - ./bootstrap/mac.sh - - -Fedora ------- - -.. code-block:: shell - - sudo ./bootstrap/fedora.sh - - -Centos 7 --------- - -.. code-block:: shell - - sudo ./bootstrap/centos.sh - - -FreeBSD -------- - -.. code-block:: shell - - sudo ./bootstrap/freebsd.sh - -Bootstrap script for FreeBSD uses ``pkg`` for package installation, -i.e. it does not use ports. - -FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see -below), you will need a compatbile shell, e.g. ``pkg install bash && -bash``. - - - Running with Docker =================== @@ -158,12 +81,10 @@ server that the domain your requesting a cert for resolves to, sudo docker auth -it --rm -p 443:443 --name letsencrypt \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest + quay.io/letsencrypt/letsencrypt:latest auth and follow the instructions. Your new cert will be available in ``/etc/letsencrypt/certs``. .. _Docker: https://docker.com .. _`install Docker`: https://docs.docker.com/docker/userguide/ - - From ecd987b8ca9dc3c31e9702d2f871b9f840ba51ed Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 22:30:18 -0700 Subject: [PATCH 59/91] letsencrypt-auto now knows about -vvvv --- letsencrypt-auto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 7ee715bf4..5106e2fae 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -11,7 +11,8 @@ VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin for arg in "$@" ; do - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] ; then + # This first clause is redundant with the third, but hedging on portability + if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- -v+ ; then VERBOSE=1 fi done From 94622f5edd136c8904a8f7aa0d78a351e4bb2ea2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 14 Oct 2015 22:53:36 -0700 Subject: [PATCH 60/91] Date only because hard to do time with no time --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 92d845538..24f032ca1 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -303,9 +303,9 @@ def _auth_from_domains(le_client, config, domains, plugins): _report_new_cert(lineage.cert) reporter_util = zope.component.getUtility(interfaces.IReporter) reporter_util.add_message( - "Your certificate will expire at {0}. To obtain a new version of the " + "Your certificate will expire on {0}. To obtain a new version of the " "certificate in the future, simply run this client again.".format( - lineage.notafter().ctime()), + lineage.notafter().date()), reporter_util.MEDIUM_PRIORITY) return lineage From 39e489d03cd935dcbbc725ad7370441d789b4c6d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:03:18 -0700 Subject: [PATCH 61/91] Do not assume sudo is present - On Digital Ocean and perhaps other platforms, the user is root by default and sudo is uninstalled --- letsencrypt-auto | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 5106e2fae..662908c47 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -10,6 +10,12 @@ VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin +if test "`id -u`" -ne "0" ; then + SUDO=sudo +else + SUDO= +fi + for arg in "$@" ; do # This first clause is redundant with the third, but hedging on portability if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- -v+ ; then @@ -28,16 +34,16 @@ then fi if [ -f /etc/debian_version ] ; then echo "Bootstrapping dependencies for Debian-based OSes..." - sudo $BOOTSTRAP/_deb_common.sh + $SUDO $BOOTSTRAP/_deb_common.sh elif [ -f /etc/arch-release ] ; then echo "Bootstrapping dependencies for Archlinux..." - sudo $BOOTSTRAP/archlinux.sh + $SUDO $BOOTSTRAP/archlinux.sh elif [ -f /etc/redhat-release ] ; then echo "Bootstrapping dependencies for RedHat-based OSes..." - sudo $BOOTSTRAP/_rpm_common.sh + $SUDO $BOOTSTRAP/_rpm_common.sh elif uname | grep -iq FreeBSD ; then echo "Bootstrapping dependencies for FreeBSD..." - sudo $BOOTSTRAP/freebsd.sh + $SUDO $BOOTSTRAP/freebsd.sh elif uname | grep -iq Darwin ; then echo "Bootstrapping dependencies for Mac OS X..." echo "WARNING: Mac support is very experimental at present..." @@ -71,5 +77,5 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... -echo "Running with virtualenv:" sudo $VENV_BIN/letsencrypt "$@" -sudo $VENV_BIN/letsencrypt $@ +echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" +$SUDO $VENV_BIN/letsencrypt $@ From ab036e98e08ad3c9b9f686702ab03f16c640657e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:06:20 -0700 Subject: [PATCH 62/91] Do not ship letsencrypt-nginx until it's somewhat plausibly working --- letsencrypt-auto | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 662908c47..20a7a40fa 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -68,11 +68,13 @@ echo "Updating letsencrypt and virtual environment dependencies..." if [ "$VERBOSE" = 1 ] ; then $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx + # nginx is buggy / disabled for now... + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache #letsencrypt-nginx else $VENV_BIN/pip install -U setuptools > /dev/null $VENV_BIN/pip install -U pip > /dev/null - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx > /dev/null + # nginx is buggy / disabled for now... + $VENV_BIN/pip install -U letsencrypt letsencrypt-apache > /dev/null fi # Explain what's about to happen, for the benefit of those getting sudo From 2794b762d6199dc2c3f1e63ede22eb6c7e08ffe2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:41:37 -0700 Subject: [PATCH 63/91] Add a subtle progress bar --- letsencrypt-auto | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index 20a7a40fa..b43c22f87 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -64,17 +64,23 @@ then fi fi -echo "Updating letsencrypt and virtual environment dependencies..." +echo -n "Updating letsencrypt and virtual environment dependencies..." if [ "$VERBOSE" = 1 ] ; then + echo $VENV_BIN/pip install -U setuptools $VENV_BIN/pip install -U pip # nginx is buggy / disabled for now... $VENV_BIN/pip install -U letsencrypt letsencrypt-apache #letsencrypt-nginx else $VENV_BIN/pip install -U setuptools > /dev/null + echo -n . $VENV_BIN/pip install -U pip > /dev/null + echo -n . # nginx is buggy / disabled for now... - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache > /dev/null + $VENV_BIN/pip install -U letsencrypt > /dev/null + echo -n . + $VENV_BIN/pip install -U letsencrypt-apache > /dev/null + echo fi # Explain what's about to happen, for the benefit of those getting sudo From be77909ef21af57a7d10a9799bd12460770db9fb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:53:50 -0700 Subject: [PATCH 64/91] Further fixes to the docs --- docs/using.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 125aa29b3..5683fba21 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -22,8 +22,6 @@ above method instead. https://github.com/letsencrypt/letsencrypt/archive/master.zip -.. _prerequisites: - Installation and Usage ====================== @@ -33,9 +31,13 @@ To install and run the client you just need to type: ./letsencrypt-auto -.. warning:: Please do **not** use ``python setup.py install``. That mode of - operation might corrupt your operating system and is **not supported** - by the Let's Encrypt team! +(Once letsencrypt is packaged by distributions, the command will just be +``letsencrypt``. ``letsencrypt-auto`` is a wrapper which installs virtualized +dependencies and provides automated updates during the beta program) + +.. warning:: Please do **not** use ``python setup.py install`` or ``sudo pip install`. + Those mode of operation might corrupt your operating system and is + **not supported** by the Let's Encrypt team! The ``letsencrypt`` commandline tool has a builtin help: @@ -48,7 +50,7 @@ Configuration file ------------------ It is possible to specify configuration file with -``letsencrypt --config cli.ini`` (or shorter ``-c cli.ini``). For +``letsencrypt-auto --config cli.ini`` (or shorter ``-c cli.ini``). For instance, if you are a contributor, you might find the following handy: @@ -72,7 +74,7 @@ By default, the following locations are searched: Running with Docker =================== -Docker_ is another way to quickly obtaintesting certs. From the +Docker_ is another way to quickly obtain testing certs. From the server that the domain your requesting a cert for resolves to, `install Docker`_, issue the following command: From 4ef385d3ad7e174452996cb41da0c9f6c02d8ab5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 14 Oct 2015 23:56:56 -0700 Subject: [PATCH 65/91] Protect quotes on the way to the actual command --- letsencrypt-auto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto b/letsencrypt-auto index b43c22f87..72c0c8aa4 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -86,4 +86,4 @@ fi # Explain what's about to happen, for the benefit of those getting sudo # password prompts... echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt $@ +$SUDO $VENV_BIN/letsencrypt "$@" From 0fb00ca039d390de7508f7a0cfba2f94dc9d4134 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:03:49 -0700 Subject: [PATCH 66/91] Try to make the dependencies link work --- docs/contributing.rst | 2 ++ letsencrypt-auto | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 8661b195e..4c3d91902 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -297,6 +297,8 @@ commands: This should generate documentation in the ``docs/_build/html`` directory. +.. _prerequisites: + Notes on OS depedencies ======================= diff --git a/letsencrypt-auto b/letsencrypt-auto index 72c0c8aa4..0b3d9b72d 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -52,7 +52,7 @@ then echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" echo echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/using.html#prerequisites" + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info" fi From 0fb8e3c47947e2b124734921e7b405c669703e16 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:07:26 -0700 Subject: [PATCH 67/91] Move augeaus & venv references --- docs/contributing.rst | 2 ++ docs/using.rst | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4c3d91902..6d0a2d4ba 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -312,6 +312,8 @@ In general: * ``virtualenv`` and ``pip`` are used for managing other python library dependencies +.. _Augeas: http://augeas.net/ +.. _Virtualenv: https://virtualenv.pypa.io Ubuntu ------ diff --git a/docs/using.rst b/docs/using.rst index 5683fba21..0a781431a 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,10 +67,6 @@ By default, the following locations are searched: .. keep it up to date with constants.py -.. _Augeas: http://augeas.net/ -.. _Virtualenv: https://virtualenv.pypa.io - - Running with Docker =================== From 64385cbd80cde04e9ee53fd046df5bc0ee6ef270 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 15 Oct 2015 00:09:42 -0700 Subject: [PATCH 68/91] Disable nginx in bootstrap/venv.sh too --- bootstrap/venv.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh index 619135d0d..ce31e6703 100755 --- a/bootstrap/venv.sh +++ b/bootstrap/venv.sh @@ -20,9 +20,7 @@ fi pip install -U setuptools pip install -U pip -# TODO: install apache and nginx plugins by default? -# --pre is not necessary for dev releases in more recent pip versions -pip install -U letsencrypt letsencrypt-apache letsencrypt-nginx +pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx echo echo "Congratulations, Let's Encrypt has been successfully installed/updated!" From e7809563b16f197659b6e77e66b6a55fe8a6a470 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 17:23:43 +0000 Subject: [PATCH 69/91] Address first batch of Seth's review comments. --- acme/acme/standalone.py | 7 ++++--- acme/acme/standalone_test.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 089c2ff18..97e52fa9f 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -46,7 +46,8 @@ class TLSServer(socketserver.TCPServer): class ACMEServerMixin: # pylint: disable=old-style-class """ACME server common settings mixin.""" - server_version = "ACME standalone client" + # TODO: c.f. #858 + server_version = "ACME client standalone challenge solver" allow_reuse_address = True def __init__(self): @@ -95,7 +96,7 @@ class SimpleHTTPServer(BaseHTTPServer.HTTPServer, ACMEServerMixin): class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """SimpleHTTP challenge handler. - Adheres to the stdlib"s `socketserver.BaseRequestHandler` interface. + Adheres to the stdlib's `socketserver.BaseRequestHandler` interface. :ivar set simple_http_resources: A set of `SimpleHTTPResource` objects. TODO: better name? @@ -119,7 +120,7 @@ class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def handle_index(self): """Handle index page.""" self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header("Content-Type", "text/html") self.end_headers() self.wfile.write(self.server.server_version.encode()) diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 7eda2fce3..14d212d6e 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -122,7 +122,8 @@ class SimpleHTTPServerTest(unittest.TestCase): def test_index(self): response = requests.get( 'http://localhost:{0}'.format(self.port), verify=False) - self.assertEqual(response.text, 'ACME standalone client') + self.assertEqual( + response.text, 'ACME client standalone challenge solver') self.assertTrue(response.ok) def test_404(self): From 6f44bcf11795814f2d4401d76321bba1fbbf763a Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 18:01:14 +0000 Subject: [PATCH 70/91] standalone2: move alread_listening to perform --- letsencrypt/plugins/standalone.py | 12 +++++++----- letsencrypt/plugins/standalone_test.py | 12 ++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cb95ec408..e742734a9 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -182,11 +182,7 @@ class Authenticator(common.Plugin): return self.__doc__ def prepare(self): # pylint: disable=missing-docstring - if any(util.already_listening(port) for port in - (self.config.dvsni_port, self.config.simple_http_port)): - raise errors.MisconfigurationError( - "At least one of the (possibly) required ports is " - "already taken.") + pass def get_chall_pref(self, domain): # pylint: disable=unused-argument,missing-docstring @@ -195,6 +191,12 @@ class Authenticator(common.Plugin): return chall_pref def perform(self, achalls): # pylint: disable=missing-docstring + if any(util.already_listening(port) for port in + (self.config.dvsni_port, self.config.simple_http_port)): + raise errors.MisconfigurationError( + "At least one of the (possibly) required ports is " + "already taken.") + try: return self.perform2(achalls) except errors.StandaloneBindError as error: diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index e99bd473a..b873da6f2 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -101,16 +101,16 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) - @mock.patch("letsencrypt.plugins.standalone.util") - def test_prepare_misconfiguration(self, mock_util): - mock_util.already_listening.return_value = True - self.assertRaises(errors.MisconfigurationError, self.auth.prepare) - mock_util.already_listening.assert_called_once_with(1234) - def test_get_chall_pref(self): self.assertEqual(set(self.auth.get_chall_pref(domain=None)), set([challenges.DVSNI, challenges.SimpleHTTP])) + @mock.patch("letsencrypt.plugins.standalone.util") + def test_perform_misconfiguration(self, mock_util): + mock_util.already_listening.return_value = True + self.assertRaises(errors.MisconfigurationError, self.auth.perform, []) + mock_util.already_listening.assert_called_once_with(1234) + @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") def test_perform(self, unused_mock_get_utility): achalls = [1, 2, 3] From ec24641511e251e9b24cb802783dbd271dfa9ec9 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:31:22 +0000 Subject: [PATCH 71/91] standalone2: run(): tls -> challenge_type. --- letsencrypt/plugins/standalone.py | 16 +++++++++------- letsencrypt/plugins/standalone_test.py | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index e742734a9..3ad823e9c 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -44,27 +44,29 @@ class ServerManager(object): self.certs = certs self.simple_http_resources = simple_http_resources - def run(self, port, tls): + def run(self, port, challenge_type): """Run ACME server on specified ``port``. This method is idempotent, i.e. all calls with the same pair of - ``(port, tls)`` will reuse the same server. + ``(port, challenge_type)`` will reuse the same server. :param int port: Port to run the server on. - :param bool tls: TLS or non-TLS? + :param challenge_type: Subclass of `acme.challenges.Challenge`, + either `acme.challenge.SimpleHTTP` or `acme.challenges.DVSNI`. :returns: Server instance. :rtype: ACMEServerMixin """ + assert challenge_type in (challenges.DVSNI, challenges.SimpleHTTP) if port in self._instances: return self._instances[port].server address = ("", port) try: - if tls: + if challenge_type is challenges.DVSNI: server = acme_standalone.DVSNIServer(address, self.certs) - else: + else: # challenges.SimpleHTTP server = acme_standalone.SimpleHTTPServer( address, self.simple_http_resources) except socket.error as error: @@ -224,7 +226,7 @@ class Authenticator(common.Plugin): for achall in achalls: if isinstance(achall, achallenges.SimpleHTTP): server = self.servers.run( - self.config.simple_http_port, tls=False) + self.config.simple_http_port, challenges.SimpleHTTP) response, validation = achall.gen_response_and_validation( tls=False) self.simple_http_resources.add( @@ -234,7 +236,7 @@ class Authenticator(common.Plugin): cert = self.simple_http_cert domain = achall.domain else: # DVSNI - server = self.servers.run(self.config.dvsni_port, tls=True) + server = self.servers.run(self.config.dvsni_port, challenges.DVSNI) response, cert, _ = achall.gen_cert_and_response(self.key) domain = response.z_domain self.certs[domain] = (self.key, cert) diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index b873da6f2..0ccdccb1f 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -32,23 +32,23 @@ class ServerManagerTest(unittest.TestCase): self.assertTrue( self.mgr.simple_http_resources is self.simple_http_resources) - def _test_run_stop(self, tls): - server = self.mgr.run(port=0, tls=tls) + def _test_run_stop(self, challenge_type): + server = self.mgr.run(port=0, challenge_type=challenge_type) port = server.socket.getsockname()[1] # pylint: disable=no-member self.assertEqual(self.mgr.running(), {port: server}) self.mgr.stop(port=port) self.assertEqual(self.mgr.running(), {}) - def test_run_stop_tls(self): - self._test_run_stop(tls=True) + def test_run_stop_dvsni(self): + self._test_run_stop(challenges.DVSNI) - def test_run_stop_non_tls(self): - self._test_run_stop(tls=False) + def test_run_stop_simplehttp(self): + self._test_run_stop(challenges.SimpleHTTP) def test_run_idempotent(self): - server = self.mgr.run(port=0, tls=False) + server = self.mgr.run(port=0, challenge_type=challenges.SimpleHTTP) port = server.socket.getsockname()[1] # pylint: disable=no-member - server2 = self.mgr.run(port=port, tls=False) + server2 = self.mgr.run(port=port, challenge_type=challenges.SimpleHTTP) self.assertEqual(self.mgr.running(), {port: server}) self.assertTrue(server is server2) self.mgr.stop(port) @@ -59,7 +59,8 @@ class ServerManagerTest(unittest.TestCase): some_server.bind(("", 0)) port = some_server.getsockname()[1] self.assertRaises( - errors.StandaloneBindError, self.mgr.run, port, tls=False) + errors.StandaloneBindError, self.mgr.run, port, + challenge_type=challenges.SimpleHTTP) self.assertEqual(self.mgr.running(), {}) @@ -165,7 +166,9 @@ class AuthenticatorTest(unittest.TestCase): self.assertTrue(isinstance(responses[1], challenges.DVSNIResponse)) self.assertEqual(self.auth.servers.run.mock_calls, [ - mock.call(4321, tls=False), mock.call(1234, tls=True)]) + mock.call(4321, challenges.SimpleHTTP), + mock.call(1234, challenges.DVSNI), + ]) self.assertEqual(self.auth.served, { "server1234": set([dvsni]), "server4321": set([simple_http]), From 50435740937ef768fc07cd6de63396e39142875c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:42:10 +0000 Subject: [PATCH 72/91] Integration tests: SimpleHTTP on port 5002. Should be pulled by @jsha in lockstep with https://github.com/letsencrypt/boulder/commit/6b8f1c27003bb43f33ffcfa9a5f63eb4154c6df9. --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index fd60b9258..ab645f6d6 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5001 \ + --simple-http-port 5002 \ --manual-test-mode \ $store_flags \ --text \ From d1b7b0553fbccfb8579615b8c38517b8596e789f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 15 Oct 2015 20:53:47 +0000 Subject: [PATCH 73/91] Back to 5001, but with env var --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index ab645f6d6..418856def 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,7 +16,7 @@ letsencrypt_test () { --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ --dvsni-port 5001 \ - --simple-http-port 5002 \ + --simple-http-port ${SIMPLE_HTTP_PORT:-5001} \ --manual-test-mode \ $store_flags \ --text \ From 22b8446efb8c8f7f4d1438e7460e3b8fa1634615 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 15 Oct 2015 14:32:52 -0700 Subject: [PATCH 74/91] Get precompiled Goose from GithHub. Rather than fetching from bitbucket and building. Bitbucket is often down, and building from scratch is slow. Github is sometimes down, but at least now we have our eggs in one basket. --- tests/boulder-start.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 530f9c598..051c832f2 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -32,11 +32,10 @@ export PATH="$GOPATH/bin:$PATH" go get -d github.com/letsencrypt/boulder/... cd $GOPATH/src/github.com/letsencrypt/boulder # goose is needed for ./test/create_db.sh -if ! go get bitbucket.org/liamstask/goose/cmd/goose ; then - echo Problems installing goose... perhaps rm -rf \$GOPATH \("$GOPATH"\) - echo and try again... - exit 1 -fi +wget https://github.com/jsha/boulder-tools/raw/master/goose.gz && \ + mkdir $GOPATH/bin && \ + zcat goose.gz > $GOPATH/bin/goose && \ + chmod +x $GOPATH/bin/goose ./test/create_db.sh ./start.py & # Hopefully start.py bootstraps before integration test is started... From 0614fba9df41df3c9b50915786e865186ad42f03 Mon Sep 17 00:00:00 2001 From: David Dworken Date: Thu, 15 Oct 2015 20:21:41 -0400 Subject: [PATCH 75/91] Make the script POSIX compliant so it works with dash (Fixes #977) --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 981c8967a..55b42ef9a 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -14,7 +14,7 @@ export PATH="/usr/sbin:$PATH" # /usr/sbin/nginx export GOPATH="${GOPATH:-/tmp/go}" export PATH="$GOPATH/bin:$PATH" -if [ `uname` == 'Darwin' ]; then +if [ `uname` = "Darwin" ];then readlink="greadlink" else readlink="readlink" From 2956d5d913a262cb11f53e8eda012abf6ad1958d Mon Sep 17 00:00:00 2001 From: lf Date: Thu, 15 Oct 2015 21:28:32 -0600 Subject: [PATCH 76/91] Trailing whitespace. Oops. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index 84b54b4db..c9a24aabe 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -87,7 +87,7 @@ class RawNginxDumper(object): def __str__(self): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' - + def as_string(self): """Return the parsed block as a string.""" return str(self) From fd1103ccda1c6c69bec57e868d508d433293df4c Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 21:25:18 +0000 Subject: [PATCH 77/91] Tox tests detect downstream deps errors. --- tox.ini | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index b10558077..4eddc0e9d 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,14 @@ envlist = py27,cover,lint [testenv] commands = - pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt - # -q does not suppress errors - python setup.py test -q - python setup.py test -q -s acme - python setup.py test -q -s letsencrypt_apache - python setup.py test -q -s letsencrypt_nginx - python setup.py test -q -s letshelp_letsencrypt + pip install -e acme[testing] + nosetests acme + pip install -r requirements.txt -e .[testing] + nosetests letsencrypt + pip install letsencrypt-apache + nosetests letsencrypt_apache + pip install letsencrypt-nginx + nosetests letsencrypt_nginx setenv = PYTHONPATH = {toxinidir} From 2f3b6251c70beacbe4bc72fb64a3fbfcebe64f7e Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 21:28:23 +0000 Subject: [PATCH 78/91] Tox: verbose nosetests --- tox.ini | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 4eddc0e9d..0bd085975 100644 --- a/tox.ini +++ b/tox.ini @@ -8,16 +8,19 @@ skipsdist = true envlist = py27,cover,lint +# nosetest -v => more verbose output, allows to detect busy waiting +# loops, especially on Travis + [testenv] commands = pip install -e acme[testing] - nosetests acme + nosetests -v acme pip install -r requirements.txt -e .[testing] - nosetests letsencrypt + nosetests -v letsencrypt pip install letsencrypt-apache - nosetests letsencrypt_apache + nosetests -v letsencrypt_apache pip install letsencrypt-nginx - nosetests letsencrypt_nginx + nosetests -v letsencrypt_nginx setenv = PYTHONPATH = {toxinidir} @@ -27,12 +30,12 @@ setenv = [testenv:py33] commands = pip install -e acme[testing] - nosetests acme + nosetests -v acme [testenv:py34] commands = pip install -e acme[testing] - nosetests acme + nosetests -v acme [testenv:cover] basepython = python2.7 From fd4528baa39452aa38357cc87a2476646fd63e85 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:04:18 +0000 Subject: [PATCH 79/91] tox: pip install -e --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0bd085975..4818a7998 100644 --- a/tox.ini +++ b/tox.ini @@ -17,9 +17,9 @@ commands = nosetests -v acme pip install -r requirements.txt -e .[testing] nosetests -v letsencrypt - pip install letsencrypt-apache + pip install -e letsencrypt-apache nosetests -v letsencrypt_apache - pip install letsencrypt-nginx + pip install -e letsencrypt-nginx nosetests -v letsencrypt_nginx setenv = From 408903b7326a61212032582a5c84a377fd44d9d4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:04:52 +0000 Subject: [PATCH 80/91] tox: bring back test for letshelp_letsencrypt --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 4818a7998..e45c1c125 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,8 @@ commands = nosetests -v letsencrypt_apache pip install -e letsencrypt-nginx nosetests -v letsencrypt_nginx + pip install -e letshelp-letsencrypt + nosetests -v letshelp_letsencrypt setenv = PYTHONPATH = {toxinidir} From 86cd5af76e2fa532d8dbb1f0026078d3894fffd3 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:17:31 +0000 Subject: [PATCH 81/91] Add comment about downstream deps fix --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index e45c1c125..febf1a2d0 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,8 @@ envlist = py27,cover,lint # loops, especially on Travis [testenv] +# packages installed separately to ensure that dowstream deps problems +# are detected, c.f. #1002 commands = pip install -e acme[testing] nosetests -v acme From 5c1858627b5fd1703b8bd2435d4573ea997073c4 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:25:20 +0000 Subject: [PATCH 82/91] pep8 love --- letsencrypt/cli.py | 2 +- letsencrypt/client.py | 2 +- letsencrypt/plugins/null.py | 2 +- letsencrypt/tests/client_test.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 24f032ca1..477cb653f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -555,7 +555,7 @@ class HelpfulArgumentParser(object): for i, token in enumerate(args): if token in VERBS: - reordered = args[:i] + args[i+1:] + [args[i]] + reordered = args[:i] + args[(i + 1):] + [args[i]] self.verb = token return reordered diff --git a/letsencrypt/client.py b/letsencrypt/client.py index bb04f4f5a..3e32ab015 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -311,7 +311,7 @@ class Client(object): return os.path.abspath(act_cert_path), cert_chain_abspath def deploy_certificate(self, domains, privkey_path, - cert_path, chain_path, fullchain_path): + cert_path, chain_path, fullchain_path): """Install certificate :param list domains: list of domains to install the certificate diff --git a/letsencrypt/plugins/null.py b/letsencrypt/plugins/null.py index e87537684..cdb96a116 100644 --- a/letsencrypt/plugins/null.py +++ b/letsencrypt/plugins/null.py @@ -31,7 +31,7 @@ class Installer(common.Plugin): return [] def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): + chain_path=None, fullchain_path=None): pass # pragma: no cover def enhance(self, domain, enhancement, options=None): diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 9a5a2bbe1..3f7b84a64 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -151,8 +151,8 @@ class ClientTest(unittest.TestCase): installer = mock.MagicMock() self.client.installer = installer - self.client.deploy_certificate(["foo.bar"], "key", "cert", "chain", - "fullchain") + self.client.deploy_certificate( + ["foo.bar"], "key", "cert", "chain", "fullchain") installer.deploy_cert.assert_called_once_with( cert_path=os.path.abspath("cert"), chain_path=os.path.abspath("chain"), From 37cb419b1c59a49b77a7bcf748a617f47de3c2ce Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 16 Oct 2015 22:28:22 +0000 Subject: [PATCH 83/91] Switch to Go 1.5.1 (fixes #955) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3041fdd82..ff8a1038a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ services: # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS # gimme has to be kept in sync with Boulder's Go version setting in .travis.yml before_install: - - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5)"' + - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5.1)"' # using separate envs with different TOXENVs creates 4x1 Travis build # matrix, which allows us to clearly distinguish which component under From 670bc1e3d3f6145e98060a069806781cc22695ba Mon Sep 17 00:00:00 2001 From: lf Date: Fri, 16 Oct 2015 19:03:21 -0600 Subject: [PATCH 84/91] Remove as_string as per request. --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index c9a24aabe..cef0756d7 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -88,10 +88,6 @@ class RawNginxDumper(object): """Return the parsed block as a string.""" return '\n'.join(self) + '\n' - def as_string(self): - """Return the parsed block as a string.""" - return str(self) - # Shortcut functions to respect Python's serialization interface # (like pyyaml, picker or json) From 78c296bbd12fadc1e675f14aabbd1986d6abd3f6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 07:43:07 +0000 Subject: [PATCH 85/91] Integration tests: force le3.wtf over DVSNI. Becuse of https://github.com/letsencrypt/boulder/issues/985, we need to force standalone to one particular type of challenge at a time. #895 didn't fix it properly - it forgot about issuance of le3.wtf (CSR-initiated), which results in Travis builds failing 50% of a time, e.g. https://travis-ci.org/letsencrypt/letsencrypt/jobs/85875483#L3318. --- tests/boulder-integration.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 0b726ccde..f01c0eff3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -29,14 +29,22 @@ common() { # TODO: boulder#985 common_dvsni() { - common --dvsni-port 5001 --simple-http-port 0 "$@" + common \ + --standalone-supported-challenges dvsni \ + --dvsni-port 5001 \ + --simple-http-port 0 \ + "$@" } common_http() { - common --dvsni-port 0 --simple-http-port ${SIMPLE_HTTP_PORT:-5001} "$@" + common \ + --standalone-supported-challenges simpleHttp \ + --dvsni-port 0 \ + --simple-http-port ${SIMPLE_HTTP_PORT:-5001} \ + "$@" } -common_dvsni --domains le1.wtf --standalone-supported-challenges dvsni auth -common_http --domains le2.wtf --standalone-supported-challenges simpleHttp run +common_dvsni --domains le1.wtf auth +common_http --domains le2.wtf run common_http -a manual -d le.wtf auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ From 5621cf325d2e3b8ee3141a5aa5ef2d47d7dafae5 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 09:02:32 +0000 Subject: [PATCH 86/91] dev-release.sh: Update setuptools/pip --- tools/dev-release.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index cebe5001c..6a981b4c6 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -74,9 +74,10 @@ python -m SimpleHTTPServer $PORT & # installed from local PyPI rather than current directory (repo root) virtualenv --no-site-packages ../venv . ../venv/bin/activate -# Now, use our local PyPI. --pre allows installation of pre-release (incl. dev) +pip install -U setuptools +pip install -U pip +# Now, use our local PyPI pip install \ - --pre \ --extra-index-url http://localhost:$PORT \ letsencrypt $SUBPKGS # stop local PyPI From da631557241f09acc3fa209e3be3342073323c2f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 09:09:57 +0000 Subject: [PATCH 87/91] dev release: clean up subpkgs/subpkg_modules confusion --- tools/dev-release.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 6a981b4c6..655f66235 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -10,8 +10,8 @@ RELEASE_GPG_KEY="${RELEASE_GPG_KEY:-148C30F6F7E429337A72D992B00B9CC82D7ADF2C}" PORT=${PORT:-1234} # subpackages to be released -SUBPKGS=${SUBPKGS:-"acme letsencrypt_apache letsencrypt_nginx letshelp_letsencrypt"} -subpkgs_dirs="$(echo $SUBPKGS | sed s/_/-/g)" +SUBPKGS=${SUBPKGS:-"acme letsencrypt-apache letsencrypt-nginx letshelp-letsencrypt"} +subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" # letsencrypt_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Let's Encrypt devs # - it causes problems when running nosetests - the latter tries to @@ -29,7 +29,7 @@ cd $root git branch -f "$DEV_RELEASE_BRANCH" git checkout "$DEV_RELEASE_BRANCH" -for pkg_dir in $subpkgs_dirs +for pkg_dir in $SUBPKGS do sed -i $x "s/^version.*/version = '$version'/" $pkg_dir/setup.py done @@ -41,7 +41,7 @@ git tag --local-user "$RELEASE_GPG_KEY" \ --sign --message "Release $version" "$tag" echo "Preparing sdists and wheels" -for pkg_dir in . $subpkgs_dirs +for pkg_dir in . $SUBPKGS do cd $pkg_dir @@ -61,7 +61,7 @@ done mkdir "dist.$version" mv dist "dist.$version/letsencrypt" -for pkg_dir in $subpkgs_dirs +for pkg_dir in $SUBPKGS do mv $pkg_dir/dist "dist.$version/$pkg_dir/" done @@ -89,7 +89,7 @@ mkdir ../kgs kgs="../kgs/$version" pip freeze | tee $kgs pip install nose -nosetests letsencrypt $SUBPKGS +nosetests letsencrypt $subpkgs_modules echo "New root: $root" echo "KGS is at $root/kgs" From 31109b3332ed6314b2ea0e17956837c1d32421a8 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 10:55:41 +0000 Subject: [PATCH 88/91] log git commit in dev release --- tools/dev-release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 655f66235..682594f81 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -25,6 +25,7 @@ git tag --delete "$tag" || true root="$(mktemp -d -t le.$version.XXX)" echo "Cloning into fresh copy at $root" # clean repo = no artificats git clone . $root +git rev-parse HEAD cd $root git branch -f "$DEV_RELEASE_BRANCH" git checkout "$DEV_RELEASE_BRANCH" From 0cf2b5a4bd5f0b469522baa946e6be3346b1d29e Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 10:57:02 +0000 Subject: [PATCH 89/91] dev release: start with fresh virtualenv --- tools/dev-release.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 682594f81..3d6f97202 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -22,6 +22,15 @@ tag="v$version" mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true +tmpvenv=$(mktemp -d) +virtualenv --no-site-packages $tmpvenv +. $tmpvenv/bin/activate +pip install -U setuptools +pip install -U pip +pip install -U wheel +# updating virtualenv does some crazy stuff... +pip install -U virtualenv + root="$(mktemp -d -t le.$version.XXX)" echo "Cloning into fresh copy at $root" # clean repo = no artificats git clone . $root From 9ebf530dc4f5882dc298fe3090d90fb894692ae7 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 11:12:46 +0000 Subject: [PATCH 90/91] dev-release: more comments about recent changes --- tools/dev-release.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 3d6f97202..57f1b8a8e 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -25,10 +25,12 @@ git tag --delete "$tag" || true tmpvenv=$(mktemp -d) virtualenv --no-site-packages $tmpvenv . $tmpvenv/bin/activate +# update setuptools/pip just like in other places in the repo pip install -U setuptools -pip install -U pip -pip install -U wheel -# updating virtualenv does some crazy stuff... +pip install -U pip # latest pip => no --pre for dev releases +pip install -U wheel # setup.py bdist_wheel newer versions of +# virtualenv inherit setuptools/pip/wheel versions from current env +# when creating a child env pip install -U virtualenv root="$(mktemp -d -t le.$version.XXX)" From 17a09eac6b5e38c1d49a14ba9bec892295a42018 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 17 Oct 2015 11:13:43 +0000 Subject: [PATCH 91/91] fix comment --- tools/dev-release.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/dev-release.sh b/tools/dev-release.sh index 57f1b8a8e..6bbc6ced4 100755 --- a/tools/dev-release.sh +++ b/tools/dev-release.sh @@ -28,9 +28,10 @@ virtualenv --no-site-packages $tmpvenv # update setuptools/pip just like in other places in the repo pip install -U setuptools pip install -U pip # latest pip => no --pre for dev releases -pip install -U wheel # setup.py bdist_wheel newer versions of -# virtualenv inherit setuptools/pip/wheel versions from current env -# when creating a child env +pip install -U wheel # setup.py bdist_wheel + +# newer versions of virtualenv inherit setuptools/pip/wheel versions +# from current env when creating a child env pip install -U virtualenv root="$(mktemp -d -t le.$version.XXX)"