From 74ce332b5a4a0747cad56afc4cbe72fa519ce31f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sat, 27 Jun 2015 19:38:00 +0000 Subject: [PATCH 1/4] Manual SimpleHTTP integration tests. --- letsencrypt/auth_handler.py | 7 +- letsencrypt/plugins/manual.py | 77 ++++++++++++++++--- letsencrypt/plugins/manual_test.py | 3 +- .../plugins/standalone/authenticator.py | 4 + tests/boulder-integration.sh | 2 + tests/integration/_common.sh | 3 +- 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 2929166c2..3d8275901 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -156,7 +156,7 @@ class AuthHandler(object): active_achalls = [] for achall, resp in itertools.izip(achalls, resps): # Don't send challenges for None and False authenticator responses - if resp: + if resp is not None and resp: self.network.answer_challenge(achall.challb, resp) # TODO: answer_challenge returns challr, with URI, # that can be used in _find_updated_challr @@ -166,6 +166,11 @@ class AuthHandler(object): chall_update[achall.domain].append(achall) else: chall_update[achall.domain] = [achall] + else: # resp is None or not resp + # XXX: make sure that achalls corresponding to None or + # False returned from Authenticator are removed from + # the queue and thus avoid infinite loop + active_achalls.append(achall) return active_achalls diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 700759194..771833d6f 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -1,6 +1,12 @@ """Manual plugin.""" import os +import logging +import shutil +import signal +import subprocess import sys +import tempfile +import time import zope.component import zope.interface @@ -8,10 +14,14 @@ import zope.interface from acme import challenges from acme import jose +from letsencrypt import errors from letsencrypt import interfaces from letsencrypt.plugins import common +logger = logging.getLogger(__name__) + + class ManualAuthenticator(common.Plugin): """Manual Authenticator. @@ -43,8 +53,8 @@ command on the target server (as root): # anything recursively under the cwd HTTP_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: python -c "import BaseHTTPServer, SimpleHTTPServer; \\ @@ -55,8 +65,8 @@ s.serve_forever()" """ # https://www.piware.de/2011/01/creating-an-https-server-in-python/ HTTPS_TEMPLATE = """\ -mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH} -cd /tmp/letsencrypt/public_html +mkdir -p {root}/public_html/{response.URI_ROOT_PATH} +cd {root}/public_html echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path} # run only once per server: openssl req -new -newkey rsa:4096 -subj "/" -days 1 -nodes -x509 -keyout ../key.pem -out ../cert.pem @@ -77,6 +87,14 @@ s.serve_forever()" """ super(ManualAuthenticator, self).__init__(*args, **kwargs) self.template = (self.HTTP_TEMPLATE if self.config.no_simple_http_tls else self.HTTPS_TEMPLATE) + self._root = (tempfile.mkdtemp() if self.conf("test-mode") + else "/tmp/letsencrypt") + + @classmethod + def add_parser_arguments(cls, add): + add("test-mode", action="store_true", + help="Test mode. Executes the manual command in subprocess. " + "Requires openssl to be installed unless --no-simple-http-tls.") def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -110,17 +128,44 @@ binary for temporary key/certificate generation.""".replace("\n", "") tls=(not self.config.no_simple_http_tls)) assert response.good_path # is encoded os.urandom(18) good? - self._notify_and_wait(self.MESSAGE_TEMPLATE.format( - achall=achall, response=response, uri=response.uri(achall.domain), - ct=response.CONTENT_TYPE, command=self.template.format( - achall=achall, response=response, ct=response.CONTENT_TYPE, - port=(response.port if self.config.simple_http_port is None - else self.config.simple_http_port)))) + command = self.template.format( + root=self._root, achall=achall, response=response, + ct=response.CONTENT_TYPE, port=( + response.port if self.config.simple_http_port is None + else self.config.simple_http_port)) + if self.conf("test-mode"): + logger.debug("Test mode. Executing the manual command: %s", command) + try: + # pylint: disable=attribute-defined-outside-init + self._httpd = subprocess.Popen( + command, + # don't care about setting stdout and stderr, + # we're in test mode anyway + shell=True, + # "preexec_fn" is UNIX specific, but so is "command" + preexec_fn=os.setsid) + except OSError as error: # ValueError should not happen! + logging.debug( + "Couldn't execute manual command", error, exc_info=True) + return False + logger.debug("Manual command running as PID %s.", self._httpd.pid) + # give it some time to bootstrap, before we try to verify + # (cert generation in case of simpleHttpS might take time) + time.sleep(4) # XXX + if self._httpd.poll(): + raise errors.Error("Couldn't execute manual command") + else: + self._notify_and_wait(self.MESSAGE_TEMPLATE.format( + achall=achall, response=response, + uri=response.uri(achall.domain), ct=response.CONTENT_TYPE, + command=command)) if response.simple_verify( achall.challb, achall.domain, self.config.simple_http_port): return response else: + if self.conf("test-mode") and self._httpd.poll(): + return False return None def _notify_and_wait(self, message): # pylint: disable=no-self-use @@ -130,5 +175,13 @@ binary for temporary key/certificate generation.""".replace("\n", "") sys.stdout.write(message) raw_input("Press ENTER to continue") - def cleanup(self, achalls): # pylint: disable=missing-docstring,no-self-use - pass # pragma: no cover + def cleanup(self, achalls): + # pylint: disable=missing-docstring,no-self-use,unused-argument + if self.conf("test-mode"): + if self._httpd.poll() is None: + logger.debug("Terminating manual command process") + os.killpg(self._httpd.pid, signal.SIGTERM) + else: + logger.debug("Manual command process already terminated " + "with %s code", self._httpd.returncode) + shutil.rmtree(self._root) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index a533bcc75..1360ebf44 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -15,7 +15,8 @@ class ManualAuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.manual import ManualAuthenticator self.config = mock.MagicMock( - no_simple_http_tls=True, simple_http_port=4430) + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=False) self.auth = ManualAuthenticator(config=self.config, name="manual") self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)] diff --git a/letsencrypt/plugins/standalone/authenticator.py b/letsencrypt/plugins/standalone/authenticator.py index 971f90266..d55d70aa0 100644 --- a/letsencrypt/plugins/standalone/authenticator.py +++ b/letsencrypt/plugins/standalone/authenticator.py @@ -197,6 +197,10 @@ class StandaloneAuthenticator(common.Plugin): """ signal.signal(signal.SIGINT, self.subproc_signal_handler) self.sock = socket.socket() + # SO_REUSEADDR flag tells the kernel to reuse a local socket + # in TIME_WAIT state, without waiting for its natural timeout + # to expire. + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: self.sock.bind(("0.0.0.0", port)) except socket.error, error: diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 06a1d8aa9..975d030da 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -23,6 +23,8 @@ common() { common --domains le1.wtf auth common --domains le2.wtf run +common -a manual -d le.wtf auth +common -a manual -d le.wtf --no-simple-http-tls auth export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 0f26d3815..de16a939a 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -10,11 +10,12 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { - # first three flags required, rest is handy defaults letsencrypt \ --server "${SERVER:-http://localhost:4000/acme/new-reg}" \ --no-verify-ssl \ --dvsni-port 5001 \ + --simple-http-port 5001 \ + --manual-test-mode \ $store_flags \ --text \ --agree-eula \ From 9e2682a025f32dabe8fe7256141785a698d0c257 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 19 Jul 2015 11:02:17 +0000 Subject: [PATCH 2/4] 100% coverage for manual test mode and related code. --- letsencrypt/auth_handler.py | 12 ++++---- letsencrypt/plugins/manual.py | 11 ++++--- letsencrypt/plugins/manual_test.py | 48 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index e1a964f77..5969dc36f 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -153,22 +153,22 @@ class AuthHandler(object): """ active_achalls = [] for achall, resp in itertools.izip(achalls, resps): + # XXX: make sure that all achalls, including those + # corresponding to None or False returned from + # Authenticator are removed from the queue and thus avoid + # infinite loop + active_achalls.append(achall) + # Don't send challenges for None and False authenticator responses if resp is not None and resp: self.acme.answer_challenge(achall.challb, resp) # TODO: answer_challenge returns challr, with URI, # that can be used in _find_updated_challr # comparisons... - active_achalls.append(achall) if achall.domain in chall_update: chall_update[achall.domain].append(achall) else: chall_update[achall.domain] = [achall] - else: # resp is None or not resp - # XXX: make sure that achalls corresponding to None or - # False returned from Authenticator are removed from - # the queue and thus avoid infinite loop - active_achalls.append(achall) return active_achalls diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 771833d6f..b5ddd2140 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -89,6 +89,7 @@ s.serve_forever()" """ else self.HTTPS_TEMPLATE) self._root = (tempfile.mkdtemp() if self.conf("test-mode") else "/tmp/letsencrypt") + self._httpd = None @classmethod def add_parser_arguments(cls, add): @@ -136,7 +137,6 @@ binary for temporary key/certificate generation.""".replace("\n", "") if self.conf("test-mode"): logger.debug("Test mode. Executing the manual command: %s", command) try: - # pylint: disable=attribute-defined-outside-init self._httpd = subprocess.Popen( command, # don't care about setting stdout and stderr, @@ -146,13 +146,13 @@ binary for temporary key/certificate generation.""".replace("\n", "") preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! logging.debug( - "Couldn't execute manual command", error, exc_info=True) + "Couldn't execute manual command: %s", error, exc_info=True) return False logger.debug("Manual command running as PID %s.", self._httpd.pid) # give it some time to bootstrap, before we try to verify # (cert generation in case of simpleHttpS might take time) time.sleep(4) # XXX - if self._httpd.poll(): + if self._httpd.poll() is not None: raise errors.Error("Couldn't execute manual command") else: self._notify_and_wait(self.MESSAGE_TEMPLATE.format( @@ -164,7 +164,8 @@ binary for temporary key/certificate generation.""".replace("\n", "") achall.challb, achall.domain, self.config.simple_http_port): return response else: - if self.conf("test-mode") and self._httpd.poll(): + if self.conf("test-mode") and self._httpd.poll() is not None: + # simply verify cause command failure... return False return None @@ -178,6 +179,8 @@ binary for temporary key/certificate generation.""".replace("\n", "") def cleanup(self, achalls): # pylint: disable=missing-docstring,no-self-use,unused-argument if self.conf("test-mode"): + assert self._httpd is not None, ( + "cleanup() must be called after perform()") if self._httpd.poll() is None: logger.debug("Terminating manual command process") os.killpg(self._httpd.pid, signal.SIGTERM) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index e57864b9d..fe95a00f0 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -1,4 +1,5 @@ """Tests for letsencrypt.plugins.manual.""" +import signal import unittest import mock @@ -6,6 +7,8 @@ import mock from acme import challenges from letsencrypt import achallenges +from letsencrypt import errors + from letsencrypt.tests import acme_util @@ -21,6 +24,12 @@ class ManualAuthenticatorTest(unittest.TestCase): self.achalls = [achallenges.SimpleHTTP( challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)] + config_test_mode = mock.MagicMock( + no_simple_http_tls=True, simple_http_port=4430, + manual_test_mode=True) + self.auth_test_mode = ManualAuthenticator( + config=config_test_mode, name="manual") + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), str)) @@ -52,6 +61,45 @@ class ManualAuthenticatorTest(unittest.TestCase): mock_verify.return_value = False self.assertEqual([None], self.auth.perform(self.achalls)) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_oserror(self, mock_popen): + mock_popen.side_effect = OSError + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_command_run_failure( + self, mock_popen, unused_mock_sleep): + mock_popen.poll.return_value = 10 + mock_popen.return_value.pid = 1234 + self.assertRaises( + errors.Error, self.auth_test_mode.perform, self.achalls) + + @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True) + @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify", + autospec=True) + @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True) + def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep): + mock_popen.return_value.poll.side_effect = [None, 10] + mock_popen.return_value.pid = 1234 + mock_verify.return_value = False + self.assertEqual([False], self.auth_test_mode.perform(self.achalls)) + self.assertEqual(1, mock_sleep.call_count) + + def test_cleanup_test_mode_already_terminated(self): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock() + httpd.poll.return_value = 0 + self.auth_test_mode.cleanup(self.achalls) + + @mock.patch("letsencrypt.plugins.manual.os.killpg", autospec=True) + def test_cleanup_test_mode_kills_still_running(self, mock_killpg): + # pylint: disable=protected-access + self.auth_test_mode._httpd = httpd = mock.Mock(pid=1234) + httpd.poll.return_value = None + self.auth_test_mode.cleanup(self.achalls) + mock_killpg.assert_called_once_with(1234, signal.SIGTERM) + if __name__ == "__main__": unittest.main() # pragma: no cover From 82147f1f5e3373ed9e623cbee618339ba8a3f2a6 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Sun, 19 Jul 2015 11:22:57 +0000 Subject: [PATCH 3/4] Travis: add le.wtf to /etc/hosts. --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7cccd20c9..587ac6ccc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,11 @@ env: - TOXENV=lint - TOXENV=cover +# make sure simplehttp simple verification works (custom /etc/hosts) +addons: + hosts: + - le.wtf + install: "travis_retry pip install tox coveralls" before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh' script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))' From ab097d128b502eaa44a8d77d2ff9d5a59dcb5126 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Fri, 24 Jul 2015 06:41:56 +0000 Subject: [PATCH 4/4] Fix logging -> logger typo. --- letsencrypt/plugins/manual.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index b5ddd2140..83f2c0f70 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -145,7 +145,7 @@ binary for temporary key/certificate generation.""".replace("\n", "") # "preexec_fn" is UNIX specific, but so is "command" preexec_fn=os.setsid) except OSError as error: # ValueError should not happen! - logging.debug( + logger.debug( "Couldn't execute manual command: %s", error, exc_info=True) return False logger.debug("Manual command running as PID %s.", self._httpd.pid)