From e8387b10c4aec3a705ef1f634b0bd3a80dd41a2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 16 Jul 2015 16:57:47 -0700 Subject: [PATCH] Finished basic Apache 2.4 Proxy --- tests/compatibility/Dockerfile | 2 +- .../configurators/apache/apache24.py | 5 ++- .../configurators/apache/common.py | 32 +++++++++---------- tests/compatibility/configurators/common.py | 26 +++++++-------- tests/compatibility/plugin_test.py | 32 +++++++++++-------- tests/compatibility/util.py | 3 +- 6 files changed, 55 insertions(+), 45 deletions(-) diff --git a/tests/compatibility/Dockerfile b/tests/compatibility/Dockerfile index 6637b9e14..446234816 100644 --- a/tests/compatibility/Dockerfile +++ b/tests/compatibility/Dockerfile @@ -10,7 +10,7 @@ RUN mkdir /var/run/apache2 && \ ENV APACHE_CONFDIR=/tmp/apache2 \ APACHE_RUN_USER=daemon \ APACHE_RUN_GROUP=daemon \ - APACHE_PID_FILE=/var/run/apache2/apache2.pid \ + APACHE_PID_FILE=/usr/local/apache2/logs/httpd.pid \ APACHE_RUN_DIR=/var/run/apache2 \ APACHE_LOCK_DIR=/var/lock \ APACHE_LOG_DIR=/usr/local/apache2/logs diff --git a/tests/compatibility/configurators/apache/apache24.py b/tests/compatibility/configurators/apache/apache24.py index 1d2d135b2..26e0686c0 100644 --- a/tests/compatibility/configurators/apache/apache24.py +++ b/tests/compatibility/configurators/apache/apache24.py @@ -44,13 +44,16 @@ class Proxy(apache_common.Proxy): if self.version[1] != 4: raise errors.Error("Apache version not 2.4") + self.execute_in_docker( + "bash -c 'export APACHE_CONFDIR={0}'".format(self.config_file)) + with open(self.test_conf, "a") as f: for module in self.modules: if module not in STATIC_MODULES: if module in INSTALLED_MODULES: f.write( "LoadModule {0}_module /usr/local/apache2/modules/" - "mod_{0}\n".format(module)) + "mod_{0}.so\n".format(module)) else: raise errors.Error( "Unsupported module {0}".format(module)) diff --git a/tests/compatibility/configurators/apache/common.py b/tests/compatibility/configurators/apache/common.py index fb5e9c161..cbdc6b845 100644 --- a/tests/compatibility/configurators/apache/common.py +++ b/tests/compatibility/configurators/apache/common.py @@ -26,8 +26,7 @@ class Proxy(configurators_common.Proxy): def __init__(self, args): """Initializes the plugin with the given command line args""" super(Proxy, self).__init__(args) - self.le_config = util.create_le_config(self.temp_dir) - self.le_config["apache_le_vhost_ext"] = "-le-ssl.conf" + self.le_config.apache_le_vhost_ext = "-le-ssl.conf" self._patch = mock.patch('letsencrypt_apache.configurator.subprocess') self._mock = self._patch.start() @@ -35,7 +34,7 @@ class Proxy(configurators_common.Proxy): self._mock.Popen = self.popen_in_docker self.server_root = self.modules = self.version = self.test_conf = None - self._config_file = self._apache_configurator = self._names = None + self.config_file = self._apache_configurator = self._names = None def __getattr__(self, name): """Wraps the Apache Configurator methods""" @@ -48,16 +47,15 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" config = self.get_next_config() - logger.debug("Loading configuration: %s", config) + logger.info("Loading configuration: %s", config) self._parse_config(config) - - self._prepare_configurator() self.preprocess_config() + self._prepare_configurator() try: self.check_call_in_docker( "apachectl -d {0} -f {1} -k restart".format( - self.server_root, self._config_file)) + self.server_root, self.config_file)) except errors.Error: raise errors.Error( "Apache failed to load {0} before tests started".format( @@ -69,7 +67,7 @@ class Proxy(configurators_common.Proxy): self.test_conf = os.path.join(self.server_root, "test.conf") open(self.test_conf, "w").close() subprocess.check_call( - ["sed", "-i", "1iInclude test.conf", self._config_file]) + ["sed", "-i", "1iInclude test.conf", self.config_file]) find = subprocess.Popen( ["find", self.server_root, "-type", "f"], stdout=subprocess.PIPE) @@ -88,15 +86,17 @@ class Proxy(configurators_common.Proxy): self._names = _get_names(config) with open(os.path.join(config, "config_file")) as f: - self._config_file = os.path.join(self.server_root, f.readline()) + config_file_base = f.readline().rstrip() + + self.config_file = os.path.join(self.server_root, config_file_base) def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - self.le_config["apache_ctl"] = "apachectl -d {0} -f {1}".format( - self.server_root, self._config_file) - self.le_config["a2enmod.sh"] = "a2enmod.sh" - self.le_config["apache_init_script"] = self.le_config["apache_ctl"] - self.le_config["apache_init_script"] += " -k" + self.le_config.apache_server_root = self.server_root + self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format( + self.server_root, self.config_file) + self.le_config.apache_enmod = "a2enmod.sh" + self.le_config.apache_init = self.le_config.apache_ctl + " -k" self._apache_configurator = configurator.ApacheConfigurator( config=configuration.NamespaceConfig(self.le_config), @@ -120,7 +120,7 @@ def _get_server_root(config): if len(subdirs) != 1: errors.Error("Malformed configuration directiory {0}".format(config)) - return subdirs[0] + return os.path.join(config, subdirs[0].rstrip()) def _get_names(config): @@ -154,7 +154,7 @@ def _get_modules(config): if line[0].isspace(): words = line.split() # Modules redundantly end in "_module" which we can discard - modules.append(words[-7]) + modules.append(words[0][:-7]) return modules diff --git a/tests/compatibility/configurators/common.py b/tests/compatibility/configurators/common.py index 4953154b9..b78046d62 100644 --- a/tests/compatibility/configurators/common.py +++ b/tests/compatibility/configurators/common.py @@ -1,8 +1,8 @@ """Provides a common base for configurator proxies""" import logging -import multiprocessing import os import tempfile +import threading import docker @@ -34,15 +34,16 @@ class Proxy(object): def __init__(self, args): """Initializes the plugin with the given command line args""" - self.temp_dir = tempfile.mkdtemp() - self.config_dir = util.extract_configs(args.configs, self.temp_dir) + temp_dir = tempfile.mkdtemp() + self.le_config = util.create_le_config(temp_dir) + self.config_dir = util.extract_configs(args.configs, temp_dir) self._configs = os.listdir(self.config_dir) self.args = args self._docker_client = docker.Client( base_url=self.args.docker_url, version="auto") self.http_port, self.https_port = util.get_two_free_ports() - self._container_id = self._log_process = None + self._container_id = self._log_thread = None def has_more_configs(self): """Returns true if there are more configs to test""" @@ -51,13 +52,13 @@ class Proxy(object): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" self._docker_client.stop(self._container_id) - self._log_process.join() + self._log_thread.join() if not self.args.no_remove: self._docker_client.remove_container(self._container_id) def get_next_config(self): """Returns the next config directory to be tested""" - return self._configs.pop() + return os.path.join(self.config_dir, self._configs.pop()) def start_docker(self, image_name): """Creates and runs a Docker container with the specified image""" @@ -78,14 +79,13 @@ class Proxy(object): self._container_id = container["Id"] self._docker_client.start(self._container_id) - self._log_process = multiprocessing.Process( - target=self._start_log_thread) - self._log_process.start() + self._log_thread = threading.Thread(target=self._start_log_thread) + self._log_thread.start() def _start_log_thread(self): client = docker.Client(base_url=self.args.docker_url, version="auto") for line in client.logs(self._container_id, stream=True): - logger.debug(line) + logger.info(line.rstrip()) def check_call_in_docker( self, command, *args, **kwargs): # pylint: disable=unused-argument @@ -118,12 +118,12 @@ class Proxy(object): if isinstance(command, list): command = " ".join(command) - returncode, output = self._execute_in_docker(command) + returncode, output = self.execute_in_docker(command) return SimplePopen(returncode, output) - def _execute_in_docker(self, command): + def execute_in_docker(self, command): """Executes command inside the running docker image""" - logger.debug("Executing '%s'", command) + logger.info("Executing '%s'", command) exec_id = self._docker_client.exec_create(self._container_id, command) output = self._docker_client.exec_start(exec_id) returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"] diff --git a/tests/compatibility/plugin_test.py b/tests/compatibility/plugin_test.py index 7e1a3cb50..04cb9da11 100644 --- a/tests/compatibility/plugin_test.py +++ b/tests/compatibility/plugin_test.py @@ -1,13 +1,14 @@ """Tests Let's Encrypt plugins against different server configurations.""" import argparse import logging -import os +from tests.compatibility import errors from tests.compatibility.configurators.apache import apache24 DESCRIPTION = """ Tests Let's Encrypt plugins against different server configuratons. It is -assumed that Docker is already installed. +assumed that Docker is already installed. If no test types is specified, all +tests that the plugin supports are performed. """ @@ -20,7 +21,9 @@ logger = logging.getLogger(__name__) def get_args(): """Returns parsed command line arguments.""" - parser = argparse.ArgumentParser(description=DESCRIPTION) + parser = argparse.ArgumentParser( + description=DESCRIPTION, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) group = parser.add_argument_group("general") group.add_argument( @@ -30,20 +33,19 @@ def get_args(): "-p", "--plugin", default="apache", help="the plugin to be tested") group.add_argument( "-a", "--auth", action="store_true", - help="tests the plugin as an authenticator") + help="tests the challenges the plugin supports") group.add_argument( "-i", "--install", action="store_true", help="tests the plugin as an installer") group.add_argument( - "-r", "--redirect", action="store_true", help="tests the plugin's " - "ability to redirect HTTP to HTTPS (implicitly includes installer " - "tests)") + "-e", "--enhance", action="store_true", help="tests the enhancements " + "the plugin supports (implicitly includes installer tests)") for plugin in PLUGINS.itervalues(): plugin.add_parser_arguments(parser) args = parser.parse_args() - if args.redirect: + if args.enhance: args.install = True elif not (args.auth or args.install): args.auth = args.install = args.redirect = True @@ -53,12 +55,10 @@ def get_args(): def setup_logging(): """Prepares logging for the program""" - fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter(fmt)) root_logger = logging.getLogger() - root_logger.setLevel(logging.DEBUG) + root_logger.setLevel(logging.INFO) root_logger.addHandler(handler) @@ -69,8 +69,14 @@ def main(): if args.plugin not in PLUGINS: raise errors.Error("Unknown plugin {0}".format(args.plugin)) - plugin = PLUGINS[args.plugin](args) - plugin.cleanup_from_tests() + plugin = None + try: + plugin = PLUGINS[args.plugin](args) + plugin.load_config() + assert plugin.get_all_names() == plugin.get_test_domain_names() + finally: + if plugin: + plugin.cleanup_from_tests() if __name__ == "__main__": diff --git a/tests/compatibility/util.py b/tests/compatibility/util.py index e6850e2e0..c217bdfa9 100644 --- a/tests/compatibility/util.py +++ b/tests/compatibility/util.py @@ -1,4 +1,5 @@ """Utility functions for Let"s Encrypt plugin tests.""" +import argparse import copy import contextlib import os @@ -26,7 +27,7 @@ def create_le_config(parent_dir): os.mkdir(config["work_dir"]) os.mkdir(config["logs_dir"]) - return config + return argparse.Namespace(**config) # pylint: disable=star-args def extract_configs(configs, parent_dir): """Extracts configs to a new dir under parent_dir and returns it"""