diff --git a/tests/compatibility/Dockerfile b/tests/compatibility/configurators/apache/Dockerfile similarity index 70% rename from tests/compatibility/Dockerfile rename to tests/compatibility/configurators/apache/Dockerfile index 446234816..8cde44ea6 100644 --- a/tests/compatibility/Dockerfile +++ b/tests/compatibility/configurators/apache/Dockerfile @@ -4,17 +4,15 @@ FROM httpd MAINTAINER Brad Warren -RUN mkdir /var/run/apache2 && \ - ln -s /usr/local/apache2/conf/mime.types /etc/mime.types +RUN mkdir /var/run/apache2 -ENV APACHE_CONFDIR=/tmp/apache2 \ - APACHE_RUN_USER=daemon \ +ENV APACHE_RUN_USER=daemon \ APACHE_RUN_GROUP=daemon \ APACHE_PID_FILE=/usr/local/apache2/logs/httpd.pid \ APACHE_RUN_DIR=/var/run/apache2 \ APACHE_LOCK_DIR=/var/lock \ APACHE_LOG_DIR=/usr/local/apache2/logs -COPY tests/compatibility/a2enmod.sh /usr/local/bin/ +COPY tests/compatibility/configurators/apache/a2enmod.sh /usr/local/bin/ CMD [ "httpd-foreground" ] diff --git a/tests/compatibility/a2enmod.sh b/tests/compatibility/configurators/apache/a2enmod.sh similarity index 69% rename from tests/compatibility/a2enmod.sh rename to tests/compatibility/configurators/apache/a2enmod.sh index 364c038c0..c22adb1fb 100755 --- a/tests/compatibility/a2enmod.sh +++ b/tests/compatibility/configurators/apache/a2enmod.sh @@ -1,9 +1,13 @@ #!/bin/bash -# An extremely simplified version of 'a2enmod' for the httpd docker image +# An extremely simplified (and hacky) version of 'a2enmod' for the httpd +# docker image. First argument is server_root and second argument is the module +# to be enabled. + +APACHE_CONFDIR=$1 enable () { echo "LoadModule "$1"_module /usr/local/apache2/modules/mod_"$1".so" >> \ - $APACHE_CONFDIR"/tests.conf" + $APACHE_CONFDIR"/test.conf" available_base="/mods-available/"$1".conf" available_conf=$APACHE_CONFDIR$available_base enabled_dir=$APACHE_CONFDIR"/mods-enabled" @@ -14,14 +18,14 @@ enable () { fi } -if [ $1 == "ssl" ] +if [ $2 == "ssl" ] then # Enables ssl and all its dependencies enable "setenvif" enable "mime" enable "socache_shmcb" enable "ssl" -elif [ $1 == "rewrite" ] +elif [ $2 == "rewrite" ] then enable "rewrite"; else diff --git a/tests/compatibility/configurators/apache/apache24.py b/tests/compatibility/configurators/apache/apache24.py index 26e0686c0..48fa6714d 100644 --- a/tests/compatibility/configurators/apache/apache24.py +++ b/tests/compatibility/configurators/apache/apache24.py @@ -7,10 +7,10 @@ from tests.compatibility.configurators.apache import common as apache_common # config uses mod_heartbeat or mod_heartmonitor (which aren't installed and # therefore the config won't be loaded), I believe this isn't a problem # http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html -STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog",]) +STATIC_MODULES = {"core", "so", "http", "mpm_event", "watchdog",} -INSTALLED_MODULES = set([ +INSTALLED_MODULES = { "log_config", "logio", "version", "unixd", "access_compat", "actions", "alias", "allowmethods", "auth_basic", "auth_digest", "auth_form", "authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file", @@ -27,7 +27,7 @@ INSTALLED_MODULES = set([ "session_cookie", "session_crypto", "session_dbd", "setenvif", "slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb", "speling", "ssl", "status", "substitute", "unique_id", "userdir", - "vhost_alias",]) + "vhost_alias",} class Proxy(apache_common.Proxy): @@ -38,15 +38,12 @@ class Proxy(apache_common.Proxy): super(Proxy, self).__init__(args) self.start_docker("bradmw/apache2.4") - def preprocess_config(self): + def preprocess_config(self, server_root): """Prepares the configuration for use in the Docker""" - super(Proxy, self).preprocess_config() + super(Proxy, self).preprocess_config(server_root) if self.version[1] != 4: raise errors.Error("Apache version not 2.4") - self.execute_in_docker( - "bash -c 'export APACHE_CONFDIR={0}'".format(self.config_file)) - with open(self.test_conf, "a") as f: for module in self.modules: if module not in STATIC_MODULES: diff --git a/tests/compatibility/configurators/apache/common.py b/tests/compatibility/configurators/apache/common.py index cbdc6b845..cf918fa9c 100644 --- a/tests/compatibility/configurators/apache/common.py +++ b/tests/compatibility/configurators/apache/common.py @@ -1,14 +1,15 @@ """Provides a common base for Apache proxies""" -import logging import re import os import subprocess import mock +import zope.interface from letsencrypt import configuration from letsencrypt_apache import configurator from tests.compatibility import errors +from tests.compatibility import interfaces from tests.compatibility import util from tests.compatibility.configurators import common as configurators_common @@ -16,25 +17,26 @@ from tests.compatibility.configurators import common as configurators_common APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) -logger = logging.getLogger(__name__) - - class Proxy(configurators_common.Proxy): # pylint: disable=too-many-instance-attributes """A common base for Apache test configurators""" + zope.interface.implements(interfaces.IConfiguratorProxy) + def __init__(self, args): """Initializes the plugin with the given command line args""" super(Proxy, self).__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" self._patch = mock.patch('letsencrypt_apache.configurator.subprocess') - self._mock = self._patch.start() - self._mock.check_call = self.check_call_in_docker - self._mock.Popen = self.popen_in_docker + subprocess_mock = self._patch.start() + subprocess_mock.check_call = self.check_call_in_docker + subprocess_mock.Popen = self.popen_in_docker + self.modules = self.version = self.test_conf = None + self._apache_configurator = None self.server_root = self.modules = self.version = self.test_conf = None - self.config_file = self._apache_configurator = self._names = None + self._apache_configurator = self._all_names = self._test_names = None def __getattr__(self, name): """Wraps the Apache Configurator methods""" @@ -46,56 +48,51 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = self.get_next_config() - logger.info("Loading configuration: %s", config) - self._parse_config(config) - self.preprocess_config() - self._prepare_configurator() + config = super(Proxy, self).load_config() + self.modules = _get_modules(config) + self.version = _get_version(config) + self._all_names, self._test_names = _get_names(config) + + server_root = _get_server_root(config) + with open(os.path.join(config, "config_file")) as f: + config_file = os.path.join(server_root, f.readline().rstrip()) + self.test_conf = _create_test_conf(server_root, config_file) + + self.preprocess_config(server_root) + self._prepare_configurator(server_root, config_file) try: self.check_call_in_docker( "apachectl -d {0} -f {1} -k restart".format( - self.server_root, self.config_file)) + server_root, config_file)) except errors.Error: raise errors.Error( "Apache failed to load {0} before tests started".format( config)) - def preprocess_config(self): - # pylint: disable=anomalous-backslash-in-string + return config + + def preprocess_config(self, server_root): + # pylint: disable=anomalous-backslash-in-string, no-self-use """Prepares the configuration for use in the Docker""" - self.test_conf = os.path.join(self.server_root, "test.conf") - open(self.test_conf, "w").close() - subprocess.check_call( - ["sed", "-i", "1iInclude test.conf", self.config_file]) find = subprocess.Popen( - ["find", self.server_root, "-type", "f"], + ["find", server_root, "-type", "f"], stdout=subprocess.PIPE) subprocess.check_call([ - "xargs", "sed", "-e", - "s/DocumentRoot.*/DocumentRoot \/usr\/local\/apache2\/htdocs/I", - "-e", - "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I", + "xargs", "sed", "-e", "s/DocumentRoot.*/" + "DocumentRoot \/usr\/local\/apache2\/htdocs/I", + "-e", "s/SSLPassPhraseDialog.*/" + "SSLPassPhraseDialog builtin/I", + "-e", "s/TypesConfig.*/" + "TypesConfig \/usr\/local\/apache2\/conf\/mime.types/I", "-i"], stdin=find.stdout) - def _parse_config(self, config): - """Parses extra information in server config directory""" - self.server_root = _get_server_root(config) - self.modules = _get_modules(config) - self.version = _get_version(config) - self._names = _get_names(config) - - with open(os.path.join(config, "config_file")) as f: - config_file_base = f.readline().rstrip() - - self.config_file = os.path.join(self.server_root, config_file_base) - - def _prepare_configurator(self): + def _prepare_configurator(self, server_root, config_file): """Prepares the Apache plugin for testing""" - self.le_config.apache_server_root = self.server_root + self.le_config.apache_server_root = server_root self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format( - self.server_root, self.config_file) - self.le_config.apache_enmod = "a2enmod.sh" + server_root, config_file) + self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root) self.le_config.apache_init = self.le_config.apache_ctl + " -k" self._apache_configurator = configurator.ApacheConfigurator( @@ -103,13 +100,34 @@ class Proxy(configurators_common.Proxy): name="apache") self._apache_configurator.prepare() - def get_test_domain_names(self): - """Returns a list of domain names to test against the plugin""" - if self._names: - return self._names + def cleanup_from_tests(self): + """Performs any necessary cleanup from running plugin tests""" + super(Proxy, self).cleanup_from_tests() + self._patch.stop() + + def get_testable_domain_names(self): + """Returns the set of domain names that can be tested against""" + if self._test_names: + return self._test_names else: raise errors.Error("No configuration file loaded") + def get_all_names_answer(self): + """Returns the set of domain names that the plugin should find""" + if self._all_names: + return self._all_names + else: + raise errors.Error("No configuration file loaded") + + +def _create_test_conf(server_root, apache_config): + """Creates a test config file and adds it to the Apache config""" + test_conf = os.path.join(server_root, "test.conf") + open(test_conf, "w").close() + subprocess.check_call( + ["sed", "-i", "1iInclude test.conf", apache_config]) + return test_conf + def _get_server_root(config): """Returns the server root directory in config""" @@ -124,25 +142,28 @@ def _get_server_root(config): def _get_names(config): - """Returns domains names for config""" - names = set() + """Returns all and testable domain names in config""" + all_names = set() + non_ip_names = set() with open(os.path.join(config, "vhosts")) as f: for line in f: # If parsing a specific vhost if line[0].isspace(): words = line.split() if words[0] == "alias": - names.add(words[1]) + all_names.add(words[1]) + non_ip_names.add(words[1]) # If for port 80 and not IP vhost elif words[1] == "80" and not util.IP_REGEX.match(words[3]): - names.add(words[3]) + all_names.add(words[3]) + non_ip_names.add(words[3]) elif "NameVirtualHost" not in line: words = line.split() - if ((words[0].endswith("*") or words[0].endswith("80")) and - util.IP_REGEX.match(words[1])): - names.add(words[1]) - - return names + if (words[0].endswith("*") or words[0].endswith("80") and + not util.IP_REGEX.match(words[1]) and + words[1].find(".") != -1): + all_names.add(words[1]) + return all_names, non_ip_names def _get_modules(config): diff --git a/tests/compatibility/configurators/common.py b/tests/compatibility/configurators/common.py index b78046d62..689f1d4a4 100644 --- a/tests/compatibility/configurators/common.py +++ b/tests/compatibility/configurators/common.py @@ -36,8 +36,8 @@ class Proxy(object): """Initializes the plugin with the given command line args""" temp_dir = tempfile.mkdtemp() self.le_config = util.create_le_config(temp_dir) - self.config_dir = util.extract_configs(args.configs, temp_dir) - self._configs = os.listdir(self.config_dir) + self._config_dir = util.extract_configs(args.configs, temp_dir) + self._configs = os.listdir(self._config_dir) self.args = args self._docker_client = docker.Client( @@ -56,9 +56,9 @@ class Proxy(object): if not self.args.no_remove: self._docker_client.remove_container(self._container_id) - def get_next_config(self): + def load_config(self): """Returns the next config directory to be tested""" - return os.path.join(self.config_dir, self._configs.pop()) + return os.path.join(self._config_dir, self._configs.pop()) def start_docker(self, image_name): """Creates and runs a Docker container with the specified image""" @@ -67,12 +67,12 @@ class Proxy(object): host_config = docker.utils.create_host_config( binds={ - self.config_dir : {"bind" : self.config_dir, "mode" : "rw"}}, + self._config_dir : {"bind" : self._config_dir, "mode" : "rw"}}, port_bindings={ 80 : ("127.0.0.1", self.http_port), 443 : ("127.0.0.1", self.https_port)},) container = self._docker_client.create_container( - image_name, ports=[80, 443], volumes=self.config_dir, + image_name, ports=[80, 443], volumes=self._config_dir, host_config=host_config) if container["Warnings"]: logger.warning(container["Warnings"]) @@ -85,7 +85,7 @@ class Proxy(object): def _start_log_thread(self): client = docker.Client(base_url=self.args.docker_url, version="auto") for line in client.logs(self._container_id, stream=True): - logger.info(line.rstrip()) + logger.debug(line.rstrip()) def check_call_in_docker( self, command, *args, **kwargs): # pylint: disable=unused-argument diff --git a/tests/compatibility/interfaces.py b/tests/compatibility/interfaces.py index 4e8c68675..a4925014f 100644 --- a/tests/compatibility/interfaces.py +++ b/tests/compatibility/interfaces.py @@ -6,8 +6,7 @@ import letsencrypt.interfaces class IPluginProxy(zope.interface.Interface): """Wraps a Let's Encrypt plugin""" - @classmethod - def add_parser_arguments(cls, parser): + def add_parser_arguments(cls, parser): # pylint: disable=no-self-argument """Adds command line arguments needed by the parser""" def __init__(self, args): @@ -24,7 +23,7 @@ class IPluginProxy(zope.interface.Interface): """Returns True if there are more configs to test""" def load_config(self): - """Loads the next configuration for the plugin to test""" + """Loads the next config and returns its name""" class IConfiguratorBaseProxy(IPluginProxy): @@ -35,8 +34,8 @@ class IConfiguratorBaseProxy(IPluginProxy): https_port = zope.interface.Attribute( "The port to connect to on localhost for HTTPS traffic") - def get_test_domain_names(self): - """Returns a list of domain names to test against the plugin""" + def get_testable_domain_names(self): + """Returns the domain names that can be used in testing""" class IAuthenticatorProxy( @@ -48,6 +47,9 @@ class IInstallerProxy( IConfiguratorBaseProxy, letsencrypt.interfaces.IInstaller): """Wraps a Let's Encrypt installer""" + def get_all_names_answer(self): + """Returns all names that should be found by the installer""" + class IConfiguratorProxy(IAuthenticatorProxy, IInstallerProxy): """Wraps a Let's Encrypt configurator""" diff --git a/tests/compatibility/plugin_test.py b/tests/compatibility/test_driver.py similarity index 70% rename from tests/compatibility/plugin_test.py rename to tests/compatibility/test_driver.py index 04cb9da11..5e15ce155 100644 --- a/tests/compatibility/plugin_test.py +++ b/tests/compatibility/test_driver.py @@ -5,6 +5,7 @@ import logging from tests.compatibility import errors from tests.compatibility.configurators.apache import apache24 + DESCRIPTION = """ Tests Let's Encrypt plugins against different server configuratons. It is assumed that Docker is already installed. If no test types is specified, all @@ -31,6 +32,9 @@ def get_args(): help="a directory or tarball containing server configurations") group.add_argument( "-p", "--plugin", default="apache", help="the plugin to be tested") + group.add_argument( + "-v", "--verbose", dest="verbose_count", action="count", + default=0, help="You know how to use this") group.add_argument( "-a", "--auth", action="store_true", help="tests the challenges the plugin supports") @@ -53,30 +57,43 @@ def get_args(): return args -def setup_logging(): +def setup_logging(args): """Prepares logging for the program""" handler = logging.StreamHandler() root_logger = logging.getLogger() - root_logger.setLevel(logging.INFO) + root_logger.setLevel(logging.WARNING - args.verbose_count * 10) root_logger.addHandler(handler) +def test_installer(plugin): + """Tests plugin as an installer""" + if plugin.get_all_names() != plugin.get_all_names_answer(): + raise errors.Error( + "Names found by plugin don't match names found by the wrapper") + + def main(): """Main test script execution.""" - setup_logging() args = get_args() + setup_logging(args) if args.plugin not in PLUGINS: raise errors.Error("Unknown plugin {0}".format(args.plugin)) - plugin = None + + plugin = PLUGINS[args.plugin](args) try: - plugin = PLUGINS[args.plugin](args) - plugin.load_config() - assert plugin.get_all_names() == plugin.get_test_domain_names() + while plugin.has_more_configs(): + try: + print "Loaded configuration: {0}".format(plugin.load_config()) + + if args.install: + test_installer(plugin) + except errors.Error as error: + print "Test failed" + print error finally: - if plugin: - plugin.cleanup_from_tests() + plugin.cleanup_from_tests() if __name__ == "__main__": diff --git a/tests/setup.py b/tests/setup.py index af4da9507..5810b66f9 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -17,7 +17,7 @@ setup( install_requires=install_requires, entry_points={ 'console_scripts': [ - 'compatibility-test = compatibility.plugin_test:main', + 'compatibility-test = compatibility.test_driver:main', ], }, )