Finished basic Apache2.4 proxy

This commit is contained in:
Brad Warren 2015-07-15 23:00:18 -07:00
parent 6c6ef2bb40
commit 4b098cdce2
7 changed files with 327 additions and 83 deletions

View file

@ -3,12 +3,12 @@
enable () {
echo "LoadModule "$1"_module /usr/local/apache2/modules/mod_"$1".so" >> \
$APACHE_CONFDIR"/modules.load"
$APACHE_CONFDIR"/tests.conf"
available_base="/mods-available/"$1".conf"
available_conf=$APACHE_CONFDIR$available_base
enabled_dir=$APACHE_CONFDIR"/mods-enabled"
enabled_conf=$enabled_dir"/"$1".conf"
if [ -e "$available_conf" -a -e "$enabled_dir" -a ! -e "$enabled_conf" ]
if [ -e "$available_conf" -a -d "$enabled_dir" -a ! -e "$enabled_conf" ]
then
ln -s "..$available_base" $enabled_conf
fi

View file

@ -0,0 +1,56 @@
"""Proxies ApacheConfigurator for Apache 2.4 tests"""
from tests.compatibility import errors
from tests.compatibility.configurators.apache import common as apache_common
# The docker image doesn't actually have the watchdog module, but unless the
# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and
# therefore the config won't be loaded), I believe this isn't a problem
# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html
STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog",])
INSTALLED_MODULES = set([
"log_config", "logio", "version", "unixd", "access_compat", "actions",
"alias", "allowmethods", "auth_basic", "auth_digest", "auth_form",
"authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file",
"authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm",
"authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex",
"buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs",
"dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter",
"file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness",
"lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap",
"log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp",
"proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi",
"proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit",
"remoteip", "reqtimeout", "request", "rewrite", "sed", "session",
"session_cookie", "session_crypto", "session_dbd", "setenvif",
"slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb",
"speling", "ssl", "status", "substitute", "unique_id", "userdir",
"vhost_alias",])
class Proxy(apache_common.Proxy):
"""Wraps the ApacheConfigurator for Apache 2.4 tests"""
def __init__(self, args):
"""Initializes the plugin with the given command line args"""
super(Proxy, self).__init__(args)
self.start_docker("bradmw/apache2.4")
def preprocess_config(self):
"""Prepares the configuration for use in the Docker"""
super(Proxy, self).preprocess_config()
if self.version[1] != 4:
raise errors.Error("Apache version not 2.4")
with open(self.test_conf, "a") as f:
for module in self.modules:
if module not in STATIC_MODULES:
if module in INSTALLED_MODULES:
f.write(
"LoadModule {0}_module /usr/local/apache2/modules/"
"mod_{0}\n".format(module))
else:
raise errors.Error(
"Unsupported module {0}".format(module))

View file

@ -1,18 +1,41 @@
"""Provides a common base for Apache tests"""
"""Provides a common base for Apache proxies"""
import logging
import re
import os
import subprocess
import mock
from tests.compatibilty import configurators
from letsencrypt import configuration
from letsencrypt_apache import configurator
from tests.compatibility import errors
from tests.compatibility import util
from tests.compatibility.configurators import common as configurators_common
class ApacheConfiguratorCommonTester(configurators.common.ConfiguratorTester):
APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
logger = logging.getLogger(__name__)
class Proxy(configurators_common.Proxy):
# pylint: disable=too-many-instance-attributes
"""A common base for Apache test configurators"""
def __init__(self, args):
"""Initializes the plugin with the given command line args"""
super(ApacheConfiguratorCommonTester, self).__init__(args)
super(Proxy, self).__init__(args)
self.le_config = util.create_le_config(self.temp_dir)
self.le_config["apache_le_vhost_ext"] = "-le-ssl.conf"
self._patch = mock.patch('letsencrypt_apache.configurator.subprocess')
self._mock = self._patch.start()
self._mock.check_call = self._check_call
self._apache_configurator = None
self._mock.check_call = self.check_call_in_docker
self._mock.Popen = self.popen_in_docker
self.server_root = self.modules = self.version = self.test_conf = None
self._config_file = self._apache_configurator = self._names = None
def __getattr__(self, name):
"""Wraps the Apache Configurator methods"""
@ -22,13 +45,132 @@ class ApacheConfiguratorCommonTester(configurators.common.ConfiguratorTester):
else:
raise AttributeError()
def _check_call(self, command, *args, **kwargs):
"""A function to mock the call to subprocess.check_call"""
def load_config(self):
"""Loads the next configuration for the plugin to test"""
raise NotImplementedError()
config = self.get_next_config()
logger.debug("Loading configuration: %s", config)
self._parse_config(config)
self._prepare_configurator()
self.preprocess_config()
try:
self.check_call_in_docker(
"apachectl -d {0} -f {1} -k restart".format(
self.server_root, self._config_file))
except errors.Error:
raise errors.Error(
"Apache failed to load {0} before tests started".format(
config))
def preprocess_config(self):
# pylint: disable=anomalous-backslash-in-string
"""Prepares the configuration for use in the Docker"""
self.test_conf = os.path.join(self.server_root, "test.conf")
open(self.test_conf, "w").close()
subprocess.check_call(
["sed", "-i", "1iInclude test.conf", self._config_file])
find = subprocess.Popen(
["find", self.server_root, "-type", "f"],
stdout=subprocess.PIPE)
subprocess.check_call([
"xargs", "sed", "-e",
"s/DocumentRoot.*/DocumentRoot \/usr\/local\/apache2\/htdocs/I",
"-e",
"s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I",
"-i"], stdin=find.stdout)
def _parse_config(self, config):
"""Parses extra information in server config directory"""
self.server_root = _get_server_root(config)
self.modules = _get_modules(config)
self.version = _get_version(config)
self._names = _get_names(config)
with open(os.path.join(config, "config_file")) as f:
self._config_file = os.path.join(self.server_root, f.readline())
def _prepare_configurator(self):
"""Prepares the Apache plugin for testing"""
self.le_config["apache_ctl"] = "apachectl -d {0} -f {1}".format(
self.server_root, self._config_file)
self.le_config["a2enmod.sh"] = "a2enmod.sh"
self.le_config["apache_init_script"] = self.le_config["apache_ctl"]
self.le_config["apache_init_script"] += " -k"
self._apache_configurator = configurator.ApacheConfigurator(
config=configuration.NamespaceConfig(self.le_config),
name="apache")
self._apache_configurator.prepare()
def get_test_domain_names(self):
"""Returns a list of domain names to test against the plugin"""
raise NotImplementedError()
if self._names:
return self._names
else:
raise errors.Error("No configuration file loaded")
def _get_server_root(config):
"""Returns the server root directory in config"""
subdirs = [
name for name in os.listdir(config)
if os.path.isdir(os.path.join(config, name))]
if len(subdirs) != 1:
errors.Error("Malformed configuration directiory {0}".format(config))
return subdirs[0]
def _get_names(config):
"""Returns domains names for config"""
names = set()
with open(os.path.join(config, "vhosts")) as f:
for line in f:
# If parsing a specific vhost
if line[0].isspace():
words = line.split()
if words[0] == "alias":
names.add(words[1])
# If for port 80 and not IP vhost
elif words[1] == "80" and not util.IP_REGEX.match(words[3]):
names.add(words[3])
elif "NameVirtualHost" not in line:
words = line.split()
if ((words[0].endswith("*") or words[0].endswith("80")) and
util.IP_REGEX.match(words[1])):
names.add(words[1])
return names
def _get_modules(config):
"""Returns the list of modules found in module_list"""
modules = []
with open(os.path.join(config, "modules")) as f:
for line in f:
# Modules list is indented, everything else is headers/footers
if line[0].isspace():
words = line.split()
# Modules redundantly end in "_module" which we can discard
modules.append(words[-7])
return modules
def _get_version(config):
"""Return version of Apache Server.
Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from
the Apache plugin.
"""
with open(os.path.join(config, "version")) as f:
# Should be on first line of input
matches = APACHE_VERSION_REGEX.findall(f.readline())
if len(matches) != 1:
raise errors.Error("Unable to find Apache version")
return tuple([int(i) for i in matches[0].split(".")])

View file

@ -1,7 +1,8 @@
"""Provides a common base for compatibility test configurators"""
"""Provides a common base for configurator proxies"""
import logging
import multiprocessing
import os
import tempfile
import docker
@ -12,7 +13,7 @@ from tests.compatibility import util
logger = logging.getLogger(__name__)
class ConfiguratorTester(object):
class Proxy(object):
# pylint: disable=too-many-instance-attributes
"""A common base for compatibility test configurators"""
@ -21,25 +22,25 @@ class ConfiguratorTester(object):
@classmethod
def add_parser_arguments(cls, parser):
"""Adds command line arguments needed by the plugin"""
if ConfiguratorTester._NOT_ADDED_ARGS:
group = parser.add_argument_group('docker')
if Proxy._NOT_ADDED_ARGS:
group = parser.add_argument_group("docker")
group.add_argument(
'--docker-url', default='unix://var/run/docker.sock',
help='URL of the docker server')
"--docker-url", default="unix://var/run/docker.sock",
help="URL of the docker server")
group.add_argument(
'--no-remove', action='store_true',
help='do not delete container on program exit')
ConfiguratorTester._NOT_ADDED_ARGS = False
"--no-remove", action="store_true",
help="do not delete container on program exit")
Proxy._NOT_ADDED_ARGS = False
def __init__(self, args):
"""Initializes the plugin with the given command line args"""
self.temp_dir = util.setup_temp_dir(args.configs)
self.config_dir = os.path.join(self.temp_dir, util.CONFIG_DIR)
self.temp_dir = tempfile.mkdtemp()
self.config_dir = util.extract_configs(args.configs, self.temp_dir)
self._configs = os.listdir(self.config_dir)
self.args = args
self._docker_client = docker.Client(
base_url=self.args.docker_url, version='auto')
base_url=self.args.docker_url, version="auto")
self.http_port, self.https_port = util.get_two_free_ports()
self._container_id = self._log_process = None
@ -65,31 +66,65 @@ class ConfiguratorTester(object):
host_config = docker.utils.create_host_config(
binds={
self.config_dir : {'bind' : self.config_dir, 'mode' : 'rw'}},
self.config_dir : {"bind" : self.config_dir, "mode" : "rw"}},
port_bindings={
80 : ('127.0.0.1', self.http_port),
443 : ('127.0.0.1', self.https_port)},)
80 : ("127.0.0.1", self.http_port),
443 : ("127.0.0.1", self.https_port)},)
container = self._docker_client.create_container(
image_name, ports=[80, 443], volumes=self.config_dir,
host_config=host_config)
if container['Warnings']:
logger.warning(container['Warnings'])
self._container_id = container['Id']
if container["Warnings"]:
logger.warning(container["Warnings"])
self._container_id = container["Id"]
self._docker_client.start(self._container_id)
self._log_process = multiprocessing.Process(
target=self._start_log_thread)
self._log_process.start()
def execute_in_docker(self, command):
"""Executes command inside the running docker image"""
exec_id = self._docker_client.exec_create(self._container_id, command)
output = self._docker_client.exec_start(exec_id)
if self._docker_client.exec_inspect(exec_id)['ExitCode']:
raise errors.Error('Docker command \'{0}\' failed'.format(command))
return output
def _start_log_thread(self):
client = docker.Client(base_url=self.args.docker_url, version='auto')
client = docker.Client(base_url=self.args.docker_url, version="auto")
for line in client.logs(self._container_id, stream=True):
logger.debug(line)
def check_call_in_docker(
self, command, *args, **kwargs): # pylint: disable=unused-argument
"""Simulates a call to check_call but executes the command in the
running docker image
"""
if self.popen_in_docker(command).returncode:
raise errors.Error(
"{0} exited with a nonzero value".format(command))
def popen_in_docker(
self, command, *args, **kwargs): # pylint: disable=unused-argument
"""Simulates a call to Popen but executes the command in the
running docker image
"""
class SimplePopen(object):
# pylint: disable=too-few-public-methods
"""Simplified Popen object"""
def __init__(self, returncode, output):
self.returncode = returncode
self._stdout = output
self._stderr = output
def communicate(self):
"""Returns stdout and stderr"""
return self._stdout, self._stderr
if isinstance(command, list):
command = " ".join(command)
returncode, output = self._execute_in_docker(command)
return SimplePopen(returncode, output)
def _execute_in_docker(self, command):
"""Executes command inside the running docker image"""
logger.debug("Executing '%s'", command)
exec_id = self._docker_client.exec_create(self._container_id, command)
output = self._docker_client.exec_start(exec_id)
returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"]
return returncode, output

View file

@ -4,7 +4,7 @@ import zope.interface
import letsencrypt.interfaces
class IPluginTester(zope.interface.Interface):
class IPluginProxy(zope.interface.Interface):
"""Wraps a Let's Encrypt plugin"""
@classmethod
def add_parser_arguments(cls, parser):
@ -27,27 +27,27 @@ class IPluginTester(zope.interface.Interface):
"""Loads the next configuration for the plugin to test"""
class IConfiguratorBaseTester(IPluginTester):
class IConfiguratorBaseProxy(IPluginProxy):
"""Common functionality for authenticator/installer tests"""
http_port = zope.interface.Attribute(
'The port to connect to on localhost for HTTP traffic')
"The port to connect to on localhost for HTTP traffic")
https_port = zope.interface.Attribute(
'The port to connect to on localhost for HTTPS traffic')
"The port to connect to on localhost for HTTPS traffic")
def get_test_domain_names(self):
"""Returns a list of domain names to test against the plugin"""
class IAuthenticatorTester(
IConfiguratorBaseTester, letsencrypt.interfaces.IAuthenticator):
class IAuthenticatorProxy(
IConfiguratorBaseProxy, letsencrypt.interfaces.IAuthenticator):
"""Wraps a Let's Encrypt authenticator"""
class IInstallerTester(
IConfiguratorBaseTester, letsencrypt.interfaces.IInstaller):
class IInstallerProxy(
IConfiguratorBaseProxy, letsencrypt.interfaces.IInstaller):
"""Wraps a Let's Encrypt installer"""
class IConfiguratorTester(IAuthenticatorTester, IInstallerTester):
class IConfiguratorProxy(IAuthenticatorProxy, IInstallerProxy):
"""Wraps a Let's Encrypt configurator"""

View file

@ -3,7 +3,7 @@ import argparse
import logging
import os
from tests.compatibility.configurators import common
from tests.compatibility.configurators.apache import apache24
DESCRIPTION = """
Tests Let's Encrypt plugins against different server configuratons. It is
@ -12,7 +12,7 @@ assumed that Docker is already installed.
"""
PLUGINS = {'common' : common.ConfiguratorTester}
PLUGINS = {"apache" : apache24.Proxy}
logger = logging.getLogger(__name__)
@ -22,22 +22,22 @@ def get_args():
"""Returns parsed command line arguments."""
parser = argparse.ArgumentParser(description=DESCRIPTION)
group = parser.add_argument_group('general')
group = parser.add_argument_group("general")
group.add_argument(
'-c', '--configs', default='configs.tar.gz',
help='a directory or tarball containing server configurations')
"-c", "--configs", default="configs.tar.gz",
help="a directory or tarball containing server configurations")
group.add_argument(
'-p', '--plugin', default='apache', help='the plugin to be tested')
"-p", "--plugin", default="apache", help="the plugin to be tested")
group.add_argument(
'-a', '--auth', action='store_true',
help='tests the plugin as an authenticator')
"-a", "--auth", action="store_true",
help="tests the plugin as an authenticator")
group.add_argument(
'-i', '--install', action='store_true',
help='tests the plugin as an installer')
"-i", "--install", action="store_true",
help="tests the plugin as an installer")
group.add_argument(
'-r', '--redirect', action='store_true', help='tests the plugin\'s '
'ability to redirect HTTP to HTTPS (implicitly includes installer '
'tests)')
"-r", "--redirect", action="store_true", help="tests the plugin's "
"ability to redirect HTTP to HTTPS (implicitly includes installer "
"tests)")
for plugin in PLUGINS.itervalues():
plugin.add_parser_arguments(parser)
@ -66,13 +66,12 @@ def main():
"""Main test script execution."""
setup_logging()
args = get_args()
if args.plugin not in PLUGINS:
raise errors.Error("Unknown plugin {0}".format(args.plugin))
plugin = PLUGINS[args.plugin](args)
plugin.start_docker('bradmw/apache2.4')
config = os.path.join(plugin.config_dir, 'apache2')
config_file = os.path.join(config, 'apache2.conf')
plugin.execute_in_docker('apachectl -d {0} -f {1} -k restart'.format(config, config_file))
#plugin.cleanup_from_tests()
plugin.cleanup_from_tests()
if __name__ == '__main__':
if __name__ == "__main__":
main()

View file

@ -1,41 +1,53 @@
"""Utility functions for Let's Encrypt plugin tests."""
"""Utility functions for Let"s Encrypt plugin tests."""
import copy
import contextlib
import os
import re
import shutil
import socket
import tarfile
import tempfile
from letsencrypt import constants
from tests.compatibility import errors
# Paths used in the program relative to the temp directory
CONFIG_DIR = "configs"
LE_CONFIG = os.path.join("letsencrypt", "config")
LE_LOGS = os.path.join("letsencrypt", "logs")
IP_REGEX = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")
def setup_temp_dir(configs):
"""Sets up a temporary directory and extracts server configs"""
temp_dir = tempfile.mkdtemp()
config_dir = os.path.join(temp_dir, CONFIG_DIR)
def create_le_config(parent_dir):
"""Sets up LE dirs in parent_dir and returns the config dict"""
config = copy.deepcopy(constants.CLI_DEFAULTS)
le_dir = os.path.join(parent_dir, "letsencrypt")
config["config_dir"] = os.path.join(le_dir, "config")
config["work_dir"] = os.path.join(le_dir, "work")
config["logs_dir"] = os.path.join(le_dir, "logs_dir")
os.makedirs(config["config_dir"])
os.mkdir(config["work_dir"])
os.mkdir(config["logs_dir"])
return config
def extract_configs(configs, parent_dir):
"""Extracts configs to a new dir under parent_dir and returns it"""
config_dir = os.path.join(parent_dir, "configs")
if os.path.isdir(configs):
shutil.copytree(configs, config_dir, symlinks=True)
elif tarfile.is_tarfile(configs):
with tarfile.open(configs, 'r') as tar:
with tarfile.open(configs, "r") as tar:
tar.extractall(config_dir)
else:
raise errors.Error('Unknown configurations file type')
raise errors.Error("Unknown configurations file type")
return temp_dir
return config_dir
def get_two_free_ports():
"""Returns two free ports to use for the tests"""
with contextlib.closing(socket.socket()) as sock1:
with contextlib.closing(socket.socket()) as sock2:
sock1.bind(('', 0))
sock2.bind(('', 0))
sock1.bind(("", 0))
sock2.bind(("", 0))
return sock1.getsockname()[1], sock2.getsockname()[1]