upstream/certbot
1
0
Fork 0
mirror of https://github.com/certbot/certbot.git synced 2026-06-08 00:02:14 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #3138 from certbot/removing_proxy

Browse source 
Remove proxy in certbot-compatibility-test
This commit is contained in:
schoen 2016-06-10 18:23:32 -07:00 committed by GitHub
commit 42f6f5174b
10 changed files with 39 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitattributes vendored
View file

@ -5,3 +5,4 @@
*.jpeg binary
*.jpg binary
*.png binary
*.gz binary

14
certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh
View file

@ -1,14 +0,0 @@
#!/bin/bash
# An extremely simplified version of `a2enmod` for disabling modules in the
# httpd docker image. First argument is the server_root and the second is the
# module to be disabled.
apache_confdir=$1
module=$2
sed -i "/.*"$module".*/d" "$apache_confdir/test.conf"
enabled_conf="$apache_confdir/mods-enabled/"$module".conf"
if [ -e "$enabled_conf" ]
then
rm $enabled_conf
fi

18
certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh
View file

@ -1,18 +0,0 @@
#!/bin/bash
# An extremely simplified version of `a2enmod` for enabling modules in the
# httpd docker image. First argument is the Apache ServerRoot which should be
# an absolute path. The second is the module to be enabled, such as `ssl`.
confdir=$1
module=$2
echo "LoadModule ${module}_module " \
"/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf"
availbase="/mods-available/${module}.conf"
availconf=$confdir$availbase
enabldir="$confdir/mods-enabled"
enablconf="$enabldir/${module}.conf"
if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ]
then
ln -s "..$availbase" $enablconf
fi

63
certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py
View file

@ -1,63 +0,0 @@
"""Proxies ApacheConfigurator for Apache 2.4 tests"""
import zope.interface
from certbot_compatibility_test import errors
from certbot_compatibility_test import interfaces
from certbot_compatibility_test.configurators.apache import common as apache_common
# The docker image doesn't actually have the watchdog module, but unless the
# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and
# therefore the config won't be loaded), I believe this isn't a problem
# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html
STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog"])
SHARED_MODULES = {
"log_config", "logio", "version", "unixd", "access_compat", "actions",
"alias", "allowmethods", "auth_basic", "auth_digest", "auth_form",
"authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file",
"authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm",
"authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex",
"buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs",
"dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter",
"file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness",
"lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap",
"log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp",
"proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi",
"proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit",
"remoteip", "reqtimeout", "request", "rewrite", "sed", "session",
"session_cookie", "session_crypto", "session_dbd", "setenvif",
"slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb",
"speling", "ssl", "status", "substitute", "unique_id", "userdir",
"vhost_alias"}
@zope.interface.implementer(interfaces.IConfiguratorProxy)
class Proxy(apache_common.Proxy):
"""Wraps the ApacheConfigurator for Apache 2.4 tests"""
def __init__(self, args):
"""Initializes the plugin with the given command line args"""
super(Proxy, self).__init__(args)
# Running init isn't ideal, but the Docker container needs to survive
# Apache restarts
self.start_docker("bradmw/apache2.4", "init")
def preprocess_config(self, server_root):
"""Prepares the configuration for use in the Docker"""
super(Proxy, self).preprocess_config(server_root)
if self.version[1] != 4:
raise errors.Error("Apache version not 2.4")
with open(self.test_conf, "a") as f:
for module in self.modules:
if module not in STATIC_MODULES:
if module in SHARED_MODULES:
f.write(
"LoadModule {0}_module /usr/local/apache2/modules/"
"mod_{0}.so\n".format(module))
else:
raise errors.Error(
"Unsupported module {0}".format(module))

166
certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py
View file

@ -1,6 +1,7 @@
"""Provides a common base for Apache proxies"""
import re
import os
import shutil
import subprocess
import mock
@ -9,6 +10,7 @@ import zope.interface
from certbot import configuration
from certbot import errors as le_errors
from certbot_apache import configurator
from certbot_apache import constants
from certbot_compatibility_test import errors
from certbot_compatibility_test import interfaces
from certbot_compatibility_test import util
@ -29,58 +31,14 @@ class Proxy(configurators_common.Proxy):
super(Proxy, self).__init__(args)
self.le_config.apache_le_vhost_ext = "-le-ssl.conf"
self._setup_mock()
self.modules = self.server_root = self.test_conf = self.version = None
self._apache_configurator = self._all_names = self._test_names = None
def _setup_mock(self):
"""Replaces specific modules with mock.MagicMock"""
mock_subprocess = mock.MagicMock()
mock_subprocess.check_call = self.check_call
mock_subprocess.Popen = self.popen
mock.patch(
"certbot_apache.configurator.subprocess",
mock_subprocess).start()
mock.patch(
"certbot_apache.parser.subprocess",
mock_subprocess).start()
mock.patch(
"certbot.util.subprocess",
mock_subprocess).start()
mock.patch(
"certbot_apache.configurator.util.exe_exists",
_is_apache_command).start()
patch = mock.patch(
"certbot_apache.configurator.display_ops.select_vhost")
mock_display = patch.start()
mock_display.side_effect = le_errors.PluginError(
"Unable to determine vhost")
def check_call(self, command, *args, **kwargs):
"""If command is an Apache command, command is executed in the
running docker image. Otherwise, subprocess.check_call is used.
"""
if _is_apache_command(command):
command = _modify_command(command)
return super(Proxy, self).check_call(command, *args, **kwargs)
else:
return subprocess.check_call(command, *args, **kwargs)
def popen(self, command, *args, **kwargs):
"""If command is an Apache command, command is executed in the
running docker image. Otherwise, subprocess.Popen is used.
"""
if _is_apache_command(command):
command = _modify_command(command)
return super(Proxy, self).popen(command, *args, **kwargs)
else:
return subprocess.Popen(command, *args, **kwargs)
def __getattr__(self, name):
"""Wraps the Apache Configurator methods"""
method = getattr(self._apache_configurator, name, None)
@ -91,29 +49,20 @@ class Proxy(configurators_common.Proxy):
def load_config(self):
"""Loads the next configuration for the plugin to test"""
if hasattr(self.le_config, "apache_init_script"):
try:
self.check_call([self.le_config.apache_init_script, "stop"])
except errors.Error:
raise errors.Error(
"Failed to stop previous apache config from running")
config = super(Proxy, self).load_config()
self.modules = _get_modules(config)
self.version = _get_version(config)
self._all_names, self._test_names = _get_names(config)
server_root = _get_server_root(config)
with open(os.path.join(config, "config_file")) as f:
config_file = os.path.join(server_root, f.readline().rstrip())
self.test_conf = _create_test_conf(server_root, config_file)
# with open(os.path.join(config, "config_file")) as f:
# config_file = os.path.join(server_root, f.readline().rstrip())
shutil.rmtree("/etc/apache2")
shutil.copytree(server_root, "/etc/apache2", symlinks=True)
self.preprocess_config(server_root)
self._prepare_configurator(server_root, config_file)
self._prepare_configurator()
try:
self.check_call("apachectl -d {0} -f {1} -k start".format(
server_root, config_file))
subprocess.check_call("apachectl -k start".split())
except errors.Error:
raise errors.Error(
"Apache failed to load {0} before tests started".format(
@ -121,34 +70,13 @@ class Proxy(configurators_common.Proxy):
return config
def preprocess_config(self, server_root):
# pylint: disable=anomalous-backslash-in-string, no-self-use
"""Prepares the configuration for use in the Docker"""
find = subprocess.Popen(
["find", server_root, "-type", "f"],
stdout=subprocess.PIPE)
subprocess.check_call([
"xargs", "sed", "-e", "s/DocumentRoot.*/DocumentRoot "
"\/usr\/local\/apache2\/htdocs/I",
"-e", "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I",
"-e", "s/TypesConfig.*/TypesConfig "
"\/usr\/local\/apache2\/conf\/mime.types/I",
"-e", "s/LoadModule/#LoadModule/I",
"-e", "s/SSLCertificateFile.*/SSLCertificateFile "
"\/usr\/local\/apache2\/conf\/empty_cert.pem/I",
"-e", "s/SSLCertificateKeyFile.*/SSLCertificateKeyFile "
"\/usr\/local\/apache2\/conf\/rsa1024_key2.pem/I",
"-i"], stdin=find.stdout)
def _prepare_configurator(self, server_root, config_file):
def _prepare_configurator(self):
"""Prepares the Apache plugin for testing"""
self.le_config.apache_server_root = server_root
self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format(
server_root, config_file)
self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root)
self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root)
self.le_config.apache_init_script = self.le_config.apache_ctl + " -k"
for k in constants.CLI_DEFAULTS_DEBIAN.keys():
setattr(self.le_config, "apache_" + k, constants.os_constant(k))
# An alias
self.le_config.apache_handle_modules = self.le_config.apache_handle_mods
self._apache_configurator = configurator.ApacheConfigurator(
config=configuration.NamespaceConfig(self.le_config),
@ -183,39 +111,6 @@ class Proxy(configurators_common.Proxy):
domain, cert_path, key_path, chain_path, fullchain_path)
def _is_apache_command(command):
"""Returns true if command is an Apache command"""
if isinstance(command, list):
command = command[0]
for apache_command in APACHE_COMMANDS:
if command.startswith(apache_command):
return True
return False
def _modify_command(command):
"""Modifies command so configtest works inside the docker image"""
if isinstance(command, list):
for i in xrange(len(command)):
if command[i] == "configtest":
command[i] = "-t"
else:
command = command.replace("configtest", "-t")
return command
def _create_test_conf(server_root, apache_config):
"""Creates a test config file and adds it to the Apache config"""
test_conf = os.path.join(server_root, "test.conf")
open(test_conf, "w").close()
subprocess.check_call(
["sed", "-i", "1iInclude test.conf", apache_config])
return test_conf
def _get_server_root(config):
"""Returns the server root directory in config"""
subdirs = [
@ -223,7 +118,7 @@ def _get_server_root(config):
if os.path.isdir(os.path.join(config, name))]
if len(subdirs) != 1:
errors.Error("Malformed configuration directiory {0}".format(config))
errors.Error("Malformed configuration directory {0}".format(config))
return os.path.join(config, subdirs[0].rstrip())
@ -251,34 +146,3 @@ def _get_names(config):
words[1].find(".") != -1):
all_names.add(words[1])
return all_names, non_ip_names
def _get_modules(config):
"""Returns the list of modules found in module_list"""
modules = []
with open(os.path.join(config, "modules")) as f:
for line in f:
# Modules list is indented, everything else is headers/footers
if line[0].isspace():
words = line.split()
# Modules redundantly end in "_module" which we can discard
modules.append(words[0][:-7])
return modules
def _get_version(config):
"""Return version of Apache Server.
Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from
the Apache plugin.
"""
with open(os.path.join(config, "version")) as f:
# Should be on first line of input
matches = APACHE_VERSION_REGEX.findall(f.readline())
if len(matches) != 1:
raise errors.Error("Unable to find Apache version")
return tuple([int(i) for i in matches[0].split(".")])

84
certbot-compatibility-test/certbot_compatibility_test/configurators/common.py
View file

@ -4,10 +4,7 @@ import os
import shutil
import tempfile
import docker
from certbot import constants
from certbot_compatibility_test import errors
from certbot_compatibility_test import util
@ -18,20 +15,9 @@ class Proxy(object):
# pylint: disable=too-many-instance-attributes
"""A common base for compatibility test configurators"""
_NOT_ADDED_ARGS = True
@classmethod
def add_parser_arguments(cls, parser):
"""Adds command line arguments needed by the plugin"""
if Proxy._NOT_ADDED_ARGS:
group = parser.add_argument_group("docker")
group.add_argument(
"--docker-url", default="unix://var/run/docker.sock",
help="URL of the docker server")
group.add_argument(
"--no-remove", action="store_true",
help="do not delete container on program exit")
Proxy._NOT_ADDED_ARGS = False
def __init__(self, args):
"""Initializes the plugin with the given command line args"""
@ -43,10 +29,8 @@ class Proxy(object):
for config in os.listdir(config_dir)]
self.args = args
self._docker_client = docker.Client(
base_url=self.args.docker_url, version="auto")
self.http_port, self.https_port = util.get_two_free_ports()
self._container_id = None
self.http_port = 80
self.https_port = 443
def has_more_configs(self):
"""Returns true if there are more configs to test"""
@ -54,9 +38,6 @@ class Proxy(object):
def cleanup_from_tests(self):
"""Performs any necessary cleanup from running plugin tests"""
self._docker_client.stop(self._container_id, 0)
if not self.args.no_remove:
self._docker_client.remove_container(self._container_id)
def load_config(self):
"""Returns the next config directory to be tested"""
@ -65,67 +46,6 @@ class Proxy(object):
os.makedirs(backup)
return self._configs.pop()
def start_docker(self, image_name, command):
"""Creates and runs a Docker container with the specified image"""
logger.warning("Pulling Docker image. This may take a minute.")
for line in self._docker_client.pull(image_name, stream=True):
logger.debug(line)
host_config = docker.utils.create_host_config(
binds={self._temp_dir: {"bind": self._temp_dir, "mode": "rw"}},
port_bindings={
80: ("127.0.0.1", self.http_port),
443: ("127.0.0.1", self.https_port)},)
container = self._docker_client.create_container(
image_name, command, ports=[80, 443], volumes=self._temp_dir,
host_config=host_config)
if container["Warnings"]:
logger.warning(container["Warnings"])
self._container_id = container["Id"]
self._docker_client.start(self._container_id)
def check_call(self, command, *args, **kwargs):
# pylint: disable=unused-argument
"""Simulates a call to check_call but executes the command in the
running docker image
"""
if self.popen(command).returncode:
raise errors.Error(
"{0} exited with a nonzero value".format(command))
def popen(self, command, *args, **kwargs):
# pylint: disable=unused-argument
"""Simulates a call to Popen but executes the command in the
running docker image
"""
class SimplePopen(object):
# pylint: disable=too-few-public-methods
"""Simplified Popen object"""
def __init__(self, returncode, output):
self.returncode = returncode
self._stdout = output
self._stderr = output
def communicate(self):
"""Returns stdout and stderr"""
return self._stdout, self._stderr
if isinstance(command, list):
command = " ".join(command)
returncode, output = self.execute_in_docker(command)
return SimplePopen(returncode, output)
def execute_in_docker(self, command):
"""Executes command inside the running docker image"""
logger.debug("Executing '%s'", command)
exec_id = self._docker_client.exec_create(self._container_id, command)
output = self._docker_client.exec_start(exec_id)
returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"]
return returncode, output
def copy_certs_and_keys(self, cert_path, key_path, chain_path=None):
"""Copies certs and keys into the temporary directory"""
cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys")

27
certbot-compatibility-test/certbot_compatibility_test/test_driver.py
View file

@ -7,6 +7,7 @@ import os
import shutil
import tempfile
import time
import sys
import OpenSSL
@ -21,17 +22,17 @@ from certbot_compatibility_test import errors
from certbot_compatibility_test import util
from certbot_compatibility_test import validator
from certbot_compatibility_test.configurators.apache import apache24
from certbot_compatibility_test.configurators.apache import common
DESCRIPTION = """
Tests Certbot plugins against different server configuratons. It is
assumed that Docker is already installed. If no test types is specified, all
Tests Certbot plugins against different server configurations. It is
assumed that Docker is already installed. If no test type is specified, all
tests that the plugin supports are performed.
"""
PLUGINS = {"apache": apache24.Proxy}
PLUGINS = {"apache": common.Proxy}
logger = logging.getLogger(__name__)
@ -61,8 +62,8 @@ def test_authenticator(plugin, config, temp_dir):
"Plugin failed to complete %s for %s in %s",
type(achalls[i]), achalls[i].domain, config)
success = False
elif isinstance(responses[i], challenges.TLSSNI01):
verify = functools.partial(responses[i].simple_verify, achalls[i],
elif isinstance(responses[i], challenges.TLSSNI01Response):
verify = functools.partial(responses[i].simple_verify, achalls[i].chall,
achalls[i].domain,
util.JWK.public_key(),
host="127.0.0.1",
@ -142,7 +143,8 @@ def test_deploy_cert(plugin, temp_dir, domains):
for domain in domains:
try:
plugin.deploy_cert(domain, cert_path, util.KEY_PATH)
plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path)
plugin.save() # Needed by the Apache plugin
except le_errors.Error as error:
logger.error("Plugin failed to deploy ceritificate for %s:", domain)
logger.exception(error)
@ -177,6 +179,7 @@ def test_enhancements(plugin, domains):
for domain in domains:
try:
plugin.enhance(domain, "redirect")
plugin.save() # Needed by the Apache plugin
except le_errors.PluginError as error:
# Don't immediately fail because a redirect may already be enabled
logger.warning("Plugin failed to enable redirect for %s:", domain)
@ -341,7 +344,7 @@ def main():
temp_dir = tempfile.mkdtemp()
plugin = PLUGINS[args.plugin](args)
try:
plugin.execute_in_docker("mkdir -p /var/log/apache2")
overall_success = True
while plugin.has_more_configs():
success = True
@ -360,10 +363,18 @@ def main():
if success:
logger.info("All tests on %s succeeded", config)
else:
overall_success = False
logger.error("Tests on %s failed", config)
finally:
plugin.cleanup_from_tests()
if overall_success:
logger.warn("All compatibility tests succeeded")
sys.exit(0)
else:
logger.warn("One or more compatibility tests failed")
sys.exit(1)
if __name__ == "__main__":
main()

BIN
certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz vendored
View file

Binary file not shown.

12
certbot-compatibility-test/certbot_compatibility_test/util.py
View file

@ -1,11 +1,9 @@
"""Utility functions for Certbot plugin tests."""
import argparse
import copy
import contextlib
import os
import re
import shutil
import socket
import tarfile
from acme import jose
@ -52,13 +50,3 @@ def extract_configs(configs, parent_dir):
raise errors.Error("Unknown configurations file type")
return config_dir
def get_two_free_ports():
"""Returns two free ports to use for the tests"""
with contextlib.closing(socket.socket()) as sock1:
with contextlib.closing(socket.socket()) as sock2:
sock1.bind(("", 0))
sock2.bind(("", 0))
return sock1.getsockname()[1], sock2.getsockname()[1]

5
certbot-compatibility-test/setup.py
View file

@ -7,9 +7,8 @@ from setuptools import find_packages
version = '0.9.0.dev0'
install_requires = [
'certbot=={0}'.format(version),
'certbot-apache=={0}'.format(version),
'docker-py',
'certbot',
'certbot-apache',
'requests',
'zope.interface',
]