diff --git a/.gitattributes b/.gitattributes index 5eee84cce..8c41b686e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,4 @@ *.jpeg binary *.jpg binary *.png binary +*.gz binary diff --git a/certbot-apache/certbot_apache/augeas_configurator.py b/certbot-apache/certbot_apache/augeas_configurator.py index 12753541c..6999120d6 100644 --- a/certbot-apache/certbot_apache/augeas_configurator.py +++ b/certbot-apache/certbot_apache/augeas_configurator.py @@ -1,7 +1,6 @@ """Class of Augeas Configurators.""" import logging -import augeas from certbot import errors from certbot import reverter @@ -29,12 +28,9 @@ class AugeasConfigurator(common.Plugin): def __init__(self, *args, **kwargs): super(AugeasConfigurator, self).__init__(*args, **kwargs) - self.aug = augeas.Augeas( - # specify a directory to load our preferred lens from - loadpath=constants.AUGEAS_LENS_DIR, - # Do not save backup (we do it ourselves), do not load - # anything by default - flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) + # Placeholder for augeas + self.aug = None + self.save_notes = "" # See if any temporary changes need to be recovered @@ -42,6 +38,16 @@ class AugeasConfigurator(common.Plugin): # because this will change the underlying configuration and potential # vhosts self.reverter = reverter.Reverter(self.config) + + def init_augeas(self): + """ Initialize the actual Augeas instance """ + import augeas + self.aug = augeas.Augeas( + # specify a directory to load our preferred lens from + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) self.recovery_routine() def check_parsing_errors(self, lens): diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index e4c06ba7e..9caa4a764 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -150,6 +150,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :raises .errors.PluginError: If there is any other error """ + # Perform the actual Augeas initialization to be able to react + try: + self.init_augeas() + except ImportError: + raise errors.NoInstallationError("Problem in Augeas installation") + # Verify Apache is installed if not util.exe_exists(constants.os_constant("restart_cmd")[0]): raise errors.NoInstallationError diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index a2e39de47..e5c09fd1d 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -55,6 +55,16 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.NoInstallationError, self.config.prepare) + @mock.patch("certbot_apache.augeas_configurator.AugeasConfigurator.init_augeas") + def test_prepare_no_augeas(self, mock_init_augeas): + """ Test augeas initialization ImportError """ + def side_effect_error(): + """ Side effect error for the test """ + raise ImportError + mock_init_augeas.side_effect = side_effect_error + self.assertRaises( + errors.NoInstallationError, self.config.prepare) + @mock.patch("certbot_apache.parser.ApacheParser") @mock.patch("certbot_apache.configurator.util.exe_exists") def test_prepare_version(self, mock_exe_exists, _): diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh deleted file mode 100755 index ca96e216f..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for disabling modules in the -# httpd docker image. First argument is the server_root and the second is the -# module to be disabled. - -apache_confdir=$1 -module=$2 - -sed -i "/.*"$module".*/d" "$apache_confdir/test.conf" -enabled_conf="$apache_confdir/mods-enabled/"$module".conf" -if [ -e "$enabled_conf" ] -then - rm $enabled_conf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh deleted file mode 100755 index 4da9288a2..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for enabling modules in the -# httpd docker image. First argument is the Apache ServerRoot which should be -# an absolute path. The second is the module to be enabled, such as `ssl`. - -confdir=$1 -module=$2 - -echo "LoadModule ${module}_module " \ - "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf" -availbase="/mods-available/${module}.conf" -availconf=$confdir$availbase -enabldir="$confdir/mods-enabled" -enablconf="$enabldir/${module}.conf" -if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ] -then - ln -s "..$availbase" $enablconf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py deleted file mode 100644 index 927c329ef..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Proxies ApacheConfigurator for Apache 2.4 tests""" - -import zope.interface - -from certbot_compatibility_test import errors -from certbot_compatibility_test import interfaces -from certbot_compatibility_test.configurators.apache import common as apache_common - - -# The docker image doesn't actually have the watchdog module, but unless the -# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and -# therefore the config won't be loaded), I believe this isn't a problem -# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html -STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog"]) - - -SHARED_MODULES = { - "log_config", "logio", "version", "unixd", "access_compat", "actions", - "alias", "allowmethods", "auth_basic", "auth_digest", "auth_form", - "authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file", - "authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm", - "authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex", - "buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs", - "dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter", - "file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness", - "lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap", - "log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp", - "proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi", - "proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit", - "remoteip", "reqtimeout", "request", "rewrite", "sed", "session", - "session_cookie", "session_crypto", "session_dbd", "setenvif", - "slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb", - "speling", "ssl", "status", "substitute", "unique_id", "userdir", - "vhost_alias"} - - -@zope.interface.implementer(interfaces.IConfiguratorProxy) -class Proxy(apache_common.Proxy): - """Wraps the ApacheConfigurator for Apache 2.4 tests""" - - def __init__(self, args): - """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) - # Running init isn't ideal, but the Docker container needs to survive - # Apache restarts - self.start_docker("bradmw/apache2.4", "init") - - def preprocess_config(self, server_root): - """Prepares the configuration for use in the Docker""" - super(Proxy, self).preprocess_config(server_root) - if self.version[1] != 4: - raise errors.Error("Apache version not 2.4") - - with open(self.test_conf, "a") as f: - for module in self.modules: - if module not in STATIC_MODULES: - if module in SHARED_MODULES: - f.write( - "LoadModule {0}_module /usr/local/apache2/modules/" - "mod_{0}.so\n".format(module)) - else: - raise errors.Error( - "Unsupported module {0}".format(module)) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 9148666fc..ed3d9d67a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -1,6 +1,7 @@ """Provides a common base for Apache proxies""" import re import os +import shutil import subprocess import mock @@ -9,6 +10,7 @@ import zope.interface from certbot import configuration from certbot import errors as le_errors from certbot_apache import configurator +from certbot_apache import constants from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces from certbot_compatibility_test import util @@ -29,58 +31,14 @@ class Proxy(configurators_common.Proxy): super(Proxy, self).__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" - self._setup_mock() - self.modules = self.server_root = self.test_conf = self.version = None self._apache_configurator = self._all_names = self._test_names = None - - def _setup_mock(self): - """Replaces specific modules with mock.MagicMock""" - mock_subprocess = mock.MagicMock() - mock_subprocess.check_call = self.check_call - mock_subprocess.Popen = self.popen - - mock.patch( - "certbot_apache.configurator.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.parser.subprocess", - mock_subprocess).start() - mock.patch( - "certbot.util.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.configurator.util.exe_exists", - _is_apache_command).start() - patch = mock.patch( "certbot_apache.configurator.display_ops.select_vhost") mock_display = patch.start() mock_display.side_effect = le_errors.PluginError( "Unable to determine vhost") - def check_call(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.check_call is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).check_call(command, *args, **kwargs) - else: - return subprocess.check_call(command, *args, **kwargs) - - def popen(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.Popen is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).popen(command, *args, **kwargs) - else: - return subprocess.Popen(command, *args, **kwargs) - def __getattr__(self, name): """Wraps the Apache Configurator methods""" method = getattr(self._apache_configurator, name, None) @@ -91,29 +49,20 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - if hasattr(self.le_config, "apache_init_script"): - try: - self.check_call([self.le_config.apache_init_script, "stop"]) - except errors.Error: - raise errors.Error( - "Failed to stop previous apache config from running") config = super(Proxy, self).load_config() - self.modules = _get_modules(config) - self.version = _get_version(config) self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) - with open(os.path.join(config, "config_file")) as f: - config_file = os.path.join(server_root, f.readline().rstrip()) - self.test_conf = _create_test_conf(server_root, config_file) + # with open(os.path.join(config, "config_file")) as f: + # config_file = os.path.join(server_root, f.readline().rstrip()) + shutil.rmtree("/etc/apache2") + shutil.copytree(server_root, "/etc/apache2", symlinks=True) - self.preprocess_config(server_root) - self._prepare_configurator(server_root, config_file) + self._prepare_configurator() try: - self.check_call("apachectl -d {0} -f {1} -k start".format( - server_root, config_file)) + subprocess.check_call("apachectl -k start".split()) except errors.Error: raise errors.Error( "Apache failed to load {0} before tests started".format( @@ -121,34 +70,13 @@ class Proxy(configurators_common.Proxy): return config - def preprocess_config(self, server_root): - # pylint: disable=anomalous-backslash-in-string, no-self-use - """Prepares the configuration for use in the Docker""" - - find = subprocess.Popen( - ["find", server_root, "-type", "f"], - stdout=subprocess.PIPE) - subprocess.check_call([ - "xargs", "sed", "-e", "s/DocumentRoot.*/DocumentRoot " - "\/usr\/local\/apache2\/htdocs/I", - "-e", "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I", - "-e", "s/TypesConfig.*/TypesConfig " - "\/usr\/local\/apache2\/conf\/mime.types/I", - "-e", "s/LoadModule/#LoadModule/I", - "-e", "s/SSLCertificateFile.*/SSLCertificateFile " - "\/usr\/local\/apache2\/conf\/empty_cert.pem/I", - "-e", "s/SSLCertificateKeyFile.*/SSLCertificateKeyFile " - "\/usr\/local\/apache2\/conf\/rsa1024_key2.pem/I", - "-i"], stdin=find.stdout) - - def _prepare_configurator(self, server_root, config_file): + def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - self.le_config.apache_server_root = server_root - self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format( - server_root, config_file) - self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root) - self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root) - self.le_config.apache_init_script = self.le_config.apache_ctl + " -k" + for k in constants.CLI_DEFAULTS_DEBIAN.keys(): + setattr(self.le_config, "apache_" + k, constants.os_constant(k)) + + # An alias + self.le_config.apache_handle_modules = self.le_config.apache_handle_mods self._apache_configurator = configurator.ApacheConfigurator( config=configuration.NamespaceConfig(self.le_config), @@ -183,39 +111,6 @@ class Proxy(configurators_common.Proxy): domain, cert_path, key_path, chain_path, fullchain_path) -def _is_apache_command(command): - """Returns true if command is an Apache command""" - if isinstance(command, list): - command = command[0] - - for apache_command in APACHE_COMMANDS: - if command.startswith(apache_command): - return True - - return False - - -def _modify_command(command): - """Modifies command so configtest works inside the docker image""" - if isinstance(command, list): - for i in xrange(len(command)): - if command[i] == "configtest": - command[i] = "-t" - else: - command = command.replace("configtest", "-t") - - return command - - -def _create_test_conf(server_root, apache_config): - """Creates a test config file and adds it to the Apache config""" - test_conf = os.path.join(server_root, "test.conf") - open(test_conf, "w").close() - subprocess.check_call( - ["sed", "-i", "1iInclude test.conf", apache_config]) - return test_conf - - def _get_server_root(config): """Returns the server root directory in config""" subdirs = [ @@ -223,7 +118,7 @@ def _get_server_root(config): if os.path.isdir(os.path.join(config, name))] if len(subdirs) != 1: - errors.Error("Malformed configuration directiory {0}".format(config)) + errors.Error("Malformed configuration directory {0}".format(config)) return os.path.join(config, subdirs[0].rstrip()) @@ -251,34 +146,3 @@ def _get_names(config): words[1].find(".") != -1): all_names.add(words[1]) return all_names, non_ip_names - - -def _get_modules(config): - """Returns the list of modules found in module_list""" - modules = [] - with open(os.path.join(config, "modules")) as f: - for line in f: - # Modules list is indented, everything else is headers/footers - if line[0].isspace(): - words = line.split() - # Modules redundantly end in "_module" which we can discard - modules.append(words[0][:-7]) - - return modules - - -def _get_version(config): - """Return version of Apache Server. - - Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from - the Apache plugin. - - """ - with open(os.path.join(config, "version")) as f: - # Should be on first line of input - matches = APACHE_VERSION_REGEX.findall(f.readline()) - - if len(matches) != 1: - raise errors.Error("Unable to find Apache version") - - return tuple([int(i) for i in matches[0].split(".")]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 4657883a3..03128cc86 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -4,10 +4,7 @@ import os import shutil import tempfile -import docker - from certbot import constants -from certbot_compatibility_test import errors from certbot_compatibility_test import util @@ -18,20 +15,9 @@ class Proxy(object): # pylint: disable=too-many-instance-attributes """A common base for compatibility test configurators""" - _NOT_ADDED_ARGS = True - @classmethod def add_parser_arguments(cls, parser): """Adds command line arguments needed by the plugin""" - if Proxy._NOT_ADDED_ARGS: - group = parser.add_argument_group("docker") - group.add_argument( - "--docker-url", default="unix://var/run/docker.sock", - help="URL of the docker server") - group.add_argument( - "--no-remove", action="store_true", - help="do not delete container on program exit") - Proxy._NOT_ADDED_ARGS = False def __init__(self, args): """Initializes the plugin with the given command line args""" @@ -43,10 +29,8 @@ class Proxy(object): for config in os.listdir(config_dir)] self.args = args - self._docker_client = docker.Client( - base_url=self.args.docker_url, version="auto") - self.http_port, self.https_port = util.get_two_free_ports() - self._container_id = None + self.http_port = 80 + self.https_port = 443 def has_more_configs(self): """Returns true if there are more configs to test""" @@ -54,9 +38,6 @@ class Proxy(object): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - self._docker_client.stop(self._container_id, 0) - if not self.args.no_remove: - self._docker_client.remove_container(self._container_id) def load_config(self): """Returns the next config directory to be tested""" @@ -65,67 +46,6 @@ class Proxy(object): os.makedirs(backup) return self._configs.pop() - def start_docker(self, image_name, command): - """Creates and runs a Docker container with the specified image""" - logger.warning("Pulling Docker image. This may take a minute.") - for line in self._docker_client.pull(image_name, stream=True): - logger.debug(line) - - host_config = docker.utils.create_host_config( - binds={self._temp_dir: {"bind": self._temp_dir, "mode": "rw"}}, - port_bindings={ - 80: ("127.0.0.1", self.http_port), - 443: ("127.0.0.1", self.https_port)},) - container = self._docker_client.create_container( - image_name, command, ports=[80, 443], volumes=self._temp_dir, - host_config=host_config) - if container["Warnings"]: - logger.warning(container["Warnings"]) - self._container_id = container["Id"] - self._docker_client.start(self._container_id) - - def check_call(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to check_call but executes the command in the - running docker image - - """ - if self.popen(command).returncode: - raise errors.Error( - "{0} exited with a nonzero value".format(command)) - - def popen(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to Popen but executes the command in the - running docker image - - """ - class SimplePopen(object): - # pylint: disable=too-few-public-methods - """Simplified Popen object""" - def __init__(self, returncode, output): - self.returncode = returncode - self._stdout = output - self._stderr = output - - def communicate(self): - """Returns stdout and stderr""" - return self._stdout, self._stderr - - if isinstance(command, list): - command = " ".join(command) - - returncode, output = self.execute_in_docker(command) - return SimplePopen(returncode, output) - - def execute_in_docker(self, command): - """Executes command inside the running docker image""" - logger.debug("Executing '%s'", command) - exec_id = self._docker_client.exec_create(self._container_id, command) - output = self._docker_client.exec_start(exec_id) - returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"] - return returncode, output - def copy_certs_and_keys(self, cert_path, key_path, chain_path=None): """Copies certs and keys into the temporary directory""" cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys") diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 6823dfdab..38abffb18 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -7,6 +7,7 @@ import os import shutil import tempfile import time +import sys import OpenSSL @@ -21,17 +22,17 @@ from certbot_compatibility_test import errors from certbot_compatibility_test import util from certbot_compatibility_test import validator -from certbot_compatibility_test.configurators.apache import apache24 +from certbot_compatibility_test.configurators.apache import common DESCRIPTION = """ -Tests Certbot plugins against different server configuratons. It is -assumed that Docker is already installed. If no test types is specified, all +Tests Certbot plugins against different server configurations. It is +assumed that Docker is already installed. If no test type is specified, all tests that the plugin supports are performed. """ -PLUGINS = {"apache": apache24.Proxy} +PLUGINS = {"apache": common.Proxy} logger = logging.getLogger(__name__) @@ -61,8 +62,8 @@ def test_authenticator(plugin, config, temp_dir): "Plugin failed to complete %s for %s in %s", type(achalls[i]), achalls[i].domain, config) success = False - elif isinstance(responses[i], challenges.TLSSNI01): - verify = functools.partial(responses[i].simple_verify, achalls[i], + elif isinstance(responses[i], challenges.TLSSNI01Response): + verify = functools.partial(responses[i].simple_verify, achalls[i].chall, achalls[i].domain, util.JWK.public_key(), host="127.0.0.1", @@ -142,7 +143,8 @@ def test_deploy_cert(plugin, temp_dir, domains): for domain in domains: try: - plugin.deploy_cert(domain, cert_path, util.KEY_PATH) + plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path) + plugin.save() # Needed by the Apache plugin except le_errors.Error as error: logger.error("Plugin failed to deploy ceritificate for %s:", domain) logger.exception(error) @@ -177,6 +179,7 @@ def test_enhancements(plugin, domains): for domain in domains: try: plugin.enhance(domain, "redirect") + plugin.save() # Needed by the Apache plugin except le_errors.PluginError as error: # Don't immediately fail because a redirect may already be enabled logger.warning("Plugin failed to enable redirect for %s:", domain) @@ -341,7 +344,7 @@ def main(): temp_dir = tempfile.mkdtemp() plugin = PLUGINS[args.plugin](args) try: - plugin.execute_in_docker("mkdir -p /var/log/apache2") + overall_success = True while plugin.has_more_configs(): success = True @@ -360,10 +363,18 @@ def main(): if success: logger.info("All tests on %s succeeded", config) else: + overall_success = False logger.error("Tests on %s failed", config) finally: plugin.cleanup_from_tests() + if overall_success: + logger.warn("All compatibility tests succeeded") + sys.exit(0) + else: + logger.warn("One or more compatibility tests failed") + sys.exit(1) + if __name__ == "__main__": main() diff --git a/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz b/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz index 7323acf74..9b819d0c7 100644 Binary files a/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz and b/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz differ diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index cbce4fb56..af951aa6a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -1,11 +1,9 @@ """Utility functions for Certbot plugin tests.""" import argparse import copy -import contextlib import os import re import shutil -import socket import tarfile from acme import jose @@ -52,13 +50,3 @@ def extract_configs(configs, parent_dir): raise errors.Error("Unknown configurations file type") return config_dir - - -def get_two_free_ports(): - """Returns two free ports to use for the tests""" - with contextlib.closing(socket.socket()) as sock1: - with contextlib.closing(socket.socket()) as sock2: - sock1.bind(("", 0)) - sock2.bind(("", 0)) - - return sock1.getsockname()[1], sock2.getsockname()[1] diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index fe2c0c9d0..fb56be65f 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -7,9 +7,8 @@ from setuptools import find_packages version = '0.9.0.dev0' install_requires = [ - 'certbot=={0}'.format(version), - 'certbot-apache=={0}'.format(version), - 'docker-py', + 'certbot', + 'certbot-apache', 'requests', 'zope.interface', ] diff --git a/certbot/cli.py b/certbot/cli.py index 643602a44..cff111f42 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -359,7 +359,8 @@ class HelpfulArgumentParser(object): " {0} conflicts with dialog_mode").format(arg) ) - hooks.validate_hooks(parsed_args) + if parsed_args.validate_hooks: + hooks.validate_hooks(parsed_args) return parsed_args @@ -800,6 +801,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "For this command, the shell variable $RENEWED_LINEAGE will point to the" "config live subdirectory containing the new certs and keys; the shell variable " "$RENEWED_DOMAINS will contain a space-delimited list of renewed cert domains") + helpful.add( + "renew", "--disable-hook-validation", + action='store_false', dest='validate_hooks', default=True, + help="Ordinarily the commands specified for --pre-hook/--post-hook/--renew-hook" + " will be checked for validity, to see if the programs being run are in the $PATH," + " so that mistakes can be caught early, even when the hooks aren't being run just yet." + " The validation is rather simplistic and fails if you use more advanced" + " shell constructs, so you can use this switch to disable it.") helpful.add_deprecated_argument("--agree-dev-preview", 0) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 6b1b8426c..1e831dd8f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -296,6 +296,32 @@ def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): csr, OpenSSL.crypto.load_certificate_request, typ) +def _get_names_from_cert_or_req(cert_or_req, load_func, typ): + loaded_cert_or_req = _load_cert_or_req(cert_or_req, load_func, typ) + common_name = loaded_cert_or_req.get_subject().CN + # pylint: disable=protected-access + sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + + +def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): + """Get a list of domains from a cert, including the CN if it is set. + + :param str cert: Certificate (encoded). + :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + + :returns: A list of domain names. + :rtype: list + + """ + return _get_names_from_cert_or_req( + csr, OpenSSL.crypto.load_certificate, typ) + + def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): """Get a list of domains from a CSR, including the CN if it is set. @@ -306,13 +332,8 @@ def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): :rtype: list """ - loaded_csr = _load_cert_or_req( + return _get_names_from_cert_or_req( csr, OpenSSL.crypto.load_certificate_request, typ) - # Use a set to avoid duplication with CN and Subject Alt Names - domains = set(d for d in (loaded_csr.get_subject().CN,) if d is not None) - # pylint: disable=protected-access - domains.update(acme_crypto_util._pyopenssl_cert_or_req_san(loaded_csr)) - return list(domains) def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): diff --git a/certbot/storage.py b/certbot/storage.py index b0c8245d3..60886e306 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -616,7 +616,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes if target is None: raise errors.CertStorageError("could not find cert file") with open(target) as f: - return crypto_util.get_sans_from_cert(f.read()) + return crypto_util.get_names_from_cert(f.read()) def autodeployment_is_enabled(self): """Is automatic deployment enabled for this cert? diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index b40b98988..f9557abfb 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -651,6 +651,18 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods out = stdout.getvalue() self.assertEqual("", out) + def test_renew_hook_validation(self): + self._make_test_renewal_conf('sample-renewal.conf') + args = ["renew", "--dry-run", "--post-hook=no-such-command"] + self._test_renewal_common(True, [], args=args, should_renew=False, + error_expected=True) + + def test_renew_no_hook_validation(self): + self._make_test_renewal_conf('sample-renewal.conf') + args = ["renew", "--dry-run", "--post-hook=no-such-command", + "--disable-hook-validation"] + self._test_renewal_common(True, [], args=args, should_renew=True, + error_expected=False) @mock.patch("certbot.cli.set_by_cli") def test_ancient_webroot_renewal_conf(self, mock_set_by_cli): diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index fa88e89e7..5a592bbb1 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -273,6 +273,32 @@ class GetSANsFromCSRTest(unittest.TestCase): [], self._call(test_util.load_vector('csr-nosans.pem'))) +class GetNamesFromCertTest(unittest.TestCase): + """Tests for certbot.crypto_util.get_names_from_cert.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.crypto_util import get_names_from_cert + return get_names_from_cert(*args, **kwargs) + + def test_single(self): + self.assertEqual( + ['example.com'], + self._call(test_util.load_vector('cert.pem'))) + + def test_san(self): + self.assertEqual( + ['example.com', 'www.example.com'], + self._call(test_util.load_vector('cert-san.pem'))) + + def test_common_name_sans_order(self): + # Tests that the common name comes first + # followed by the SANS in alphabetical order + self.assertEqual( + ['example.com'] + ['{0}.example.com'.format(c) for c in 'abcd'], + self._call(test_util.load_vector('cert-5sans.pem'))) + + class GetNamesFromCSRTest(unittest.TestCase): """Tests for certbot.crypto_util.get_names_from_csr.""" @classmethod diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0c88d3d55..0d907eca3 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -84,18 +84,20 @@ class BaseRenewableCertTest(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) + def _write_out_kind(self, kind, ver, value=None): + link = getattr(self.test_rc, kind) + if os.path.lexists(link): + os.unlink(link) + os.symlink(os.path.join(os.path.pardir, os.path.pardir, "archive", + "example.org", "{0}{1}.pem".format(kind, ver)), + link) + with open(link, "w") as f: + f.write(kind if value is None else value) + def _write_out_ex_kinds(self): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}12.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}11.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 12) + self._write_out_kind(kind, 11) class RenewableCertTests(BaseRenewableCertTest): @@ -204,10 +206,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_current_target(self): # Relative path logic - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert17.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") + self._write_out_kind("cert", 17) self.assertTrue(os.path.samefile(self.test_rc.current_target("cert"), os.path.join(self.tempdir, "archive", "example.org", @@ -225,12 +224,8 @@ class RenewableCertTests(BaseRenewableCertTest): def test_current_version(self): for ver in (1, 5, 10, 20): - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert{0}.pem".format(ver)), - self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") - os.unlink(self.test_rc.cert) + self._write_out_kind("cert", ver) + os.unlink(self.test_rc.cert) os.symlink(os.path.join("..", "..", "archive", "example.org", "cert10.pem"), self.test_rc.cert) self.assertEqual(self.test_rc.current_version("cert"), 10) @@ -241,61 +236,30 @@ class RenewableCertTests(BaseRenewableCertTest): def test_latest_and_next_versions(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(self.test_rc.latest_common_version(), 5) self.assertEqual(self.test_rc.next_free_version(), 6) # Having one kind of file of a later version doesn't change the # result - os.unlink(self.test_rc.privkey) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "privkey7.pem"), self.test_rc.privkey) - with open(self.test_rc.privkey, "w") as f: - f.write("privkey") + self._write_out_kind("privkey", 7) self.assertEqual(self.test_rc.latest_common_version(), 5) # ... although it does change the next free version self.assertEqual(self.test_rc.next_free_version(), 8) # Nor does having three out of four change the result - os.unlink(self.test_rc.cert) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert7.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") - os.unlink(self.test_rc.fullchain) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "fullchain7.pem"), self.test_rc.fullchain) - with open(self.test_rc.fullchain, "w") as f: - f.write("fullchain") + self._write_out_kind("cert", 7) + self._write_out_kind("fullchain", 7) self.assertEqual(self.test_rc.latest_common_version(), 5) # If we have everything from a much later version, it does change # the result - ver = 17 for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 17) self.assertEqual(self.test_rc.latest_common_version(), 17) self.assertEqual(self.test_rc.next_free_version(), 18) def test_update_link_to(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) # pylint: disable=protected-access self.test_rc._update_link_to("cert", 3) @@ -312,10 +276,7 @@ class RenewableCertTests(BaseRenewableCertTest): "chain3000.pem") def test_version(self): - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert12.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") + self._write_out_kind("cert", 12) # TODO: We should probably test that the directory is still the # same, but it's tricky because we can get an absolute # path out when we put a relative path in. @@ -325,13 +286,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_update_all_links_to_success(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) self.assertEqual(self.test_rc.latest_common_version(), 5) for ver in xrange(1, 6): @@ -376,13 +331,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_has_pending_deployment(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) for ver in xrange(1, 6): self.test_rc.update_all_links_to(ver) @@ -395,24 +344,22 @@ class RenewableCertTests(BaseRenewableCertTest): def test_names(self): # Trying the current version - test_cert = test_util.load_vector("cert-san.pem") - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert12.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write(test_cert) + self._write_out_kind("cert", 12, test_util.load_vector("cert-san.pem")) self.assertEqual(self.test_rc.names(), ["example.com", "www.example.com"]) # Trying a non-current version - test_cert = test_util.load_vector("cert.pem") - os.unlink(self.test_rc.cert) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert15.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write(test_cert) + self._write_out_kind("cert", 15, test_util.load_vector("cert.pem")) self.assertEqual(self.test_rc.names(12), ["example.com", "www.example.com"]) + # Testing common name is listed first + self._write_out_kind( + "cert", 12, test_util.load_vector("cert-5sans.pem")) + self.assertEqual( + self.test_rc.names(12), + ["example.com"] + ["{0}.example.com".format(c) for c in "abcd"]) + # Trying missing cert os.unlink(self.test_rc.cert) self.assertRaises(errors.CertStorageError, self.test_rc.names) @@ -480,13 +427,7 @@ class RenewableCertTests(BaseRenewableCertTest): # No pending deployment for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertFalse(self.test_rc.should_autodeploy()) def test_autorenewal_is_enabled(self): @@ -507,11 +448,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(self.test_rc.should_autorenew()) self.test_rc.configuration["autorenew"] = "1" for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}12.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 12) # Mandatory renewal on the basis of OCSP revocation mock_ocsp.return_value = True self.assertTrue(self.test_rc.should_autorenew()) @@ -525,13 +462,7 @@ class RenewableCertTests(BaseRenewableCertTest): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) self.assertEqual( 6, self.test_rc.save_successor(3, "new cert", None, diff --git a/certbot/tests/testdata/cert-5sans.pem b/certbot/tests/testdata/cert-5sans.pem new file mode 100644 index 000000000..5de7cc6cb --- /dev/null +++ b/certbot/tests/testdata/cert-5sans.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICkTCCAjugAwIBAgIJAJNbfABWQ8bbMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMScwJQYDVQQKDB5FbGVjdHJvbmljIEZyb250aWVyIEZvdW5kYXRpb24xFDAS +BgNVBAMMC2V4YW1wbGUuY29tMB4XDTE2MDYwOTIzMDEzNloXDTE2MDcwOTIzMDEz +NloweTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xJzAlBgNVBAoMHkVsZWN0cm9uaWMgRnJvbnRpZXIgRm91 +bmRhdGlvbjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0BAQEFAANL +ADBIAkEArHVztFHtH92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580rv2+6QWE +30cWgdmJS86ObRz6lUTor4R0T+3C5QIDAQABo4GlMIGiMB0GA1UdDgQWBBQmz8jt +S9eUsuQlA1gkjwTAdNWXijAfBgNVHSMEGDAWgBQmz8jtS9eUsuQlA1gkjwTAdNWX +ijAMBgNVHRMEBTADAQH/MFIGA1UdEQRLMEmCDWEuZXhhbXBsZS5jb22CDWIuZXhh +bXBsZS5jb22CDWMuZXhhbXBsZS5jb22CDWQuZXhhbXBsZS5jb22CC2V4YW1wbGUu +Y29tMA0GCSqGSIb3DQEBCwUAA0EAVXmZxB+IJdgFvY2InOYeytTD1QmouDZRtj/T +H/HIpSdsfO7qr4d/ZprI2IhLRxp2S4BiU5Qc5HUkeADcpNd06A== +-----END CERTIFICATE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 89d345062..569f6d6f3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -476,7 +476,7 @@ Bootstrap() { BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" + echo "# pacman -S certbot certbot-apache" echo echo "If you would like to use the virtualenv way, please run the script again with the" echo "--debug flag." diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 43d8bc7e1..73d819b4a 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -173,7 +173,7 @@ Bootstrap() { BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" + echo "# pacman -S certbot certbot-apache" echo echo "If you would like to use the virtualenv way, please run the script again with the" echo "--debug flag."