From df850ee980960abb79b09fb6139334439891f2ea Mon Sep 17 00:00:00 2001 From: Dominic Date: Wed, 19 Nov 2014 09:33:31 +0100 Subject: [PATCH 01/52] Reduce the matching of REWRITE_HTTPS_ARGS and add query string If the backreference of the match is not used, it's enough to match '^' instead of '^.*$'. It's slightly faster. ^ -> Match, if it starts ^.*$` -> Match, if everything matches In addition it might be useful to append the query string with the flag: QSA --- letsencrypt/client/CONFIG.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/client/CONFIG.py b/letsencrypt/client/CONFIG.py index 8c7c4b6d4..910e5c881 100644 --- a/letsencrypt/client/CONFIG.py +++ b/letsencrypt/client/CONFIG.py @@ -56,4 +56,4 @@ CHALLENGE_PREFERENCES = ["dvsni", "recoveryToken"] EXCLUSIVE_CHALLENGES = [set(["dvsni", "simpleHttps"])] # Rewrite rule arguments used for redirections to https vhost -REWRITE_HTTPS_ARGS = ["^.*$", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,R=permanent]"] +REWRITE_HTTPS_ARGS = ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] From 0f7804cd94934373c44f0565ebaa99794ee86b49 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 1 Jul 2015 16:50:48 -0700 Subject: [PATCH 02/52] Initial modules and runtime parameter parsing --- letsencrypt/cli.py | 1 - letsencrypt_apache/augeas_configurator.py | 6 ++- letsencrypt_apache/configurator.py | 14 ++--- letsencrypt_apache/parser.py | 65 +++++++++++++++++++---- letsencrypt_apache/tests/util.py | 9 +++- 5 files changed, 74 insertions(+), 21 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e64044077..2ef05ff9f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -644,7 +644,6 @@ def _setup_logging(args): def main2(cli_args, args, config, plugins): """Continued main script execution.""" - # Displayer if args.text_mode: displayer = display_util.FileDisplay(sys.stdout) diff --git a/letsencrypt_apache/augeas_configurator.py b/letsencrypt_apache/augeas_configurator.py index ffe363035..ea4bd0384 100644 --- a/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt_apache/augeas_configurator.py @@ -3,6 +3,7 @@ import logging import augeas +from letsencrypt import errors from letsencrypt import reverter from letsencrypt.plugins import common @@ -23,7 +24,6 @@ class AugeasConfigurator(common.Plugin): :type reverter: :class:`letsencrypt.reverter.Reverter` """ - def __init__(self, *args, **kwargs): super(AugeasConfigurator, self).__init__(*args, **kwargs) @@ -54,10 +54,12 @@ class AugeasConfigurator(common.Plugin): lens_path = self.aug.get(path + "/lens") # As aug.get may return null if lens_path and lens in lens_path: - logger.error( + msg = ( "There has been an error in parsing the file (%s): %s", # Strip off /augeas/files and /error path[13:len(path) - 6], self.aug.get(path + "/message")) + logger.error(msg) + raise errors.PluginError(msg) def save(self, title=None, temporary=False): """Saves all changes to the configuration files. diff --git a/letsencrypt_apache/configurator.py b/letsencrypt_apache/configurator.py index c8083b406..d9491d98a 100644 --- a/letsencrypt_apache/configurator.py +++ b/letsencrypt_apache/configurator.py @@ -93,8 +93,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): add("server-root", default=constants.CLI_DEFAULTS["server_root"], help="Apache server root directory.") add("ctl", default=constants.CLI_DEFAULTS["ctl"], - help="Path to the 'apache2ctl' binary, used for 'configtest' and " - "retrieving Apache2 version number.") + help="Path to the 'apache2ctl' binary, used for 'configtest', " + "retrieving the Apache2 version number, and initialization " + "parameters.") add("enmod", default=constants.CLI_DEFAULTS["enmod"], help="Path to the Apache 'a2enmod' binary.") add("init-script", default=constants.CLI_DEFAULTS["init_script"], @@ -136,7 +137,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def prepare(self): """Prepare the authenticator/installer.""" self.parser = parser.ApacheParser( - self.aug, self.conf("server-root"), self.mod_ssl_conf) + self.aug, self.conf("server-root"), self.mod_ssl_conf, + self.conf("ctl")) # Check for errors in parsing files with Augeas self.check_parsing_errors("httpd.aug") @@ -528,9 +530,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError("Only one vhost per file is allowed") self.parser.add_dir(vh_p[0], "SSLCertificateFile", - "/etc/ssl/certs/ssl-cert-snakeoil.pem") + "insert_cert_file_path") self.parser.add_dir(vh_p[0], "SSLCertificateKeyFile", - "/etc/ssl/private/ssl-cert-snakeoil.key") + "insert_key_file_path") self.parser.add_dir(vh_p[0], "Include", self.parser.loc["ssl_options"]) # Log actions and create save notes @@ -935,7 +937,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): subprocess.check_call([self.conf("enmod"), mod_name], stdout=open("/dev/null", "w"), stderr=open("/dev/null", "w")) - apache_restart(self.conf("init")) + apache_restart(self.conf("init_script")) except (OSError, subprocess.CalledProcessError): logger.exception("Error enabling mod_%s", mod_name) raise errors.MisconfigurationError( diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index caec69b57..9233c6bdd 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -1,10 +1,16 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" +import itertools +import logging import os import re +import subprocess from letsencrypt import errors +logger = logging.getLogger(__name__) + + class ApacheParser(object): """Class handles the fine details of parsing the Apache Configuration. @@ -12,8 +18,7 @@ class ApacheParser(object): directory. Without trailing slash. """ - - def __init__(self, aug, root, ssl_options): + def __init__(self, aug, root, ssl_options, ctl): # Find configuration root and make sure augeas can parse it. self.aug = aug self.root = os.path.abspath(root) @@ -26,6 +31,46 @@ class ApacheParser(object): # This problem has been fixed in Augeas 1.0 self.standardize_excl() + self.modules = self._init_modules() + self.parameters = self._init_parameters(ctl) + + def _init_modules(self): + matches = self.find_dir(case_i("LoadModule")) + + iterator = iter(matches) + + modules = set() + for match_name, match_filename in itertools.izip(iterator, iterator): + modules.add(self.aug.get(match_name)) + modules.add( + os.path.basename(self.aug.get(match_filename))[:-2] + "c") + + return modules + + def _init_parameters(self, ctl): + try: + proc = subprocess.Popen( + [ctl, "-D", "DUMP_RUN_CFG"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + + except (OSError, ValueError): + logger.error( + "Error accessing {0} for runtime parameters!{1}".format( + ctl, os.linesep)) + raise errors.MisconfigurationError( + "Error accessing loaded Apache parameters: %s", ctl) + # Small errors that do not impede + if proc.returncode != 0: + logger.warn("Error in checking parameter list: %s", stderr) + raise errors.MisconfigurationError( + "Apache is unable to check whether or not the module is " + "loaded because Apache is misconfigured.") + + matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) + matches.remove("DUMP_RUN_CFG") + return set(matches) def add_dir_to_ifmodssl(self, aug_conf_path, directive, val): """Adds directive and value to IfMod ssl block. @@ -135,8 +180,8 @@ class ApacheParser(object): "[self::arg=~regexp('%s')]" % (start, directive, arg))) - incl_regex = "(%s)|(%s)" % (case_i('Include'), - case_i('IncludeOptional')) + incl_regex = "(%s)|(%s)" % (case_i("Include"), + case_i("IncludeOptional")) includes = self.aug.match(("%s//* [self::directive=~regexp('%s')]/* " "[label()='arg']" % (start, incl_regex))) @@ -233,13 +278,13 @@ class ApacheParser(object): # Checkout fnmatch.py in venv/local/lib/python2.7/fnmatch.py regex = "" for letter in clean_fn_match: - if letter == '.': + if letter == ".": regex = regex + r"\." - elif letter == '*': + elif letter == "*": regex = regex + ".*" # According to apache.org ? shouldn't appear # but in case it is valid... - elif letter == '?': + elif letter == "?": regex = regex + "." else: regex = regex + letter @@ -360,12 +405,12 @@ class ApacheParser(object): # Basic check to see if httpd.conf exists and # in hierarchy via direct include # httpd.conf was very common as a user file in Apache 2.2 - if (os.path.isfile(os.path.join(self.root, 'httpd.conf')) and + if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and self.find_dir( case_i("Include"), case_i("httpd.conf"), root)): - return os.path.join(self.root, 'httpd.conf') + return os.path.join(self.root, "httpd.conf") else: - return os.path.join(self.root, 'apache2.conf') + return os.path.join(self.root, "apache2.conf") def case_i(string): diff --git a/letsencrypt_apache/tests/util.py b/letsencrypt_apache/tests/util.py index 0769f6050..734393903 100644 --- a/letsencrypt_apache/tests/util.py +++ b/letsencrypt_apache/tests/util.py @@ -35,9 +35,13 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods def get_apache_configurator( - config_path, config_dir, work_dir, version=(2, 4, 7)): - """Create an Apache Configurator with the specified options.""" + config_path, config_dir, work_dir, version=(2, 4, 7), + conf=mock.MagicMock()): + """Create an Apache Configurator with the specified options. + :param conf: Function that returns binary paths. self.conf in Configurator + + """ backups = os.path.join(work_dir, "backups") with mock.patch("letsencrypt_apache.configurator." @@ -55,6 +59,7 @@ def get_apache_configurator( work_dir=work_dir), name="apache", version=version) + config.conf = conf config.prepare() From 24675bef5bc9eda5db2cef31a8cda9a1b1726753 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 1 Jul 2015 18:07:53 -0700 Subject: [PATCH 03/52] Initial exclude directives based on modules/parameters --- letsencrypt_apache/parser.py | 53 +++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index 9233c6bdd..d516866b7 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -19,6 +19,9 @@ class ApacheParser(object): """ def __init__(self, aug, root, ssl_options, ctl): + # This uses the binary, so it can be done first. + self.parameters = self._init_parameters(ctl) + # Find configuration root and make sure augeas can parse it. self.aug = aug self.root = os.path.abspath(root) @@ -31,10 +34,14 @@ class ApacheParser(object): # This problem has been fixed in Augeas 1.0 self.standardize_excl() + + # Temporarily set modules to be empty, so that find_dirs can work + self.modules = set() self.modules = self._init_modules() - self.parameters = self._init_parameters(ctl) def _init_modules(self): + # TODO: This needs to be iterative, until no new modules are loaded. + # This is due to ifmod... load mod behavior. matches = self.find_dir(case_i("LoadModule")) iterator = iter(matches) @@ -137,6 +144,8 @@ class ApacheParser(object): .. todo:: Add order to directives returned. Last directive comes last.. .. todo:: arg should probably be a list + .. todo:: Check //* notation for including directories not intended + to be included. Note: Augeas is inherently case sensitive while Apache is case insensitive. Augeas 1.0 allows case insensitive regexes like @@ -180,12 +189,16 @@ class ApacheParser(object): "[self::arg=~regexp('%s')]" % (start, directive, arg))) + matches = self._exclude_dirs(matches) + incl_regex = "(%s)|(%s)" % (case_i("Include"), case_i("IncludeOptional")) includes = self.aug.match(("%s//* [self::directive=~regexp('%s')]/* " "[label()='arg']" % (start, incl_regex))) + includes = self._exclude_dirs(includes) + # for inc in includes: # print inc, self.aug.get(inc) @@ -197,6 +210,44 @@ class ApacheParser(object): return matches + def _exclude_dirs(self, matches): + """Exclude directives that are not loaded into the configuration.""" + filters = [("ifmodule", self.modules), ("ifdefine", self.parameters)] + + valid_matches = [] + + for match in matches: + for filter in filters: + if not self._pass_filter(match, filter): + break + else: + valid_matches.append(match) + return valid_matches + + def _pass_filter(self, match, filter): + """Determine if directive passes a filter. + + :param str match: Augeas path + :param list filter: list of tuples of form + [("lowercase if directive", set of relevant parameters)] + + """ + match_l = match.lower() + last_match_idx = match_l.find(filter[0]) + + while last_match_idx != -1: + # Check args + end_of_if = match_l.find("/", last_match_idx) + expression = self.aug.get(match[:end_of_if] + "/arg") + + expected = not expression.startswith("!") + if expected != expression in filter[1]: + return False + + last_match_idx = match_l.find(filter[0], end_of_if) + + return True + def _get_include_path(self, cur_dir, arg): """Converts an Apache Include directive into Augeas path. From 5c24a4f499aff781e8879428a4648ae4ce2c1125 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 2 Jul 2015 15:18:39 -0700 Subject: [PATCH 04/52] Iterate on loaded modules --- letsencrypt_apache/parser.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index d516866b7..4a37bc484 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -37,22 +37,24 @@ class ApacheParser(object): # Temporarily set modules to be empty, so that find_dirs can work self.modules = set() - self.modules = self._init_modules() + self._init_modules() def _init_modules(self): - # TODO: This needs to be iterative, until no new modules are loaded. - # This is due to ifmod... load mod behavior. + """Iterates on the configuration until no new modules are loaded.""" matches = self.find_dir(case_i("LoadModule")) iterator = iter(matches) + # Make sure prev_size != cur_size for do: while: iteration + prev_size = -1 - modules = set() - for match_name, match_filename in itertools.izip(iterator, iterator): - modules.add(self.aug.get(match_name)) - modules.add( - os.path.basename(self.aug.get(match_filename))[:-2] + "c") + while len(self.modules) != prev_size: + prev_size = len(self.modules) - return modules + for match_name, match_filename in itertools.izip( + iterator, iterator): + self.modules.add(self.aug.get(match_name)) + self.modules.add( + os.path.basename(self.aug.get(match_filename))[:-2] + "c") def _init_parameters(self, ctl): try: From 9b263f9859bb8eed1fdde2f9cd8ae9a8edf7afdc Mon Sep 17 00:00:00 2001 From: James Kasten Date: Tue, 7 Jul 2015 13:18:22 -0700 Subject: [PATCH 05/52] outline init path --- letsencrypt_apache/parser.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index 4a37bc484..f02b34889 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -14,12 +14,17 @@ logger = logging.getLogger(__name__) class ApacheParser(object): """Class handles the fine details of parsing the Apache Configuration. - :ivar str root: Normalized abosulte path to the server root + :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. + .. todo:: Handle UnDefine Directive + .. todo:: Handle Define directive for parameters within find_dirs + """ def __init__(self, aug, root, ssl_options, ctl): # This uses the binary, so it can be done first. + # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine + # This only handles invocation parameters... not Define directive params self.parameters = self._init_parameters(ctl) # Find configuration root and make sure augeas can parse it. @@ -36,9 +41,15 @@ class ApacheParser(object): self.standardize_excl() # Temporarily set modules to be empty, so that find_dirs can work + # https://httpd.apache.org/docs/2.4/mod/core.html#ifmodule self.modules = set() self._init_modules() + # Set Apache variables + # https://httpd.apache.org/docs/2.4/mod/core.html#define + self.variables = set() + self._init_variables() + def _init_modules(self): """Iterates on the configuration until no new modules are loaded.""" matches = self.find_dir(case_i("LoadModule")) @@ -81,6 +92,14 @@ class ApacheParser(object): matches.remove("DUMP_RUN_CFG") return set(matches) + def _init_variables(self): + #print "Define Directive:", self.find_dir(case_i("Define")) + # This works + #matches = self.aug.match("/files/etc/apache2/apache2.conf/*/* [count(arg) = 2]") + matches = self.aug.match("/files/etc/apache2/apache2.conf/*[self::directive=~regexp('Define')]/arg") + for match in matches: + print match, self.aug.get(match) + def add_dir_to_ifmodssl(self, aug_conf_path, directive, val): """Adds directive and value to IfMod ssl block. From 9d17ac734789f45ff2c2cf52ba7782a8f778a515 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 8 Jul 2015 17:17:54 -0700 Subject: [PATCH 06/52] Use binary for all Define parameters --- letsencrypt_apache/parser.py | 89 ++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index f02b34889..d664bfaa6 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -1,4 +1,5 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" +import collections import itertools import logging import os @@ -23,9 +24,10 @@ class ApacheParser(object): """ def __init__(self, aug, root, ssl_options, ctl): # This uses the binary, so it can be done first. + # https://httpd.apache.org/docs/2.4/mod/core.html#define # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine - # This only handles invocation parameters... not Define directive params - self.parameters = self._init_parameters(ctl) + # This only handles invocation parameters and Define directives! + self.variables = self._init_runtime_variables(ctl) # Find configuration root and make sure augeas can parse it. self.aug = aug @@ -45,13 +47,13 @@ class ApacheParser(object): self.modules = set() self._init_modules() - # Set Apache variables - # https://httpd.apache.org/docs/2.4/mod/core.html#define - self.variables = set() - self._init_variables() - def _init_modules(self): - """Iterates on the configuration until no new modules are loaded.""" + """Iterates on the configuration until no new modules are loaded. + + ..todo:: This should be attempted to be done with a binary to avoid + the iteration issue. Else... do a better job of parsing to avoid it + + """ matches = self.find_dir(case_i("LoadModule")) iterator = iter(matches) @@ -67,7 +69,34 @@ class ApacheParser(object): self.modules.add( os.path.basename(self.aug.get(match_filename))[:-2] + "c") - def _init_parameters(self, ctl): + def _init_runtime_variables(self, ctl): + """" + + ..todo:: Also use apache2ctl -V for compiled parameters + + """ + stdout = self._get_runtime_info(ctl) + + variables = dict() + matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) + matches.remove("DUMP_RUN_CFG") + + for match in matches: + if match.count("=") > 1: + logger.error("Unexpected number of equal signs in " + "apache2ctl -D DUMP_RUN_CFG") + raise errors.PluginError( + "Error parsing Apache runtime variables") + parts = match.partition("=") + variables[parts[0]] = parts[2] + print variables + + def _get_runtime_cfg(self, ctl): + """Get runtime configuration info. + + :returns: stdout from DUMP_RUN_CFG + + """ try: proc = subprocess.Popen( [ctl, "-D", "DUMP_RUN_CFG"], @@ -88,17 +117,37 @@ class ApacheParser(object): "Apache is unable to check whether or not the module is " "loaded because Apache is misconfigured.") - matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) - matches.remove("DUMP_RUN_CFG") - return set(matches) + return stdout - def _init_variables(self): - #print "Define Directive:", self.find_dir(case_i("Define")) - # This works - #matches = self.aug.match("/files/etc/apache2/apache2.conf/*/* [count(arg) = 2]") - matches = self.aug.match("/files/etc/apache2/apache2.conf/*[self::directive=~regexp('Define')]/arg") - for match in matches: - print match, self.aug.get(match) + def _filter_args_num(self, matches, args): + """Filter out directives with specific number of arguments. + + This function makes the assumption that all related arguments are given + in order. Thus /files/apache/directive[5]/arg[2] must come immediately + after /files/apache/directive[5]/arg[1]. Runs in 1 linear pass. + + :param string matches: Matches of all directives with arg nodes + :param int args: Number of args you would like to filter + + :returns: List of directives that contain # of arguments. + (arg is stripped off) + + """ + filtered = [] + if args == 1: + for i in range(matches): + if matches[i].endswith("/arg"): + filtered.append(matches[i][:-4]) + else: + for i in range(matches): + if matches[i].endswith("/arg[%d]", args): + # Make sure we don't cause an IndexError (end of list) + # Check to make sure arg + 1 doesn't exist + if (i == (len(matches) - 1) or + not matches[i + 1].endswith("/arg[%d]" % args + 1)): + filtered.append(matches[i][:-len("/arg[%d]" % args)]) + + return filtered def add_dir_to_ifmodssl(self, aug_conf_path, directive, val): """Adds directive and value to IfMod ssl block. @@ -233,7 +282,7 @@ class ApacheParser(object): def _exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" - filters = [("ifmodule", self.modules), ("ifdefine", self.parameters)] + filters = [("ifmodule", self.modules), ("ifdefine", self.variables)] valid_matches = [] From ac32e5479829be0e26d9777c04b95f1abb04f743 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 9 Jul 2015 16:15:45 -0700 Subject: [PATCH 07/52] In order directive search --- letsencrypt_apache/configurator.py | 11 +++--- letsencrypt_apache/parser.py | 54 ++++++++++++++---------------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/letsencrypt_apache/configurator.py b/letsencrypt_apache/configurator.py index d9491d98a..33f7765a0 100644 --- a/letsencrypt_apache/configurator.py +++ b/letsencrypt_apache/configurator.py @@ -65,7 +65,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): is not ready to handle very complex configurations. .. todo:: Add support for config file variables Define rootDir /var/www/ - .. todo:: Add proper support for module configuration The API of this class will change in the coming weeks as the exact needs of clients are clarified with the new and developing protocol. @@ -198,14 +197,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) - self.aug.set(path["cert_path"][0], cert_path) - self.aug.set(path["cert_key"][0], key_path) + # Assign the final directives; order is maintained in find_dir + self.aug.set(path["cert_path"][-1], cert_path) + self.aug.set(path["cert_key"][-1], key_path) if chain_path is not None: if not path["chain_path"]: self.parser.add_dir( vhost.path, "SSLCertificateChainFile", chain_path) else: - self.aug.set(path["chain_path"][0], chain_path) + self.aug.set(path["chain_path"][-1], chain_path) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -430,8 +430,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check for Listen 443 # Note: This could be made to also look for ip:443 combo - # TODO: Need to search only open directives and IfMod mod_ssl.c - if len(self.parser.find_dir(parser.case_i("Listen"), "443")) == 0: + if not self.parser.find_dir(parser.case_i("Listen"), "443"): logger.debug("No Listen 443 directive found. Setting the " "Apache Server to Listen on port 443") path = self.parser.add_dir_to_ifmodssl( diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index d664bfaa6..be1aa7a52 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -15,12 +15,11 @@ logger = logging.getLogger(__name__) class ApacheParser(object): """Class handles the fine details of parsing the Apache Configuration. + .. todo:: Make parsing general... remove sites-available etc... + :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. - .. todo:: Handle UnDefine Directive - .. todo:: Handle Define directive for parameters within find_dirs - """ def __init__(self, aug, root, ssl_options, ctl): # This uses the binary, so it can be done first. @@ -75,7 +74,7 @@ class ApacheParser(object): ..todo:: Also use apache2ctl -V for compiled parameters """ - stdout = self._get_runtime_info(ctl) + stdout = self._get_runtime_cfg(ctl) variables = dict() matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) @@ -89,7 +88,8 @@ class ApacheParser(object): "Error parsing Apache runtime variables") parts = match.partition("=") variables[parts[0]] = parts[2] - print variables + + return variables def _get_runtime_cfg(self, ctl): """Get runtime configuration info. @@ -212,7 +212,6 @@ class ApacheParser(object): Recursively searches through config files to find directives Directives should be in the form of a case insensitive regex currently - .. todo:: Add order to directives returned. Last directive comes last.. .. todo:: arg should probably be a list .. todo:: Check //* notation for including directories not intended to be included. @@ -237,8 +236,6 @@ class ApacheParser(object): if not start: start = get_aug_path(self.loc["root"]) - # Debug code - # print "find_dir:", directive, "arg:", arg, " | Looking in:", start # No regexp code # if arg is None: # matches = self.aug.match(start + @@ -251,34 +248,33 @@ class ApacheParser(object): # includes = self.aug.match(start + # "//* [self::directive='Include']/* [label()='arg']") - if arg is None: - matches = self.aug.match(("%s//*[self::directive=~regexp('%s')]/arg" - % (start, directive))) - else: - matches = self.aug.match(("%s//*[self::directive=~regexp('%s')]/*" - "[self::arg=~regexp('%s')]" % - (start, directive, arg))) + regex = "(%s)|(%s)|(%s)" % (directive, + case_i("Include"), + case_i("IncludeOptional")) + matches = self.aug.match( + "%s//*[self::directive=~regexp('%s')]" % (start, regex)) matches = self._exclude_dirs(matches) - incl_regex = "(%s)|(%s)" % (case_i("Include"), - case_i("IncludeOptional")) - includes = self.aug.match(("%s//* [self::directive=~regexp('%s')]/* " - "[label()='arg']" % (start, incl_regex))) + if arg is None: + arg_suffix = "/arg" + else: + arg_suffix = "/*[self::arg=~regexp('%s')]" % arg - includes = self._exclude_dirs(includes) + ordered_matches = [] - # for inc in includes: - # print inc, self.aug.get(inc) + for match in matches: + dir = self.aug.get(match).lower() + if dir == "include" or dir == "includeoptional": + # start[6:] to strip off /files + ordered_matches.extend(self.find_dir( + directive, arg, self._get_include_path( + strip_dir(start[6:]), self.aug.get(match + "/arg")))) + else: + ordered_matches.extend(self.aug.match(match + arg_suffix)) - for include in includes: - # start[6:] to strip off /files - matches.extend(self.find_dir( - directive, arg, self._get_include_path( - strip_dir(start[6:]), self.aug.get(include)))) - - return matches + return ordered_matches def _exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" From a402382a4957234334f869589900644669bb512d Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 9 Jul 2015 17:37:32 -0700 Subject: [PATCH 08/52] Remove enable mod_ssl/configuration changes on prepare() --- letsencrypt_apache/configurator.py | 23 ++++++++++------------- letsencrypt_apache/dvsni.py | 3 +++ letsencrypt_apache/parser.py | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/letsencrypt_apache/configurator.py b/letsencrypt_apache/configurator.py index 33f7765a0..97588d483 100644 --- a/letsencrypt_apache/configurator.py +++ b/letsencrypt_apache/configurator.py @@ -148,13 +148,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() - # Enable mod_ssl if it isn't already enabled - # This is Let's Encrypt... we enable mod_ssl on initialization :) - # TODO: attempt to make the check faster... this enable should - # be asynchronous as it shouldn't be that time sensitive - # on initialization - self._prepare_server_https() - temp_install(self.mod_ssl_conf) def deploy_cert(self, domain, cert_path, key_path, chain_path=None): @@ -167,8 +160,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): to the correct destination. After the certificate is installed, the VirtualHost is enabled if it isn't already. - .. todo:: Make sure last directive is changed - .. todo:: Might be nice to remove chain directive if none exists This shouldn't happen within letsencrypt though @@ -424,7 +415,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): is appropriately listening on port 443. """ - if not self.mod_loaded("ssl_module"): + if "ssl_module" not in self.parser.modules: logger.info("Loading mod_ssl into Apache Server") self.enable_mod("ssl") @@ -609,7 +600,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: (bool, :class:`~letsencrypt_apache.obj.VirtualHost`) """ - if not self.mod_loaded("rewrite_module"): + if "rewrite_module" not in self.parser.modules: self.enable_mod("rewrite") general_v = self._general_vhost(ssl_vhost) @@ -911,6 +902,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if self.is_site_enabled(vhost.filep): return True + if vhost.ssl: + self._prepare_server_https() + if self.save_notes: + self.save("Enabled TLS for Apache") + if "/sites-available/" in vhost.filep: enabled_path = ("%s/sites-enabled/%s" % (self.parser.root, os.path.basename(vhost.filep))) @@ -927,7 +923,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Both enables and restarts Apache so module is active. - :param str mod_name: Name of the module to enable. + :param str mod_name: Name of the module to enable. (e.g. 'ssl') """ try: @@ -936,11 +932,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): subprocess.check_call([self.conf("enmod"), mod_name], stdout=open("/dev/null", "w"), stderr=open("/dev/null", "w")) - apache_restart(self.conf("init_script")) except (OSError, subprocess.CalledProcessError): logger.exception("Error enabling mod_%s", mod_name) raise errors.MisconfigurationError( "Missing enable_mod binary or lack privileges") + self.parser.modules.add(mod_name + "_module") + self.parser.modules.add("mod_" + mod_name) def mod_loaded(self, module): """Checks to see if mod_ssl is loaded diff --git a/letsencrypt_apache/dvsni.py b/letsencrypt_apache/dvsni.py index 2542b242f..61c990ab5 100644 --- a/letsencrypt_apache/dvsni.py +++ b/letsencrypt_apache/dvsni.py @@ -76,6 +76,9 @@ class ApacheDvsni(common.Dvsni): # Setup the configuration self._mod_config(addresses) + # Prepare the server for HTTPS + self.configurator._prepare_https_server() + # Save reversible changes self.configurator.save("SNI Challenge", True) diff --git a/letsencrypt_apache/parser.py b/letsencrypt_apache/parser.py index be1aa7a52..1f233f187 100644 --- a/letsencrypt_apache/parser.py +++ b/letsencrypt_apache/parser.py @@ -50,7 +50,7 @@ class ApacheParser(object): """Iterates on the configuration until no new modules are loaded. ..todo:: This should be attempted to be done with a binary to avoid - the iteration issue. Else... do a better job of parsing to avoid it + the iteration issue. Else... parse and enable mods at same time. """ matches = self.find_dir(case_i("LoadModule")) From 53e01c19af1fde7aeed193b64b9fc54c1846c471 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 9 Jul 2015 17:51:08 -0700 Subject: [PATCH 09/52] Use config_test raise appropriate errors --- letsencrypt_apache/configurator.py | 37 ++++++++++++++++++------------ 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/letsencrypt_apache/configurator.py b/letsencrypt_apache/configurator.py index 97588d483..0d9fc4bda 100644 --- a/letsencrypt_apache/configurator.py +++ b/letsencrypt_apache/configurator.py @@ -135,6 +135,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def prepare(self): """Prepare the authenticator/installer.""" + # Make sure configuration is valid + self.config_test() + self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.mod_ssl_conf, self.conf("ctl")) @@ -983,8 +986,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def config_test(self): # pylint: disable=no-self-use """Check the configuration of Apache for errors. - :returns: Success - :rtype: bool + :raises .errors.PluginError: If Unable to run apache2ctl + :raises .errors.MisconfigurationError: If config_test fails """ try: @@ -999,10 +1002,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if proc.returncode != 0: # Enter recovery routine... - logger.error("Configtest failed\n%s\n%s", stdout, stderr) - return False - - return True + logger.error("Apache Configtest failed\n%s\n%s", stdout, stderr) + raise errors.MisconfigurationError( + "Apache Configtest failure:\n%s\n%s" % (stdout, stderr)) def verify_setup(self): """Verify the setup to ensure safe operating environment. @@ -1124,24 +1126,29 @@ def apache_restart(apache_init_script): need to be moved into the class again. Perhaps this version can live on... for testing purposes. + :raises .errors.PluginError: If unable to restart with apache_init_script + :raises .errors.MisconfigurationError: If unable to restart due to a + configuration problem. + """ try: proc = subprocess.Popen([apache_init_script, "restart"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - - if proc.returncode != 0: - # Enter recovery routine... - logger.error("Apache Restart Failed!\n%s\n%s", stdout, stderr) - return False except (OSError, ValueError): logger.fatal( - "Apache Restart Failed - Please Check the Configuration") - raise errors.MisconfigurationError("Unable to restart Apache process") + "Unable to restart the Apache process with %s", apache_init_script) + raise errors.PluginError( + "Unable to restart Apache process with %s" % apache_init_script) - return True + stdout, stderr = proc.communicate() + + if proc.returncode != 0: + # Enter recovery routine... + logger.error("Apache Restart Failed!\n%s\n%s", stdout, stderr) + raise errors.MisconfigurationError( + "Error while restarting Apache:\n%s\n%s" % (stdout, stderr)) def get_file_path(vhost_path): From 50f1db4b11e5599bbfc17b0e57087b8947f26b29 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 9 Jul 2015 18:55:08 -0700 Subject: [PATCH 10/52] Move dvsni config file to server-root --- letsencrypt_apache/dvsni.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/letsencrypt_apache/dvsni.py b/letsencrypt_apache/dvsni.py index 61c990ab5..c267f44f5 100644 --- a/letsencrypt_apache/dvsni.py +++ b/letsencrypt_apache/dvsni.py @@ -44,6 +44,13 @@ class ApacheDvsni(common.Dvsni): """ + def __init__(self): + super(ApacheDvsni, self).__init__(*args, **kwargs) + + self.challenge_conf = os.path.join( + self.configurator.conf("server-root"), + "le_dvsni_cert_challenge.conf") + def perform(self): """Peform a DVSNI challenge.""" if not self.achalls: From 15975465162a12128b0304e9e95a36d5add7c064 Mon Sep 17 00:00:00 2001 From: Ceesjan Luiten Date: Mon, 13 Jul 2015 21:18:11 +0200 Subject: [PATCH 11/52] Consume longest possible match --- letsencrypt-nginx/letsencrypt_nginx/nginxparser.py | 2 +- .../letsencrypt_nginx/tests/nginxparser_test.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py index 7870581b4..814b5f15e 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py +++ b/letsencrypt-nginx/letsencrypt_nginx/nginxparser.py @@ -37,7 +37,7 @@ class RawNginxParser(object): + Group(ZeroOrMore(Group(comment | assignment) | block)) + right_bracket) - script = OneOrMore(Group(comment | assignment) | block) + stringEnd + script = OneOrMore(Group(comment | assignment) ^ block) + stringEnd def __init__(self, source): self.source = source diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/nginxparser_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/nginxparser_test.py index 0d6e5c453..2130b4824 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/nginxparser_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/nginxparser_test.py @@ -5,7 +5,7 @@ import unittest from pyparsing import ParseException from letsencrypt_nginx.nginxparser import ( - RawNginxParser, load, dumps, dump) + RawNginxParser, loads, load, dumps, dump) from letsencrypt_nginx.tests import util @@ -160,6 +160,13 @@ class TestRawNginxParser(unittest.TestCase): ['#', ' listen 80;']]], ]) + def test_issue_518(self): + parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }') + + self.assertEqual(parsed, [ + [['if', '($http_accept ~* "webp")'], + [['set', '$webp "true"']]] + ]) if __name__ == '__main__': unittest.main() # pragma: no cover From 89d810c06a966efc2e946351a81f7974fd320af7 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Mon, 13 Jul 2015 14:16:51 -0700 Subject: [PATCH 12/52] Fix enable_mod --- .../letsencrypt_apache/configurator.py | 56 +++++++++++++++---- .../letsencrypt_apache/parser.py | 22 +++++++- letsencrypt_apache | 1 - letsencrypt_nginx | 1 - 4 files changed, 66 insertions(+), 14 deletions(-) delete mode 120000 letsencrypt_apache delete mode 120000 letsencrypt_nginx diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 0d9fc4bda..cb6029477 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -64,7 +64,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): This class can adequately configure most typical configurations but is not ready to handle very complex configurations. - .. todo:: Add support for config file variables Define rootDir /var/www/ + .. todo:: Always use self.parser.aug_get rather than self.aug.get + .. todo:: Verify permissions on configuration root... it is easier than + checking permissions on each of the relative directories and less error + prone. + The API of this class will change in the coming weeks as the exact needs of clients are clarified with the new and developing protocol. @@ -929,26 +933,56 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str mod_name: Name of the module to enable. (e.g. 'ssl') """ - try: - # Use check_output so the command will finish before reloading - # TODO: a2enmod is debian specific... - subprocess.check_call([self.conf("enmod"), mod_name], - stdout=open("/dev/null", "w"), - stderr=open("/dev/null", "w")) - except (OSError, subprocess.CalledProcessError): - logger.exception("Error enabling mod_%s", mod_name) + # Support Debian specific setup + if (not os.path.isdir(os.path.join(self.parser.root, "mods-available")) + or not os.path.isdir( + os.path.join(self.parser.root, "mods-enabled"))): raise errors.MisconfigurationError( - "Missing enable_mod binary or lack privileges") + "Unsupported directory layout. You may try to enable mod %s " + "and try again." % mod_name) + + self._enable_mod_debian(mod_name) + self.parser.modules.add(mod_name + "_module") self.parser.modules.add("mod_" + mod_name) + def _enable_mod_debian(self, mod_name): + """Assumes mods-available, mods-enabled layout.""" + + # TODO: This can be further updated to not require all files. + if mod_name == "ssl": + self._enable_mod_debian_files(["ssl.conf", "ssl.load"]) + elif mod_name == "rewrite": + self._enable_mod_debian_files(["rewrite.load"]) + else: + raise NotImplemented + + def _enable_mod_debian_files(self, filenames): + """Move over all required files into mods-enabled.""" + mods_available = os.path.join(self.parser.root, "mods-available") + mods_enabled = os.path.join(self.parser.root, "mods-enabled") + + # Check to see all files are available. + for filename in filenames: + if not os.path.isfile(os.path.join(mods_available, filename)): + raise errors.MisconfigurationError( + "Unable to enable module. Required files missing from " + "mods-available. %s" % str(filenames)) + + # Register and symlink files + for filename in files: + enabled_path = os.path.join(mods_enabled, filename) + self.reverter.register_file_creation(False, enabled_path) + os.symlink(os.path.join(mods_available, filename), enabled_path) + + def mod_loaded(self, module): """Checks to see if mod_ssl is loaded Uses ``apache_ctl`` to get loaded module list. This also effectively serves as a config_test. - :returns: If ssl_module is included and active in Apache + :returns: If module is loaded. :rtype: bool """ diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 1f233f187..dfd94db98 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -12,6 +12,9 @@ from letsencrypt import errors logger = logging.getLogger(__name__) +arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") + + class ApacheParser(object): """Class handles the fine details of parsing the Apache Configuration. @@ -71,7 +74,9 @@ class ApacheParser(object): def _init_runtime_variables(self, ctl): """" - ..todo:: Also use apache2ctl -V for compiled parameters + .. note:: Compile time variables (apache2ctl -V) are not used within the + dynamic configuration files. These should not be parsed or + interpreted. """ stdout = self._get_runtime_cfg(ctl) @@ -276,6 +281,21 @@ class ApacheParser(object): return ordered_matches + def get_arg(self, match): + """Uses augeas.get to get argument value and interprets result. + + This also converts all variables and parameters appropriately. + + """ + value = self.aug.get(match) + variables = arg_var_interpreter.findall(value) + + for var in variables: + # Strip off ${ and } + value = value.replace(var, self.variables[var[2:-1]]) + + return value + def _exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" filters = [("ifmodule", self.modules), ("ifdefine", self.variables)] diff --git a/letsencrypt_apache b/letsencrypt_apache deleted file mode 120000 index f50a82d01..000000000 --- a/letsencrypt_apache +++ /dev/null @@ -1 +0,0 @@ -letsencrypt-apache \ No newline at end of file diff --git a/letsencrypt_nginx b/letsencrypt_nginx deleted file mode 120000 index c2f40bb9d..000000000 --- a/letsencrypt_nginx +++ /dev/null @@ -1 +0,0 @@ -letsencrypt-nginx/ \ No newline at end of file From ec065e8b58e7217cf62b8c9ef37278c4f3813910 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Mon, 13 Jul 2015 23:18:33 -0700 Subject: [PATCH 13/52] Parse variables and fnmatch --- .../letsencrypt_apache/configurator.py | 27 ++++-- .../letsencrypt_apache/parser.py | 87 +++++++------------ 2 files changed, 50 insertions(+), 64 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index cb6029477..a513557af 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -64,7 +64,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): This class can adequately configure most typical configurations but is not ready to handle very complex configurations. - .. todo:: Always use self.parser.aug_get rather than self.aug.get .. todo:: Verify permissions on configuration root... it is easier than checking permissions on each of the relative directories and less error prone. @@ -332,7 +331,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for name in name_match: args = self.aug.match(name + "/*") for arg in args: - host.add_name(self.aug.get(arg)) + host.add_name(self.parser.get_arg(arg)) def _create_vhost(self, path): """Used by get_virtual_hosts to create vhost objects @@ -346,7 +345,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addrs = set() args = self.aug.match(path + "/arg") for arg in args: - addrs.add(common.Addr.fromstring(self.aug.get(arg))) + addrs.add(common.Addr.fromstring(self.parser.get_arg(arg))) is_ssl = False if self.parser.find_dir( @@ -514,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for addr in ssl_addr_p: old_addr = common.Addr.fromstring( - str(self.aug.get(addr))) + str(self.parser.get_arg(addr))) ssl_addr = old_addr.get_addr_obj("443") self.aug.set(addr, str(ssl_addr)) ssl_addrs.add(ssl_addr) @@ -868,8 +867,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): errors.MisconfigurationError( "Too many cert/key directives in vhost") - cert = os.path.abspath(self.aug.get(cert_path[0])) - key = os.path.abspath(self.aug.get(key_path[0])) + cert = os.path.abspath(self.parser.get_arg(cert_path[0])) + key = os.path.abspath(self.parser.get_arg(key_path[0])) c_k.add((cert, key, get_file_path(cert_path[0]))) return c_k @@ -942,13 +941,19 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "and try again." % mod_name) self._enable_mod_debian(mod_name) + self.save_notes += "Enabled %s module in Apache" % mod_name + logger.debug("Enabled Apache %s module", mod_name) + + # Modules can enable additional config files. Variables may be defined + # within these new configuration sections. + # Restart is not necessary as DUMP_RUN_CFG uses latest config. + self.parser.update_runtime_variables() self.parser.modules.add(mod_name + "_module") self.parser.modules.add("mod_" + mod_name) def _enable_mod_debian(self, mod_name): """Assumes mods-available, mods-enabled layout.""" - # TODO: This can be further updated to not require all files. if mod_name == "ssl": self._enable_mod_debian_files(["ssl.conf", "ssl.load"]) @@ -957,7 +962,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): else: raise NotImplemented - def _enable_mod_debian_files(self, filenames): + def _enable_mod_debian_files(self, filenames, mod_name): """Move over all required files into mods-enabled.""" mods_available = os.path.join(self.parser.root, "mods-available") mods_enabled = os.path.join(self.parser.root, "mods-enabled") @@ -972,10 +977,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Register and symlink files for filename in files: enabled_path = os.path.join(mods_enabled, filename) + if os.path.isfile(enabled_path): + logger.debug( + "Error - enabling module %s, filepath already exists " + "%s", mod_name, enabled_path) + raise errors.PluginError("Error enabling module %s" % mod_name) self.reverter.register_file_creation(False, enabled_path) os.symlink(os.path.join(mods_available, filename), enabled_path) - def mod_loaded(self, module): """Checks to see if mod_ssl is loaded diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index dfd94db98..30e97814b 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -1,5 +1,6 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" import collections +import fnmatch import itertools import logging import os @@ -13,6 +14,7 @@ logger = logging.getLogger(__name__) arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") +fnmatch_chars = set(["*", "?", "\\", "[", "]"]) class ApacheParser(object): @@ -29,7 +31,8 @@ class ApacheParser(object): # https://httpd.apache.org/docs/2.4/mod/core.html#define # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine # This only handles invocation parameters and Define directives! - self.variables = self._init_runtime_variables(ctl) + self.variables = {} + self.update_runtime_variables(ctl) # Find configuration root and make sure augeas can parse it. self.aug = aug @@ -67,11 +70,11 @@ class ApacheParser(object): for match_name, match_filename in itertools.izip( iterator, iterator): - self.modules.add(self.aug.get(match_name)) + self.modules.add(self.get_arg(match_name)) self.modules.add( - os.path.basename(self.aug.get(match_filename))[:-2] + "c") + os.path.basename(self.get_arg(match_filename))[:-2] + "c") - def _init_runtime_variables(self, ctl): + def update_runtime_variables(self, ctl): """" .. note:: Compile time variables (apache2ctl -V) are not used within the @@ -94,7 +97,7 @@ class ApacheParser(object): parts = match.partition("=") variables[parts[0]] = parts[2] - return variables + self.variables = variables def _get_runtime_cfg(self, ctl): """Get runtime configuration info. @@ -269,13 +272,15 @@ class ApacheParser(object): ordered_matches = [] + # TODO: Wildcards should be included in alphabetical order + # https://httpd.apache.org/docs/2.4/mod/core.html#include for match in matches: dir = self.aug.get(match).lower() if dir == "include" or dir == "includeoptional": # start[6:] to strip off /files ordered_matches.extend(self.find_dir( directive, arg, self._get_include_path( - strip_dir(start[6:]), self.aug.get(match + "/arg")))) + self.get_arg(match + "/arg")))) else: ordered_matches.extend(self.aug.match(match + arg_suffix)) @@ -334,7 +339,7 @@ class ApacheParser(object): return True - def _get_include_path(self, cur_dir, arg): + def _get_include_path(self, arg): """Converts an Apache Include directive into Augeas path. Converts an Apache Include directive argument into an Augeas @@ -342,29 +347,12 @@ class ApacheParser(object): .. todo:: convert to use os.path.join() - :param str cur_dir: current working directory - :param str arg: Argument of Include directive :returns: Augeas path string :rtype: str """ - # Sanity check argument - maybe - # Question: what can the attacker do with control over this string - # Effect parse file... maybe exploit unknown errors in Augeas - # If the attacker can Include anything though... and this function - # only operates on Apache real config data... then the attacker has - # already won. - # Perhaps it is better to simply check the permissions on all - # included files? - # check_config to validate apache config doesn't work because it - # would create a race condition between the check and this input - - # TODO: Maybe... although I am convinced we have lost if - # Apache files can't be trusted. The augeas include path - # should be made to be exact. - # Check to make sure only expected characters are used <- maybe remove # validChars = re.compile("[a-zA-Z0-9.*?_-/]*") # matchObj = validChars.match(arg) @@ -374,11 +362,8 @@ class ApacheParser(object): # Standardize the include argument based on server root if not arg.startswith("/"): - arg = cur_dir + arg - # conf/ is a special variable for ServerRoot in Apache - elif arg.startswith("conf/"): - arg = self.root + arg[4:] - # TODO: Test if Apache allows ../ or ~/ for Includes + # Normpath will condense ../ + arg = os.path.normpath(os.path.join(self.root, arg)) # Attempts to add a transform to the file if one does not already exist self._parse_file(arg) @@ -386,46 +371,38 @@ class ApacheParser(object): # Argument represents an fnmatch regular expression, convert it # Split up the path and convert each into an Augeas accepted regex # then reassemble - if "*" in arg or "?" in arg: - split_arg = arg.split("/") - for idx, split in enumerate(split_arg): - # * and ? are the two special fnmatch characters - if "*" in split or "?" in split: - # Turn it into a augeas regex - # TODO: Can this instead be an augeas glob instead of regex - split_arg[idx] = ("* [label()=~regexp('%s')]" % - self.fnmatch_to_re(split)) - # Reassemble the argument - arg = "/".join(split_arg) + split_arg = arg.split("/") + for idx, split in enumerate(split_arg): + if any(char in fnmatch_chars for char in split): + # Turn it into a augeas regex + # TODO: Can this instead be an augeas glob instead of regex + split_arg[idx] = ("* [label()=~regexp('%s')]" % + self.fnmatch_to_re(split)) + # Reassemble the argument + arg = "/".join(split_arg) # If the include is a directory, just return the directory as a file if arg.endswith("/"): - return get_aug_path(arg[:len(arg)-1]) + return get_aug_path(arg[:-1]) return get_aug_path(arg) def fnmatch_to_re(self, clean_fn_match): # pylint: disable=no-self-use """Method converts Apache's basic fnmatch to regular expression. + Assumption - Configs are assumed to be well-formed and only writable by + privileged users. + + https://apr.apache.org/docs/apr/2.0/apr__fnmatch_8h_source.html + http://apache2.sourcearchive.com/documentation/2.2.16-6/apr__fnmatch_8h_source.html + :param str clean_fn_match: Apache style filename match, similar to globs :returns: regex suitable for augeas :rtype: str """ - # Checkout fnmatch.py in venv/local/lib/python2.7/fnmatch.py - regex = "" - for letter in clean_fn_match: - if letter == ".": - regex = regex + r"\." - elif letter == "*": - regex = regex + ".*" - # According to apache.org ? shouldn't appear - # but in case it is valid... - elif letter == "?": - regex = regex + "." - else: - regex = regex + letter - return regex + # This strips off final /Z(?ms) + return fnmatch.translate(clean_fn_match)[:-7] def _parse_file(self, filepath): """Parse file with Augeas From 1b13458463ecdb486602c1f808147decad4aa5b0 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Tue, 14 Jul 2015 14:03:43 -0700 Subject: [PATCH 14/52] Redesign choose_vhost and prepare_https, Cleanup make_vhost_ssl --- .../letsencrypt_apache/configurator.py | 204 ++++++++++++------ .../letsencrypt_apache/dvsni.py | 3 +- 2 files changed, 137 insertions(+), 70 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index a513557af..49b234ee2 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -231,29 +231,19 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Allows for domain names to be associated with a virtual host - # Client isn't using create_dn_server_assoc(self, dn, vh) yet if target_name in self.assoc: return self.assoc[target_name] - # Check for servernames/aliases for ssl hosts - for vhost in self.vhosts: - if vhost.ssl and target_name in vhost.names: - self.assoc[target_name] = vhost - return vhost - # Checking for domain name in vhost address - # This technique is not recommended by Apache but is technically valid - target_addr = common.Addr((target_name, "443")) - for vhost in self.vhosts: - if target_addr in vhost.addrs: - self.assoc[target_name] = vhost - return vhost - # Check for non ssl vhosts with servernames/aliases == "name" - for vhost in self.vhosts: - if not vhost.ssl and target_name in vhost.names: - vhost = self.make_vhost_ssl(vhost) - self.assoc[target_name] = vhost - return vhost + # Make a new association + vhost = self._find_best_vhost(target_name) + if vhost is not None: + if not vhost.ssl: + vhost = self.make_vhost_ssl(non_ssl_vhost) + self.assoc[target_name] = vhost + return vhost + + # Select a vhost from a list vhost = display_ops.select_vhost(target_name, self.vhosts) if vhost is not None: self.assoc[target_name] = vhost @@ -268,10 +258,35 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return vhost - # # No matches, search for the default - # for vhost in self.vhosts: - # if "_default_:443" in vhost.addrs: - # return vhost + def _find_best_vhost(self, target_name): + """Finds the best vhost for a target_name. + + :returns: VHost or None + + """ + # Points 4 - Servername SSL + # Points 3 - Address name with SSL + # Points 2 - Servername no SSL + # Points 1 - Address name with no SSL + best_candidate = None + best_points = 0 + + for vhost in self.vhosts: + if target_name in vhost.names: + points = 2 + elif any(addr.get_addr() == target_name for addr in vhost.addrs): + points = 1 + else: + continue + + if vhost.ssl: + points += 2 + + if points > best_points: + best_points = points + best_candidate = vhost + + return best_candidate def create_dn_server_assoc(self, domain, vhost): """Create an association between a domain name and virtual host. @@ -414,25 +429,29 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr self.save_notes += "\tDirective added to %s\n" % path - def _prepare_server_https(self): + def _prepare_server_https(self, port): """Prepare the server for HTTPS. Make sure that the ssl_module is loaded and that the server - is appropriately listening on port 443. + is appropriately listening on port. + + :param int port: Port to listen on """ if "ssl_module" not in self.parser.modules: logger.info("Loading mod_ssl into Apache Server") self.enable_mod("ssl") - # Check for Listen 443 + # Check for Listen # Note: This could be made to also look for ip:443 combo - if not self.parser.find_dir(parser.case_i("Listen"), "443"): - logger.debug("No Listen 443 directive found. Setting the " - "Apache Server to Listen on port 443") + if not self.parser.find_dir(parser.case_i("Listen"), str(port)): + logger.debug("No Listen {0} directive found. Setting the " + "Apache Server to Listen on port {0}".format(port)) path = self.parser.add_dir_to_ifmodssl( - parser.get_aug_path(self.parser.loc["listen"]), "Listen", "443") - self.save_notes += "Added Listen 443 directive to %s\n" % path + parser.get_aug_path( + self.parser.loc["listen"]), "Listen", str(port)) + self.save_notes += "Added Listen %d directive to %s\n" % ( + port, path) def make_server_sni_ready(self, vhost, default_addr="*:443"): """Checks to see if the server is ready for SNI challenges. @@ -443,6 +462,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str default_addr: TODO - investigate function further """ + # Version 2.4 and later are automatically SNI ready. if self.version >= (2, 4): return # Check for NameVirtualHost @@ -481,12 +501,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ avail_fp = nonssl_vhost.filep - # Get filepath of new ssl_vhost - if avail_fp.endswith(".conf"): - ssl_fp = avail_fp[:-(len(".conf"))] + self.conf("le_vhost_ext") - else: - ssl_fp = avail_fp + self.conf("le_vhost_ext") + ssl_fp = self._get_ssl_vhost_path(avail_fp) + self._copy_create_ssl_vhost_skeleton(avail_fp, ssl_fp) + + # Reload augeas to take into account the new vhost + self.aug.load() + + # Get Vhost augeas path for new vhost + vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (ssl_fp, parser.case_i("VirtualHost"))) + if len(vh_p) != 1: + logger.error("Error: should only be one vhost in %s", avail_fp) + raise errors.PluginError("Only one vhost per file is allowed") + else: + # This simplifies the process + vh_p = vh_p[0] + + # Update Addresses + ssl_addrs = self._update_ssl_vhosts_addrs(vh_p) + + # Add directives + self._add_dummy_ssl_directives(vh_p) + + # Log actions and create save notes + logger.info("Created an SSL vhost at %s", ssl_fp) + self.save_notes += "Created ssl vhost at %s\n" % ssl_fp + self.save() + + # We know the length is one because of the assertion above + # Create the Vhost object + ssl_vhost = self._create_vhost(vh_p[0]) + self.vhosts.append(ssl_vhost) + + + # NOTE: Searches through Augeas seem to ruin changes to directives + # The configuration must also be saved before being searched + # for the new directives; For these reasons... this is tacked + # on after fully creating the new vhost + + # Now check if addresses need to be added as NameBasedVhost addrs + # This is for compliance with versions of Apache < 2.4 + self._add_name_vhost_if_necessary(ssl_vhost) + + return ssl_vhost + + def _get_ssl_vhost_path(self, non_ssl_vh_fp): + # Get filepath of new ssl_vhost + if non_ssl_vh_fp.endswith(".conf"): + return non_ssl_vh_fp[:-(len(".conf"))] + self.conf("le_vhost_ext") + else: + return non_ssl_vh_fp + self.conf("le_vhost_ext") + + def _copy_create_ssl_vhost_skeleton(self, avail_fp, ssl_fp): + """Copies over existing Vhost with IfModule mod_ssl.c> skeleton. + + :param str avail_fp: Pointer to the original available non-ssl vhost + :param str ssl_fp: Full path where the new ssl_vhost will reside. + + A new file is created on the filesystem. + + """ # First register the creation so that it is properly removed if # configuration is rolled back self.reverter.register_file_creation(False, ssl_fp) @@ -502,14 +577,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.fatal("Error writing/reading to file in make_vhost_ssl") raise errors.PluginError("Unable to write/read in make_vhost_ssl") - self.aug.load() - + def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() - - # change address to address:443 - addr_match = "/files%s//* [label()=~regexp('%s')]/arg" - ssl_addr_p = self.aug.match( - addr_match % (ssl_fp, parser.case_i("VirtualHost"))) + ssl_addr_p = self.aug.match(vh_path + "/arg") for addr in ssl_addr_p: old_addr = common.Addr.fromstring( @@ -518,37 +588,31 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug.set(addr, str(ssl_addr)) ssl_addrs.add(ssl_addr) - # Add directives - vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % - (ssl_fp, parser.case_i("VirtualHost"))) - if len(vh_p) != 1: - logger.error("Error: should only be one vhost in %s", avail_fp) - raise errors.PluginError("Only one vhost per file is allowed") + return ssl_addrs - self.parser.add_dir(vh_p[0], "SSLCertificateFile", + def _add_dummy_ssl_directives(self, vh_path): + self.parser.add_dir(vh_path, "SSLCertificateFile", "insert_cert_file_path") - self.parser.add_dir(vh_p[0], "SSLCertificateKeyFile", + self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_p[0], "Include", self.parser.loc["ssl_options"]) + self.parser.add_dir(vh_path, "Include", self.parser.loc["ssl_options"]) - # Log actions and create save notes - logger.info("Created an SSL vhost at %s", ssl_fp) - self.save_notes += "Created ssl vhost at %s\n" % ssl_fp - self.save() + def _add_name_vhost_if_necessary(self, vhost): + """Add NameVirtualHost Directives if necessary for new vhost. - # We know the length is one because of the assertion above - ssl_vhost = self._create_vhost(vh_p[0]) - self.vhosts.append(ssl_vhost) + NameVirtualHosts was a directive in Apache < 2.4 + https://httpd.apache.org/docs/2.2/mod/core.html#namevirtualhost - # NOTE: Searches through Augeas seem to ruin changes to directives - # The configuration must also be saved before being searched - # for the new directives; For these reasons... this is tacked - # on after fully creating the new vhost + :param vhost: New virtual host that was recently created. + :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + + """ need_to_save = False + # See if the exact address appears in any other vhost - for addr in ssl_addrs: - for vhost in self.vhosts: - if (ssl_vhost.filep != vhost.filep and addr in vhost.addrs and + for addr in vhost.addrs: + for test_vhost in self.vhosts: + if (vhost.filep != test_vh.filep and addr in test_vh.addrs and not self.is_name_vhost(addr)): self.add_name_vhost(addr) logger.info("Enabling NameVirtualHosts on %s", addr) @@ -557,8 +621,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if need_to_save: self.save() - return ssl_vhost - + ############################################################################ + # Enhancements + ############################################################################ def supported_enhancements(self): # pylint: disable=no-self-use """Returns currently supported enhancements.""" return ["redirect"] @@ -909,7 +974,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True if vhost.ssl: - self._prepare_server_https() + # TODO: Make this based on addresses + self._prepare_server_https(443) if self.save_notes: self.save("Enabled TLS for Apache") diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index 1f7daa3fb..eace6ce90 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -84,7 +84,8 @@ class ApacheDvsni(common.Dvsni): self._mod_config(addresses) # Prepare the server for HTTPS - self.configurator._prepare_https_server() + # TODO: Base on addresses + self.configurator._prepare_https_server(443) # Save reversible changes self.configurator.save("SNI Challenge", True) From 0d69e5cff416311f6f0ea06d891c807fed119c7e Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 15 Jul 2015 14:34:24 -0700 Subject: [PATCH 15/52] Support dvsni_port, reorganize dvsni --- .../letsencrypt_apache/configurator.py | 57 ++++++++--------- .../letsencrypt_apache/dvsni.py | 63 ++++++++++++------- letsencrypt-apache/letsencrypt_apache/obj.py | 34 ++++++++++ 3 files changed, 99 insertions(+), 55 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 49b234ee2..7edcb8cc8 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -53,25 +53,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Apache configurator. - State of Configurator: This code has been tested under Ubuntu 12.04 - Apache 2.2 and this code works for Ubuntu 14.04 Apache 2.4. Further - notes below. - - This class was originally developed for Apache 2.2 and I have been slowly - transitioning the codebase to work with all of the 2.4 features. - I have implemented most of the changes... the missing ones are - mod_ssl.c vs ssl_mod, and I need to account for configuration variables. - This class can adequately configure most typical configurations but - is not ready to handle very complex configurations. + State of Configurator: This code has been been tested and built for Ubuntu + 14.04 Apache 2.4 and it works for Ubuntu 12.04 Apache 2.2 .. todo:: Verify permissions on configuration root... it is easier than checking permissions on each of the relative directories and less error prone. - - The API of this class will change in the coming weeks as the exact - needs of clients are clarified with the new and developing protocol. - :ivar config: Configuration. :type config: :class:`~letsencrypt.interfaces.IConfig` @@ -204,6 +192,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): else: self.aug.set(path["chain_path"][-1], chain_path) + # Save notes about the transaction that took place self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, ", ".join(str(addr) for addr in vhost.addrs))) @@ -360,7 +349,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addrs = set() args = self.aug.match(path + "/arg") for arg in args: - addrs.add(common.Addr.fromstring(self.parser.get_arg(arg))) + addrs.add(obj.Addr.fromstring(self.parser.get_arg(arg))) is_ssl = False if self.parser.find_dir( @@ -400,7 +389,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): now NameVirtualHosts. If version is earlier than 2.4, check if addr has a NameVirtualHost directive in the Apache config - :param str target_addr: vhost address ie. \*:443 + :param target_addr: vhost address + :type target_addr: :class:~letsencrypt_apache.obj.Addr :returns: Success :rtype: bool @@ -419,7 +409,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. - :param str addr: Address that will be added as NameVirtualHost directive + :param addr: Address that will be added as NameVirtualHost directive + :type addr: :class:~letsencrypt_apache.obj.Addr """ path = self.parser.add_dir_to_ifmodssl( @@ -435,7 +426,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): Make sure that the ssl_module is loaded and that the server is appropriately listening on port. - :param int port: Port to listen on + :param str port: Port to listen on """ if "ssl_module" not in self.parser.modules: @@ -444,30 +435,34 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check for Listen # Note: This could be made to also look for ip:443 combo - if not self.parser.find_dir(parser.case_i("Listen"), str(port)): + if not self.parser.find_dir(parser.case_i("Listen"), port): logger.debug("No Listen {0} directive found. Setting the " "Apache Server to Listen on port {0}".format(port)) path = self.parser.add_dir_to_ifmodssl( parser.get_aug_path( - self.parser.loc["listen"]), "Listen", str(port)) - self.save_notes += "Added Listen %d directive to %s\n" % ( + self.parser.loc["listen"]), "Listen", port) + self.save_notes += "Added Listen %s directive to %s\n" % ( port, path) - def make_server_sni_ready(self, vhost, default_addr="*:443"): + def make_addrs_sni_ready( + self, addrs, default_addr=obj.Addr(("*", "443"))): """Checks to see if the server is ready for SNI challenges. - :param vhost: VirtualHost to check SNI compatibility - :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + :param addrs: Addresses to check SNI compatibility + :type addrs: :class:`~letsencrypt_apache.obj.Addr` - :param str default_addr: TODO - investigate function further + :param default_addr: TODO - investigate function further + :type default_addr: :class:~letsencrypt_apache.obj.Addr """ # Version 2.4 and later are automatically SNI ready. if self.version >= (2, 4): return + + # TODO: Review this 3-year old demo code # Check for NameVirtualHost # First see if any of the vhost addresses is a _default_ addr - for addr in vhost.addrs: + for addr in addrs: if addr.get_addr() == "_default_": if not self.is_name_vhost(default_addr): logger.debug("Setting all VirtualHosts on %s to be " @@ -475,7 +470,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.add_name_vhost(default_addr) # No default addresses... so set each one individually - for addr in vhost.addrs: + for addr in addrs: if not self.is_name_vhost(addr): logger.debug("Setting VirtualHost at %s to be a name " "based virtual host", addr) @@ -582,7 +577,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ssl_addr_p = self.aug.match(vh_path + "/arg") for addr in ssl_addr_p: - old_addr = common.Addr.fromstring( + old_addr = obj.Addr.fromstring( str(self.parser.get_arg(addr))) ssl_addr = old_addr.get_addr_obj("443") self.aug.set(addr, str(ssl_addr)) @@ -880,8 +875,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Instead... should look for vhost of the form *:80 # Should we prompt the user? ssl_addrs = ssl_vhost.addrs - if ssl_addrs == common.Addr.fromstring("_default_:443"): - ssl_addrs = [common.Addr.fromstring("*:443")] + if ssl_addrs == obj.Addr.fromstring("_default_:443"): + ssl_addrs = [obj.Addr.fromstring("*:443")] for vhost in self.vhosts: found = 0 @@ -975,7 +970,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if vhost.ssl: # TODO: Make this based on addresses - self._prepare_server_https(443) + self._prepare_server_https("443") if self.save_notes: self.save("Enabled TLS for Apache") diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index eace6ce90..ac0a3039a 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -3,6 +3,7 @@ import os from letsencrypt.plugins import common +from letsencrypt_apache import obj from letsencrypt_apache import parser @@ -59,21 +60,6 @@ class ApacheDvsni(common.Dvsni): # About to make temporary changes to the config self.configurator.save() - addresses = [] - default_addr = "*:443" - for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain) - - # TODO - @jdkasten review this code to make sure it makes sense - self.configurator.make_server_sni_ready(vhost, default_addr) - - for addr in vhost.addrs: - if "_default_" == addr.get_addr(): - addresses.append([default_addr]) - break - else: - addresses.append(list(vhost.addrs)) - responses = [] # Create all of the challenge certs @@ -81,29 +67,37 @@ class ApacheDvsni(common.Dvsni): responses.append(self._setup_challenge_cert(achall)) # Setup the configuration - self._mod_config(addresses) + dvsni_addrs = self._mod_config() + + self.configurator.make_addrs_sni_ready(dvsni_addrs) # Prepare the server for HTTPS - # TODO: Base on addresses - self.configurator._prepare_https_server(443) + self.configurator._prepare_https_server( + str(self.configurator.config.dvsni_port)) # Save reversible changes self.configurator.save("SNI Challenge", True) return responses - def _mod_config(self, ll_addrs): + def _mod_config(self): """Modifies Apache config files to include challenge vhosts. Result: Apache config includes virtual servers for issued challs - :param list ll_addrs: list of list of `~.common.Addr` to apply + :returns: All DVSNI addresses used + :rtype: set """ - # TODO: Use ip address of existing vhost instead of relying on FQDN + dvsni_addrs = set() config_text = "\n" - for idx, lis in enumerate(ll_addrs): - config_text += self._get_config_text(self.achalls[idx], lis) + + for achall in self.achalls: + achall_addrs = self.get_dvsni_addrs(achall) + dvsni_addrs.update(achall_addrs) + + config_text += self._get_config_text(self.achalls, achall_addrs) + config_text += "\n" self._conf_include_check(self.configurator.parser.loc["default"]) @@ -113,6 +107,27 @@ class ApacheDvsni(common.Dvsni): with open(self.challenge_conf, "w") as new_conf: new_conf.write(config_text) + return dvsni_addrs + + def get_dvsni_addrs(self, achall): + """Return the Apache addresses needed for DVSNI.""" + vhost = self.configurator.choose_vhost(achall.domain) + + # TODO: Checkout _default_ rules. + # TODO: Need to separate out test mode and normal mode for DVSNI addrs + dvsni_addrs = set() + default_addr = obj.Addr(("*", self.configurator.config.dvsni_port)) + + for addr in vhost.addrs: + # I don't think there can be two _default_ namebasedvhosts + if "_default_" == addr.get_addr(): + dvsni_addrs.add(default_addr) + else: + dvsni_addrs.add( + addr.get_sni_addr(self.configurator.config.dvsni_port)) + + return dvsni_addrs + def _conf_include_check(self, main_config): """Adds DVSNI challenge conf file into configuration. @@ -136,7 +151,7 @@ class ApacheDvsni(common.Dvsni): :type achall: :class:`letsencrypt.achallenges.DVSNI` :param list ip_addrs: addresses of challenged domain - :class:`list` of type `~.common.Addr` + :class:`list` of type `~.obj.Addr` :returns: virtual host configuration text :rtype: str diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index 13e00edd8..956b6999f 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -1,4 +1,38 @@ """Module contains classes used by the Apache Configurator.""" +from letsencrypt.plugins import common + +class Addr(common.Addr): + + def __eq__(self, other): + """This is defined as equalivalent within Apache. + + ip_addr:* == ip_addr + + """ + if isinstance(other, self.__class__): + return ((self.tup == other.tup) or + (self.tup[0] == other.tup[0] + and self.is_wildcard() and other.is_wildcard())) + return False + + def is_wildcard(self): + return tup[1] == "*" or not tup[1] + + def get_sni_addr(self, port): + """Returns the least specific address that resolves on the port. + + Example: + 1.2.3.4:443 -> 1.2.3.4: + 1.2.3.4:* -> 1.2.3.4:* + + :param str port: Desired port + + """ + if self.is_wildcard(): + return self + + return self.get_addr_obj(port) + class VirtualHost(object): # pylint: disable=too-few-public-methods From 679dda10af866d284ce054004344597244a969b4 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 15 Jul 2015 15:25:34 -0700 Subject: [PATCH 16/52] Fix parsing tests --- .../letsencrypt_apache/tests/parser_test.py | 65 +++++++++++++------ 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 3d5e80362..24aa359ed 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -23,31 +23,18 @@ class ApacheParserTest(util.ApacheTest): zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) from letsencrypt_apache.parser import ApacheParser - self.aug = augeas.Augeas(flags=augeas.Augeas.NONE) - self.parser = ApacheParser(self.aug, self.config_path, self.ssl_options) + self.aug = augeas.Augeas( + flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD) + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + self.parser = ApacheParser( + self.aug, self.config_path, self.ssl_options, "dummy_ctl_path") def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - def test_root_normalized(self): - from letsencrypt_apache.parser import ApacheParser - path = os.path.join(self.temp_dir, "debian_apache_2_4/////" - "two_vhost_80/../two_vhost_80/apache2") - parser = ApacheParser(self.aug, path, None) - self.assertEqual(parser.root, self.config_path) - - def test_root_absolute(self): - from letsencrypt_apache.parser import ApacheParser - parser = ApacheParser(self.aug, os.path.relpath(self.config_path), None) - self.assertEqual(parser.root, self.config_path) - - def test_root_no_trailing_slash(self): - from letsencrypt_apache.parser import ApacheParser - parser = ApacheParser(self.aug, self.config_path + os.path.sep, None) - self.assertEqual(parser.root, self.config_path) - def test_parse_file(self): """Test parse_file. @@ -124,5 +111,45 @@ class ApacheParserTest(util.ApacheTest): self.assertEqual(results["default"], results["name"]) +class ParserInitTest(util.ApacheTest): + def setUp(self): + super(ParserInitTest, self).setUp() + self.aug = augeas.Augeas( + flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD) + + def tearDown(self): + shutil.rmtree(self.temp_dir) + shutil.rmtree(self.config_dir) + shutil.rmtree(self.work_dir) + + def test_root_normalized(self): + from letsencrypt_apache.parser import ApacheParser + + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + path = os.path.join( + self.temp_dir, + "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2") + parser = ApacheParser(self.aug, path, None, "dummy_ctl") + + self.assertEqual(parser.root, self.config_path) + + def test_root_absolute(self): + from letsencrypt_apache.parser import ApacheParser + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + parser = ApacheParser( + self.aug, os.path.relpath(self.config_path), None, "dummy_ctl") + + self.assertEqual(parser.root, self.config_path) + + def test_root_no_trailing_slash(self): + from letsencrypt_apache.parser import ApacheParser + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + parser = ApacheParser( + self.aug, self.config_path + os.path.sep, None, "dummy_ctl") + self.assertEqual(parser.root, self.config_path) + if __name__ == "__main__": unittest.main() # pragma: no cover From de4540a1c7740605c6c1ced2a8306ee3c497f4e6 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 17 Jul 2015 14:09:46 -0700 Subject: [PATCH 17/52] some cleanup --- .../letsencrypt_apache/configurator.py | 60 +++++++++++++------ .../letsencrypt_apache/parser.py | 27 ++++++--- .../tests/configurator_test.py | 7 +-- .../letsencrypt_apache/tests/parser_test.py | 2 +- .../testdata/complex_parsing/apache2.conf | 53 ++++++++++++++++ .../tests/testdata/complex_parsing/test.conf | 1 + .../letsencrypt_apache/tests/util.py | 9 +-- 7 files changed, 123 insertions(+), 36 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 7edcb8cc8..69b635bf0 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -59,6 +59,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): .. todo:: Verify permissions on configuration root... it is easier than checking permissions on each of the relative directories and less error prone. + .. todo:: Write a server protocol finder. Listen or + Protocol . This can verify partial setups are correct :ivar config: Configuration. :type config: :class:`~letsencrypt.interfaces.IConfig` @@ -78,6 +80,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): description = "Apache Web Server - Alpha" + # Kept in same function to avoid multiple compilations of the regex + + private_ips_regex = re.compile( + r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" + r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") + @classmethod def add_parser_arguments(cls, add): add("server-root", default=constants.CLI_DEFAULTS["server_root"], @@ -223,7 +231,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if target_name in self.assoc: return self.assoc[target_name] - # Make a new association + # Try to find a reasonable vhost vhost = self._find_best_vhost(target_name) if vhost is not None: if not vhost.ssl: @@ -300,24 +308,35 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ all_names = set() - # Kept in same function to avoid multiple compilations of the regex - priv_ip_regex = (r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" - r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") - private_ips = re.compile(priv_ip_regex) - for vhost in self.vhosts: all_names.update(vhost.names) for addr in vhost.addrs: - # If it isn't a private IP, do a reverse DNS lookup - if not private_ips.match(addr.get_addr()): - try: - socket.inet_aton(addr.get_addr()) - all_names.add(socket.gethostbyaddr(addr.get_addr())[0]) - except (socket.error, socket.herror, socket.timeout): - continue + name = get_name_from_ip(addr) + if name: + all_names.add(name) return all_names + def get_name_from_ip(self, addr): + """Returns a reverse dns name if available. + + :param addr: IP Address + :type addr: ~.common.Addr + + :returns: name + :rtype: str + + """ + # If it isn't a private IP, do a reverse DNS lookup + if not private_ips_regex.match(addr.get_addr()): + try: + socket.inet_aton(addr.get_addr()) + return socket.gethostbyaddr(addr.get_addr())[0] + except (socket.error, socket.herror, socket.timeout): + pass + + return "" + def _add_servernames(self, host): """Helper function for get_virtual_hosts(). @@ -415,7 +434,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ path = self.parser.add_dir_to_ifmodssl( parser.get_aug_path( - self.parser.loc["name"]), "NameVirtualHost", str(addr)) + self.parser.loc["name"]), "NameVirtualHost", [str(addr)]) self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr self.save_notes += "\tDirective added to %s\n" % path @@ -438,11 +457,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not self.parser.find_dir(parser.case_i("Listen"), port): logger.debug("No Listen {0} directive found. Setting the " "Apache Server to Listen on port {0}".format(port)) - path = self.parser.add_dir_to_ifmodssl( + + if port == "443": + args = [port] + else: + # Non-standard ports should specify https protocol + args = [port, "https"] + + self.parser.add_dir_to_ifmodssl( parser.get_aug_path( - self.parser.loc["listen"]), "Listen", port) + self.parser.loc["listen"]), "Listen", args) self.save_notes += "Added Listen %s directive to %s\n" % ( - port, path) + port, self.parser.loc["listen"]) def make_addrs_sni_ready( self, addrs, default_addr=obj.Addr(("*", "443"))): diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 30e97814b..ffbacc066 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -81,6 +81,8 @@ class ApacheParser(object): dynamic configuration files. These should not be parsed or interpreted. + .. todo:: Create separate compile time variables... simply for arg_get() + """ stdout = self._get_runtime_cfg(ctl) @@ -157,7 +159,7 @@ class ApacheParser(object): return filtered - def add_dir_to_ifmodssl(self, aug_conf_path, directive, val): + def add_dir_to_ifmodssl(self, aug_conf_path, directive, args): """Adds directive and value to IfMod ssl block. Adds given directive and value along configuration path within @@ -165,8 +167,9 @@ class ApacheParser(object): the file, it is created. :param str aug_conf_path: Desired Augeas config path to add directive - :param str directive: Directive you would like to add - :param str val: Value of directive ie. Listen 443, 443 is the value + :param str directive: Directive you would like to add, e.g. Listen + :param args: Values of the directive; str "443" or list of str + :type args: list """ # TODO: Add error checking code... does the path given even exist? @@ -176,7 +179,12 @@ class ApacheParser(object): self.aug.insert(if_mod_path + "arg", "directive", False) nvh_path = if_mod_path + "directive[1]" self.aug.set(nvh_path, directive) - self.aug.set(nvh_path + "/arg", val) + if len(args) == 1: + self.aug.set(nvh_path + "/arg", args[0]) + else: + for i, arg in enumerate(args): + self.aug.set("%s/arg[%d]" % (nvh_path, i), arg) + def _get_ifmod(self, aug_conf_path, mod): """Returns the path to and creates one if it doesn't exist. @@ -195,7 +203,7 @@ class ApacheParser(object): # Strip off "arg" at end of first ifmod path return if_mods[0][:len(if_mods[0]) - 3] - def add_dir(self, aug_conf_path, directive, arg): + def add_dir(self, aug_conf_path, directive, args): """Appends directive to the end fo the file given by aug_conf_path. .. note:: Not added to AugeasConfigurator because it may depend @@ -203,16 +211,17 @@ class ApacheParser(object): :param str aug_conf_path: Augeas configuration path to add directive :param str directive: Directive to add - :param str arg: Value of the directive. ie. Listen 443, 443 is arg + :param args: Value of the directive. ie. Listen 443, 443 is arg + :type args: list or str """ self.aug.set(aug_conf_path + "/directive[last() + 1]", directive) - if isinstance(arg, list): - for i, value in enumerate(arg, 1): + if isinstance(args, list): + for i, value in enumerate(args, 1): self.aug.set( "%s/directive[last()]/arg[%d]" % (aug_conf_path, i), value) else: - self.aug.set(aug_conf_path + "/directive[last()]/arg", arg) + self.aug.set(aug_conf_path + "/directive[last()]/arg", args) def find_dir(self, directive, arg=None, start=None): """Finds directive in the configuration. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 9304b634f..c156ebe66 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -28,11 +28,8 @@ class TwoVhost80Test(util.ApacheTest): def setUp(self): super(TwoVhost80Test, self).setUp() - with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator." - "mod_loaded") as mock_load: - mock_load.return_value = True - self.config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir) + self.config = util.get_apache_configurator( + self.config_path, self.config_dir, self.work_dir) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/two_vhost_80") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 24aa359ed..05224b56d 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -82,7 +82,7 @@ class ApacheParserTest(util.ApacheTest): from letsencrypt_apache.parser import get_aug_path self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), - "FakeDirective", "123") + "FakeDirective", ["123"]) matches = self.parser.find_dir("FakeDirective", "123") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf new file mode 100644 index 000000000..a1f4e05fc --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf @@ -0,0 +1,53 @@ +# Global configuration + +PidFile ${APACHE_PID_FILE} + +# +# Timeout: The number of seconds before receives and sends time out. +# +Timeout 300 + +# +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +# +KeepAlive On + +# These need to be set in /etc/apache2/envvars +User ${APACHE_RUN_USER} +Group ${APACHE_RUN_GROUP} + +ErrorLog ${APACHE_LOG_DIR}/error.log + +LogLevel warn + +# Include module configuration: +IncludeOptional mods-enabled/*.load +IncludeOptional mods-enabled/*.conf + + + Options FollowSymLinks + AllowOverride None + Require all denied + + + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + +# Include generic snippets of statements +IncludeOptional conf-enabled/*.conf + +# Include the virtual host configurations: +IncludeOptional sites-enabled/*.conf + +Define COMPLEX + +Define tls_port 1234 +Define example_path1 Documents/root + + +Include test.conf +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf new file mode 100644 index 000000000..f724756d5 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf @@ -0,0 +1 @@ +TESTDIRECTIVE ${tls_port} diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 81de82742..55349c366 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -44,10 +44,11 @@ def get_apache_configurator( """ backups = os.path.join(work_dir, "backups") - with mock.patch("letsencrypt_apache.configurator." - "subprocess.Popen") as mock_popen: - # This just states that the ssl module is already loaded - mock_popen().communicate.return_value = ("ssl_module", "") + with mock.patch("letsencrypt_apache.configurator.subprocess.Popen") as mock_popen: + # This indicates config_test passes + mock_popen().communicate.return_value = ("Fine output", "No problems") + mock_popen.returncode.return_value = 0 + # mock_popen().communicate.return_value = ("ssl_module", "") config = configurator.ApacheConfigurator( config=mock.MagicMock( apache_server_root=config_path, From 78fd81aed789675efe6649567b3ce98d8183cd0a Mon Sep 17 00:00:00 2001 From: James Kasten Date: Sun, 19 Jul 2015 02:22:10 -0700 Subject: [PATCH 18/52] Cleanup Apache --- .../letsencrypt_apache/configurator.py | 95 ++++++++++--------- .../letsencrypt_apache/dvsni.py | 14 ++- letsencrypt-apache/letsencrypt_apache/obj.py | 5 +- .../letsencrypt_apache/parser.py | 61 ++++++------ .../tests/configurator_test.py | 69 ++++++++++---- .../letsencrypt_apache/tests/dvsni_test.py | 44 ++++----- .../letsencrypt_apache/tests/parser_test.py | 11 ++- .../letsencrypt_apache/tests/util.py | 52 +++++----- 8 files changed, 192 insertions(+), 159 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 69b635bf0..af6dad395 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1,4 +1,5 @@ """Apache Configuration based off of Augeas Configurator.""" +# pylint: disable=too-many-lines import logging import os import re @@ -16,8 +17,6 @@ from letsencrypt import errors from letsencrypt import interfaces from letsencrypt import le_util -from letsencrypt.plugins import common - from letsencrypt_apache import augeas_configurator from letsencrypt_apache import constants from letsencrypt_apache import display_ops @@ -168,17 +167,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ vhost = self.choose_vhost(domain) + # This is done first so that ssl module is enabled and cert_path, + # cert_key... can all be parsed appropriately + self.prepare_server_https("443") + path = {} - path["cert_path"] = self.parser.find_dir(parser.case_i( - "SSLCertificateFile"), None, vhost.path) - path["cert_key"] = self.parser.find_dir(parser.case_i( - "SSLCertificateKeyFile"), None, vhost.path) + path["cert_path"] = self.parser.find_dir( + "SSLCertificateFile", None, vhost.path) + path["cert_key"] = self.parser.find_dir( + "SSLCertificateKeyFile", None, vhost.path) # Only include if a certificate chain is specified if chain_path is not None: path["chain_path"] = self.parser.find_dir( - parser.case_i("SSLCertificateChainFile"), None, vhost.path) + "SSLCertificateChainFile", None, vhost.path) if not path["cert_path"] or not path["cert_key"]: # Throw some can't find all of the directives error" @@ -186,7 +189,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "Cannot find a cert or key directive in %s. " "VirtualHost was not modified", vhost.path) # Presumably break here so that the virtualhost is not modified - return False + raise errors.PluginError( + "Unable to find cert and/or key directives") logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) @@ -235,7 +239,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost = self._find_best_vhost(target_name) if vhost is not None: if not vhost.ssl: - vhost = self.make_vhost_ssl(non_ssl_vhost) + vhost = self.make_vhost_ssl(vhost) self.assoc[target_name] = vhost return vhost @@ -311,13 +315,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for vhost in self.vhosts: all_names.update(vhost.names) for addr in vhost.addrs: - name = get_name_from_ip(addr) + name = self.get_name_from_ip(addr) if name: all_names.add(name) return all_names - def get_name_from_ip(self, addr): + def get_name_from_ip(self, addr): # pylint: disable=no-self-use """Returns a reverse dns name if available. :param addr: IP Address @@ -328,7 +332,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # If it isn't a private IP, do a reverse DNS lookup - if not private_ips_regex.match(addr.get_addr()): + if not ApacheConfigurator.private_ips_regex.match(addr.get_addr()): try: socket.inet_aton(addr.get_addr()) return socket.gethostbyaddr(addr.get_addr())[0] @@ -371,12 +375,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addrs.add(obj.Addr.fromstring(self.parser.get_arg(arg))) is_ssl = False - if self.parser.find_dir( - parser.case_i("SSLEngine"), parser.case_i("on"), path): + if self.parser.find_dir("SSLEngine", "on", path, exclude=False): is_ssl = True filename = get_file_path(path) is_enabled = self.is_site_enabled(filename) + vhost = obj.VirtualHost(filename, path, addrs, is_ssl, is_enabled) self._add_servernames(vhost) return vhost @@ -394,6 +398,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): paths = self.aug.match( ("/files%s/sites-available//*[label()=~regexp('%s')]" % (self.parser.root, parser.case_i("VirtualHost")))) + vhs = [] for path in paths: @@ -402,7 +407,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return vhs def is_name_vhost(self, target_addr): - r"""Returns if vhost is a name based vhost + """Returns if vhost is a name based vhost NameVirtualHost was deprecated in Apache 2.4 as all VirtualHosts are now NameVirtualHosts. If version is earlier than 2.4, check if addr @@ -421,9 +426,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # search for NameVirtualHost directive for ip_addr # note ip_addr can be FQDN although Apache does not recommend it return (self.version >= (2, 4) or - self.parser.find_dir( - parser.case_i("NameVirtualHost"), - parser.case_i(str(target_addr)))) + self.parser.find_dir("NameVirtualHost", str(target_addr))) def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. @@ -432,14 +435,17 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :type addr: :class:~letsencrypt_apache.obj.Addr """ - path = self.parser.add_dir_to_ifmodssl( - parser.get_aug_path( - self.parser.loc["name"]), "NameVirtualHost", [str(addr)]) + loc = parser.get_aug_path(self.parser.loc["name"]) + if addr.get_port == "443": + path = self.parser.add_dir_to_ifmodssl( + loc, "NameVirtualHost", [str(addr)]) + else: + path = self.parser.add_dir(loc, "NameVirtualHost", [str(addr)]) self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr self.save_notes += "\tDirective added to %s\n" % path - def _prepare_server_https(self, port): + def prepare_server_https(self, port): """Prepare the server for HTTPS. Make sure that the ssl_module is loaded and that the server @@ -454,9 +460,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check for Listen # Note: This could be made to also look for ip:443 combo - if not self.parser.find_dir(parser.case_i("Listen"), port): - logger.debug("No Listen {0} directive found. Setting the " - "Apache Server to Listen on port {0}".format(port)) + if not self.parser.find_dir("Listen", port): + logger.debug("No Listen %s directive found. Setting the " + "Apache Server to Listen on port %s", port, port) if port == "443": args = [port] @@ -540,7 +546,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vh_p = vh_p[0] # Update Addresses - ssl_addrs = self._update_ssl_vhosts_addrs(vh_p) + self._update_ssl_vhosts_addrs(vh_p) # Add directives self._add_dummy_ssl_directives(vh_p) @@ -552,7 +558,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # We know the length is one because of the assertion above # Create the Vhost object - ssl_vhost = self._create_vhost(vh_p[0]) + ssl_vhost = self._create_vhost(vh_p) self.vhosts.append(ssl_vhost) @@ -632,7 +638,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # See if the exact address appears in any other vhost for addr in vhost.addrs: - for test_vhost in self.vhosts: + for test_vh in self.vhosts: if (vhost.filep != test_vh.filep and addr in test_vh.addrs and not self.is_name_vhost(addr)): self.add_name_vhost(addr) @@ -746,10 +752,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: bool, int """ - rewrite_path = self.parser.find_dir( - parser.case_i("RewriteRule"), None, vhost.path) - redirect_path = self.parser.find_dir( - parser.case_i("Redirect"), None, vhost.path) + rewrite_path = self.parser.find_dir("RewriteRule", None, vhost.path) + redirect_path = self.parser.find_dir("Redirect", None, vhost.path) if redirect_path: # "Existing Redirect directive for virtualhost" @@ -942,9 +946,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for vhost in self.vhosts: if vhost.ssl: cert_path = self.parser.find_dir( - parser.case_i("SSLCertificateFile"), None, vhost.path) + "SSLCertificateFile", None, vhost.path) key_path = self.parser.find_dir( - parser.case_i("SSLCertificateKeyFile"), None, vhost.path) + "SSLCertificateKeyFile", None, vhost.path) # Can be removed once find directive can return ordered results if len(cert_path) != 1 or len(key_path) != 1: @@ -981,7 +985,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """Enables an available site, Apache restart required. .. todo:: This function should number subdomains before the domain vhost - .. todo:: Make sure link is not broken... :param vhost: vhost to enable @@ -996,9 +999,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if vhost.ssl: # TODO: Make this based on addresses - self._prepare_server_https("443") + self.prepare_server_https("443") if self.save_notes: - self.save("Enabled TLS for Apache") + self.save() if "/sites-available/" in vhost.filep: enabled_path = ("%s/sites-enabled/%s" % @@ -1034,20 +1037,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Modules can enable additional config files. Variables may be defined # within these new configuration sections. # Restart is not necessary as DUMP_RUN_CFG uses latest config. - self.parser.update_runtime_variables() + self.parser.update_runtime_variables(self.conf("ctl")) self.parser.modules.add(mod_name + "_module") - self.parser.modules.add("mod_" + mod_name) + self.parser.modules.add("mod_" + mod_name + ".c") def _enable_mod_debian(self, mod_name): """Assumes mods-available, mods-enabled layout.""" # TODO: This can be further updated to not require all files. if mod_name == "ssl": - self._enable_mod_debian_files(["ssl.conf", "ssl.load"]) + self._enable_mod_debian_files( + ["ssl.conf", "ssl.load"], "ssl_module") elif mod_name == "rewrite": - self._enable_mod_debian_files(["rewrite.load"]) + self._enable_mod_debian_files(["rewrite.load"], "rewrite_module") else: - raise NotImplemented + raise NotImplementedError def _enable_mod_debian_files(self, filenames, mod_name): """Move over all required files into mods-enabled.""" @@ -1058,11 +1062,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for filename in filenames: if not os.path.isfile(os.path.join(mods_available, filename)): raise errors.MisconfigurationError( - "Unable to enable module. Required files missing from " - "mods-available. %s" % str(filenames)) + "Unable to enable module. Required files missing from " + "mods-available. %s" % str(filenames)) # Register and symlink files - for filename in files: + for filename in filenames: enabled_path = os.path.join(mods_enabled, filename) if os.path.isfile(enabled_path): logger.debug( @@ -1131,6 +1135,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError("Unable to run apache2ctl") if proc.returncode != 0: + print proc.returncode # Enter recovery routine... logger.error("Apache Configtest failed\n%s\n%s", stdout, stderr) raise errors.MisconfigurationError( diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index ac0a3039a..e10c0cbf9 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -45,7 +45,7 @@ class ApacheDvsni(common.Dvsni): """ - def __init__(self): + def __init__(self, *args, **kwargs): super(ApacheDvsni, self).__init__(*args, **kwargs) self.challenge_conf = os.path.join( @@ -60,6 +60,10 @@ class ApacheDvsni(common.Dvsni): # About to make temporary changes to the config self.configurator.save() + # Prepare the server for HTTPS + self.configurator.prepare_server_https( + str(self.configurator.config.dvsni_port)) + responses = [] # Create all of the challenge certs @@ -68,13 +72,8 @@ class ApacheDvsni(common.Dvsni): # Setup the configuration dvsni_addrs = self._mod_config() - self.configurator.make_addrs_sni_ready(dvsni_addrs) - # Prepare the server for HTTPS - self.configurator._prepare_https_server( - str(self.configurator.config.dvsni_port)) - # Save reversible changes self.configurator.save("SNI Challenge", True) @@ -96,7 +95,7 @@ class ApacheDvsni(common.Dvsni): achall_addrs = self.get_dvsni_addrs(achall) dvsni_addrs.update(achall_addrs) - config_text += self._get_config_text(self.achalls, achall_addrs) + config_text += self._get_config_text(achall, achall_addrs) config_text += "\n" @@ -114,7 +113,6 @@ class ApacheDvsni(common.Dvsni): vhost = self.configurator.choose_vhost(achall.domain) # TODO: Checkout _default_ rules. - # TODO: Need to separate out test mode and normal mode for DVSNI addrs dvsni_addrs = set() default_addr = obj.Addr(("*", self.configurator.config.dvsni_port)) diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index 956b6999f..ae84f7d26 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -2,7 +2,7 @@ from letsencrypt.plugins import common class Addr(common.Addr): - + """Represents an Apache address.""" def __eq__(self, other): """This is defined as equalivalent within Apache. @@ -16,7 +16,8 @@ class Addr(common.Addr): return False def is_wildcard(self): - return tup[1] == "*" or not tup[1] + """Returns if address has a wildcard port.""" + return self.tup[1] == "*" or not self.tup[1] def get_sni_addr(self, port): """Returns the least specific address that resolves on the port. diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index ffbacc066..a0bc6fd12 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -1,5 +1,4 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" -import collections import fnmatch import itertools import logging @@ -13,10 +12,6 @@ from letsencrypt import errors logger = logging.getLogger(__name__) -arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") -fnmatch_chars = set(["*", "?", "\\", "[", "]"]) - - class ApacheParser(object): """Class handles the fine details of parsing the Apache Configuration. @@ -26,6 +21,9 @@ class ApacheParser(object): directory. Without trailing slash. """ + arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") + fnmatch_chars = set(["*", "?", "\\", "[", "]"]) + def __init__(self, aug, root, ssl_options, ctl): # This uses the binary, so it can be done first. # https://httpd.apache.org/docs/2.4/mod/core.html#define @@ -59,7 +57,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - matches = self.find_dir(case_i("LoadModule")) + matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -101,7 +99,7 @@ class ApacheParser(object): self.variables = variables - def _get_runtime_cfg(self, ctl): + def _get_runtime_cfg(self, ctl): # pylint: disable=no-self-use """Get runtime configuration info. :returns: stdout from DUMP_RUN_CFG @@ -116,8 +114,7 @@ class ApacheParser(object): except (OSError, ValueError): logger.error( - "Error accessing {0} for runtime parameters!{1}".format( - ctl, os.linesep)) + "Error accessing %s for runtime parameters!%s", ctl, os.linesep) raise errors.MisconfigurationError( "Error accessing loaded Apache parameters: %s", ctl) # Small errors that do not impede @@ -129,7 +126,7 @@ class ApacheParser(object): return stdout - def _filter_args_num(self, matches, args): + def _filter_args_num(self, matches, args): # pylint: disable=no-self-use """Filter out directives with specific number of arguments. This function makes the assumption that all related arguments are given @@ -223,7 +220,7 @@ class ApacheParser(object): else: self.aug.set(aug_conf_path + "/directive[last()]/arg", args) - def find_dir(self, directive, arg=None, start=None): + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. Recursively searches through config files to find directives @@ -241,12 +238,13 @@ class ApacheParser(object): compatibility. :param str directive: Directive to look for - :param arg: Specific value directive must have, None if all should be considered :type arg: str or None :param str start: Beginning Augeas path to begin looking + :param bool exclude: Whether or not to exclude directives based on + variables and enabled modules """ # Cannot place member variable in the definition of the function so... @@ -265,32 +263,33 @@ class ApacheParser(object): # includes = self.aug.match(start + # "//* [self::directive='Include']/* [label()='arg']") - regex = "(%s)|(%s)|(%s)" % (directive, + regex = "(%s)|(%s)|(%s)" % (case_i(directive), case_i("Include"), case_i("IncludeOptional")) matches = self.aug.match( "%s//*[self::directive=~regexp('%s')]" % (start, regex)) - matches = self._exclude_dirs(matches) - + if exclude: + matches = self._exclude_dirs(matches) if arg is None: arg_suffix = "/arg" else: - arg_suffix = "/*[self::arg=~regexp('%s')]" % arg + arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) ordered_matches = [] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include for match in matches: - dir = self.aug.get(match).lower() - if dir == "include" or dir == "includeoptional": + dir_ = self.aug.get(match).lower() + if dir_ == "include" or dir_ == "includeoptional": # start[6:] to strip off /files ordered_matches.extend(self.find_dir( directive, arg, self._get_include_path( self.get_arg(match + "/arg")))) - else: + # This additionally allows Include + if dir_ == directive.lower(): ordered_matches.extend(self.aug.match(match + arg_suffix)) return ordered_matches @@ -302,11 +301,14 @@ class ApacheParser(object): """ value = self.aug.get(match) - variables = arg_var_interpreter.findall(value) + variables = ApacheParser.arg_var_interpreter.findall(value) for var in variables: # Strip off ${ and } - value = value.replace(var, self.variables[var[2:-1]]) + try: + value = value.replace(var, self.variables[var[2:-1]]) + except KeyError: + raise errors.PluginError("Error Parsing variable: %s" % var) return value @@ -317,14 +319,14 @@ class ApacheParser(object): valid_matches = [] for match in matches: - for filter in filters: - if not self._pass_filter(match, filter): + for filter_ in filters: + if not self._pass_filter(match, filter_): break else: valid_matches.append(match) return valid_matches - def _pass_filter(self, match, filter): + def _pass_filter(self, match, filter_): """Determine if directive passes a filter. :param str match: Augeas path @@ -333,7 +335,7 @@ class ApacheParser(object): """ match_l = match.lower() - last_match_idx = match_l.find(filter[0]) + last_match_idx = match_l.find(filter_[0]) while last_match_idx != -1: # Check args @@ -341,10 +343,10 @@ class ApacheParser(object): expression = self.aug.get(match[:end_of_if] + "/arg") expected = not expression.startswith("!") - if expected != expression in filter[1]: + if expected != (expression in filter_[1]): return False - last_match_idx = match_l.find(filter[0], end_of_if) + last_match_idx = match_l.find(filter_[0], end_of_if) return True @@ -382,7 +384,7 @@ class ApacheParser(object): # then reassemble split_arg = arg.split("/") for idx, split in enumerate(split_arg): - if any(char in fnmatch_chars for char in split): + if any(char in ApacheParser.fnmatch_chars for char in split): # Turn it into a augeas regex # TODO: Can this instead be an augeas glob instead of regex split_arg[idx] = ("* [label()=~regexp('%s')]" % @@ -529,8 +531,7 @@ class ApacheParser(object): # in hierarchy via direct include # httpd.conf was very common as a user file in Apache 2.2 if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and - self.find_dir( - case_i("Include"), case_i("httpd.conf"), root)): + self.find_dir("Include", "httpd.conf", root)): return os.path.join(self.root, "httpd.conf") else: return os.path.join(self.root, "apache2.conf") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index c156ebe66..111e82a2d 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -1,6 +1,5 @@ """Test for letsencrypt_apache.configurator.""" import os -import re import shutil import unittest @@ -12,18 +11,16 @@ from letsencrypt import achallenges from letsencrypt import errors from letsencrypt import le_util -from letsencrypt.plugins import common - from letsencrypt.tests import acme_util from letsencrypt_apache import configurator -from letsencrypt_apache import parser +from letsencrypt_apache import obj from letsencrypt_apache.tests import util class TwoVhost80Test(util.ApacheTest): - """Test two standard well configured HTTP vhosts.""" + """Test two standard well-configured HTTP vhosts.""" def setUp(self): super(TwoVhost80Test, self).setUp() @@ -48,7 +45,7 @@ class TwoVhost80Test(util.ApacheTest): """Make sure all vhosts are being properly found. .. note:: If test fails, only finding 1 Vhost... it is likely that - it is a problem with is_enabled. + it is a problem with is_enabled. If finding only 3, likely is_ssl """ vhs = self.config.get_virtual_hosts() @@ -60,6 +57,8 @@ class TwoVhost80Test(util.ApacheTest): if vhost == truth: found += 1 break + else: + raise Exception("Missed: %s" % vhost) self.assertEqual(found, 4) @@ -77,7 +76,35 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].filep)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].filep)) - def test_deploy_cert(self): + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_enable_mod(self, mock_popen): + mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + mock_popen().returncode = 0 + + self.config.enable_mod("ssl") + for filename in ["ssl.conf", "ssl.load"]: + self.assertTrue( + os.path.isfile(os.path.join( + self.config.conf("server-root"), "mods-enabled", filename))) + + self.assertTrue("ssl_module" in self.config.parser.modules) + self.assertTrue("mod_ssl.c" in self.config.parser.modules) + + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_enable_site(self, mock_popen): + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + + # Default 443 vhost + self.assertFalse(self.vh_truth[1].enabled) + self.config.enable_site(self.vh_truth[1]) + self.assertTrue(self.vh_truth[1].enabled) + + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_deploy_cert(self, mock_popen): + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] self.config.deploy_cert( @@ -85,15 +112,17 @@ class TwoVhost80Test(util.ApacheTest): "example/cert.pem", "example/key.pem", "example/cert_chain.pem") self.config.save() + # Verify ssl_module was enabled. + self.assertTrue(self.vh_truth[1].enabled) + self.assertTrue("ssl_module" in self.config.parser.modules) + loc_cert = self.config.parser.find_dir( - parser.case_i("sslcertificatefile"), - re.escape("example/cert.pem"), self.vh_truth[1].path) + "sslcertificatefile", "example/cert.pem", self.vh_truth[1].path) loc_key = self.config.parser.find_dir( - parser.case_i("sslcertificateKeyfile"), - re.escape("example/key.pem"), self.vh_truth[1].path) + "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path) loc_chain = self.config.parser.find_dir( - parser.case_i("SSLCertificateChainFile"), - re.escape("example/cert_chain.pem"), self.vh_truth[1].path) + "SSLCertificateChainFile", "example/cert_chain.pem", + self.vh_truth[1].path) # Verify one directive was found in the correct file self.assertEqual(len(loc_cert), 1) @@ -109,15 +138,15 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[1].filep) def test_is_name_vhost(self): - addr = common.Addr.fromstring("*:80") + addr = obj.Addr.fromstring("*:80") self.assertTrue(self.config.is_name_vhost(addr)) self.config.version = (2, 2) self.assertFalse(self.config.is_name_vhost(addr)) def test_add_name_vhost(self): - self.config.add_name_vhost("*:443") + self.config.add_name_vhost(obj.Addr.fromstring("*:443")) self.assertTrue(self.config.parser.find_dir( - "NameVirtualHost", re.escape("*:443"))) + "NameVirtualHost", "*:443")) def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -130,17 +159,15 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([common.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) self.assertEqual(ssl_vhost.names, set(["encryption-example.demo"])) self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) self.assertTrue(self.config.parser.find_dir( - "SSLCertificateFile", None, ssl_vhost.path)) + "SSLCertificateFile", None, ssl_vhost.path, False)) self.assertTrue(self.config.parser.find_dir( - "SSLCertificateKeyFile", None, ssl_vhost.path)) - self.assertTrue(self.config.parser.find_dir( - "Include", self.ssl_options, ssl_vhost.path)) + "SSLCertificateKeyFile", None, ssl_vhost.path, False)) self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]), self.config.is_name_vhost(ssl_vhost)) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py index 933656e94..b6f9bc5e0 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py @@ -6,9 +6,9 @@ import mock from acme import challenges -from letsencrypt.plugins import common from letsencrypt.plugins import common_test +from letsencrypt_apache import obj from letsencrypt_apache.tests import util @@ -20,11 +20,9 @@ class DvsniPerformTest(util.ApacheTest): def setUp(self): super(DvsniPerformTest, self).setUp() - with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator." - "mod_loaded") as mock_load: - mock_load.return_value = True - config = util.get_apache_configurator( - self.config_path, self.config_dir, self.work_dir) + config = util.get_apache_configurator( + self.config_path, self.config_dir, self.work_dir) + config.config.dvsni_port = 443 from letsencrypt_apache import dvsni self.sni = dvsni.ApacheDvsni(config) @@ -38,7 +36,11 @@ class DvsniPerformTest(util.ApacheTest): resp = self.sni.perform() self.assertEqual(len(resp), 0) - def test_perform1(self): + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_perform1(self, mock_popen): + mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + mock_popen().returncode = 0 + achall = self.achalls[0] self.sni.add_chall(achall) mock_setup_cert = mock.MagicMock( @@ -53,12 +55,14 @@ class DvsniPerformTest(util.ApacheTest): # Check to make sure challenge config path is included in apache config. self.assertEqual( len(self.sni.configurator.parser.find_dir( - "Include", self.sni.challenge_conf)), - 1) + "Include", self.sni.challenge_conf)), 1) self.assertEqual(len(responses), 1) self.assertEqual(responses[0].s, "randomS1") def test_perform2(self): + # Avoid load module + self.sni.configurator.parser.modules.add("ssl_module") + for achall in self.achalls: self.sni.add_chall(achall) @@ -89,13 +93,8 @@ class DvsniPerformTest(util.ApacheTest): def test_mod_config(self): for achall in self.achalls: self.sni.add_chall(achall) - v_addr1 = [common.Addr(("1.2.3.4", "443")), - common.Addr(("5.6.7.8", "443"))] - v_addr2 = [common.Addr(("127.0.0.1", "443"))] - ll_addr = [] - ll_addr.append(v_addr1) - ll_addr.append(v_addr2) - self.sni._mod_config(ll_addr) # pylint: disable=protected-access + + self.sni._mod_config() # pylint: disable=protected-access self.sni.configurator.save() self.sni.configurator.parser.find_dir( @@ -109,15 +108,10 @@ class DvsniPerformTest(util.ApacheTest): vhs.append(self.sni.configurator._create_vhost(match)) self.assertEqual(len(vhs), 2) for vhost in vhs: - if vhost.addrs == set(v_addr1): - self.assertEqual( - vhost.names, - set([self.achalls[0].nonce_domain])) - else: - self.assertEqual(vhost.addrs, set(v_addr2)) - self.assertEqual( - vhost.names, - set([self.achalls[1].nonce_domain])) + self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")])) + self.assertTrue( + vhost.names == set([self.achalls[0].nonce_domain]) or + vhost.names == set([self.achalls[1].nonce_domain])) if __name__ == "__main__": diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 05224b56d..3a74055ad 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -54,11 +54,11 @@ class ApacheParserTest(util.ApacheTest): self.assertTrue(matches) def test_find_dir(self): - from letsencrypt_apache.parser import case_i - test = self.parser.find_dir(case_i("Listen"), "443") + test = self.parser.find_dir("Listen", "80") # This will only look in enabled hosts - test2 = self.parser.find_dir(case_i("documentroot")) - self.assertEqual(len(test), 2) + test2 = self.parser.find_dir("documentroot") + + self.assertEqual(len(test), 1) self.assertEqual(len(test2), 3) def test_add_dir(self): @@ -80,6 +80,9 @@ class ApacheParserTest(util.ApacheTest): """ from letsencrypt_apache.parser import get_aug_path + # This makes sure that find_dir will work + self.parser.modules.add("mod_ssl.c") + self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), "FakeDirective", ["123"]) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 55349c366..ae2f25b9f 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -35,8 +35,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods def get_apache_configurator( - config_path, config_dir, work_dir, version=(2, 4, 7), - conf=mock.MagicMock()): + config_path, config_dir, work_dir, version=(2, 4, 7), conf=None): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -44,25 +43,30 @@ def get_apache_configurator( """ backups = os.path.join(work_dir, "backups") - with mock.patch("letsencrypt_apache.configurator.subprocess.Popen") as mock_popen: - # This indicates config_test passes - mock_popen().communicate.return_value = ("Fine output", "No problems") - mock_popen.returncode.return_value = 0 - # mock_popen().communicate.return_value = ("ssl_module", "") - config = configurator.ApacheConfigurator( - config=mock.MagicMock( - apache_server_root=config_path, - apache_le_vhost_ext=constants.CLI_DEFAULTS["le_vhost_ext"], - backup_dir=backups, - config_dir=config_dir, - temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), - in_progress_dir=os.path.join(backups, "IN_PROGRESS"), - work_dir=work_dir), - name="apache", - version=version) - config.conf = conf + with mock.patch("letsencrypt_apache.configurator." + "subprocess.Popen") as mock_popen: + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + # This indicates config_test passes + mock_popen().communicate.return_value = ("Fine output", "No problems") + mock_popen().returncode = 0 - config.prepare() + config = configurator.ApacheConfigurator( + config=mock.MagicMock( + apache_server_root=config_path, + apache_le_vhost_ext=constants.CLI_DEFAULTS["le_vhost_ext"], + backup_dir=backups, + config_dir=config_dir, + temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), + in_progress_dir=os.path.join(backups, "IN_PROGRESS"), + work_dir=work_dir), + name="apache", + version=version) + # This allows testing scripts to set it a bit more quickly + if conf is not None: + config.conf = conf + + config.prepare() return config @@ -77,21 +81,21 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "encryption-example.conf"), os.path.join(aug_pre, "encryption-example.conf/VirtualHost"), - set([common.Addr.fromstring("*:80")]), + set([obj.Addr.fromstring("*:80")]), False, True, set(["encryption-example.demo"])), obj.VirtualHost( os.path.join(prefix, "default-ssl.conf"), os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), - set([common.Addr.fromstring("_default_:443")]), True, False), + set([obj.Addr.fromstring("_default_:443")]), True, False), obj.VirtualHost( os.path.join(prefix, "000-default.conf"), os.path.join(aug_pre, "000-default.conf/VirtualHost"), - set([common.Addr.fromstring("*:80")]), False, True, + set([obj.Addr.fromstring("*:80")]), False, True, set(["ip-172-30-0-17"])), obj.VirtualHost( os.path.join(prefix, "letsencrypt.conf"), os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"), - set([common.Addr.fromstring("*:80")]), False, True, + set([obj.Addr.fromstring("*:80")]), False, True, set(["letsencrypt.demo"])), ] return vh_truth From 32068806731ecae8e7bf5ca3115f766fe6949d69 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Sun, 19 Jul 2015 16:48:27 -0700 Subject: [PATCH 19/52] add complex parsing tests --- .../letsencrypt_apache/parser.py | 21 ++-- .../tests/complex_parsing_test.py | 101 ++++++++++++++++++ .../tests/configurator_test.py | 27 ++++- .../letsencrypt_apache/tests/dvsni_test.py | 2 +- .../letsencrypt_apache/tests/parser_test.py | 25 ++--- .../testdata/complex_parsing/apache2.conf | 4 +- .../complex_parsing/conf-enabled/dummy.conf | 9 ++ .../tests/testdata/complex_parsing/test.conf | 1 - .../complex_parsing/test_fnmatch.conf | 1 + .../complex_parsing/test_variables.conf | 65 +++++++++++ .../letsencrypt_apache/tests/util.py | 31 +++++- 11 files changed, 253 insertions(+), 34 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/conf-enabled/dummy.conf delete mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_fnmatch.conf create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_variables.conf diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index a0bc6fd12..48234bfe5 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -126,7 +126,7 @@ class ApacheParser(object): return stdout - def _filter_args_num(self, matches, args): # pylint: disable=no-self-use + def filter_args_num(self, matches, args): # pylint: disable=no-self-use """Filter out directives with specific number of arguments. This function makes the assumption that all related arguments are given @@ -142,16 +142,16 @@ class ApacheParser(object): """ filtered = [] if args == 1: - for i in range(matches): + for i in range(len(matches)): if matches[i].endswith("/arg"): filtered.append(matches[i][:-4]) else: - for i in range(matches): - if matches[i].endswith("/arg[%d]", args): + for i in range(len(matches)): + if matches[i].endswith("/arg[%d]" % args): # Make sure we don't cause an IndexError (end of list) # Check to make sure arg + 1 doesn't exist if (i == (len(matches) - 1) or - not matches[i + 1].endswith("/arg[%d]" % args + 1)): + not matches[i + 1].endswith("/arg[%d]" % (args + 1))): filtered.append(matches[i][:-len("/arg[%d]" % args)]) return filtered @@ -340,11 +340,16 @@ class ApacheParser(object): while last_match_idx != -1: # Check args end_of_if = match_l.find("/", last_match_idx) + # This should be aug.get (vars are not used e.g. parser.aug_get) expression = self.aug.get(match[:end_of_if] + "/arg") - expected = not expression.startswith("!") - if expected != (expression in filter_[1]): - return False + if expression.startswith("!"): + # Strip off "!" + if expression[1:] in filter_[1]: + return False + else: + if expression not in filter_[1]: + return False last_match_idx = match_l.find(filter_[0], end_of_if) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py new file mode 100644 index 000000000..7281061e4 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py @@ -0,0 +1,101 @@ +"""Tests for letsencrypt_apache.parser.""" +import os +import shutil +import unittest + +from letsencrypt_apache.tests import util + + +class ComplexParserTest(util.ParserTest): + """Apache Parser Test.""" + + def setUp(self): # pylint: disable=arguments-differ + super(ComplexParserTest, self).setUp( + "complex_parsing", "complex_parsing") + + self.setup_variables() + # This needs to happen after due to setup_variables not being run + # until after + self.parser._init_modules() # pylint: disable=protected-access + + def tearDown(self): + shutil.rmtree(self.temp_dir) + shutil.rmtree(self.config_dir) + shutil.rmtree(self.work_dir) + + def setup_variables(self): + """Set up variables for parser.""" + self.parser.variables.update( + { + "COMPLEX": "", + "tls_port": "1234", + "fnmatch_filename": "test_fnmatch.conf", + } + ) + + def test_filter_args_num(self): + matches = self.parser.find_dir("TestArgsDirective") + + self.assertEqual(len(self.parser.filter_args_num(matches, 1)), 3) + self.assertEqual(len(self.parser.filter_args_num(matches, 2)), 2) + self.assertEqual(len(self.parser.filter_args_num(matches, 3)), 1) + + def test_basic_variable_parsing(self): + matches = self.parser.find_dir("TestVariablePort") + + self.assertEqual(len(matches), 1) + self.assertEqual(self.parser.get_arg(matches[0]), "1234") + + def test_basic_ifdefine(self): + self.assertEqual(len(self.parser.find_dir("VAR_DIRECTIVE")), 2) + self.assertEqual(len(self.parser.find_dir("INVALID_VAR_DIRECTIVE")), 0) + + def test_basic_ifmodule(self): + self.assertEqual(len(self.parser.find_dir("MOD_DIRECTIVE")), 2) + self.assertEqual( + len(self.parser.find_dir("INVALID_MOD_DIRECTIVE")), 0) + + def test_nested(self): + self.assertEqual(len(self.parser.find_dir("NESTED_DIRECTIVE")), 3) + self.assertEqual( + len(self.parser.find_dir("INVALID_NESTED_DIRECTIVE")), 0) + + + def test_load_modules(self): + """If only first is found, there is bad variable parsing.""" + self.assertTrue("status_module" in self.parser.modules) + self.assertTrue("mod_status.c" in self.parser.modules) + + # This is in an IfDefine + self.assertTrue("ssl_module" in self.parser.modules) + self.assertTrue("mod_ssl.c" in self.parser.modules) + + def verify_fnmatch(self, arg, hit=True): + """Test if Include was correctly parsed.""" + from letsencrypt_apache import parser + self.parser.add_dir(parser.get_aug_path(self.parser.loc["default"]), + "Include", [arg]) + if hit: + self.assertTrue(self.parser.find_dir("FNMATCH_DIRECTIVE")) + else: + self.assertFalse(self.parser.find_dir("FNMATCH_DIRECTIVE")) + + def test_include(self): + self.verify_fnmatch("test_fnmatch.?onf") + + def test_include_complex(self): + self.verify_fnmatch("../complex_parsing/[te][te]st_*.?onf") + + def test_include_fullpath(self): + self.verify_fnmatch(os.path.join(self.config_path, "test_fnmatch.conf")) + + def test_include_variable(self): + self.verify_fnmatch("../complex_parsing/${fnmatch_filename}") + + def test_include_missing(self): + # This should miss + self.verify_fnmatch("test_*.onf", False) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 111e82a2d..e68b21945 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -22,7 +22,7 @@ from letsencrypt_apache.tests import util class TwoVhost80Test(util.ApacheTest): """Test two standard well-configured HTTP vhosts.""" - def setUp(self): + def setUp(self): # pylint: disable=arguments-differ super(TwoVhost80Test, self).setUp() self.config = util.get_apache_configurator( @@ -137,6 +137,18 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(configurator.get_file_path(loc_chain[0]), self.vh_truth[1].filep) + def test_deploy_cert_invalid_vhost(self): + self.config.parser.modules.add("ssl_module") + mock_find = mock.MagicMock() + mock_find.return_value = [] + self.config.parser.find_dir = mock_find + + # Get the default 443 vhost + self.config.assoc["random.demo"] = self.vh_truth[1] + self.assertRaises( + errors.PluginError, self.config.deploy_cert, "random.demo", + "example/cert.pem", "example/key.pem", "example/cert_chain.pem") + def test_is_name_vhost(self): addr = obj.Addr.fromstring("*:80") self.assertTrue(self.config.is_name_vhost(addr)) @@ -148,6 +160,19 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:443")) + def test_prepare_server_https(self): + self.config.parser.modules.add("ssl_module") + mock_find = mock.Mock() + mock_add_dir = mock.Mock() + mock_find.return_value = [] + + # This will test the Add listen + self.config.parser.find_dir = mock_find + self.config.parser.add_dir_to_ifmodssl = mock_add_dir + + self.config.prepare_server_https("443") + self.assertTrue(mock_add_dir.called) + def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py index b6f9bc5e0..b0aec4f8a 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py @@ -17,7 +17,7 @@ class DvsniPerformTest(util.ApacheTest): achalls = common_test.DvsniTest.achalls - def setUp(self): + def setUp(self): # pylint: disable=arguments-differ super(DvsniPerformTest, self).setUp() config = util.get_apache_configurator( diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 3a74055ad..49575e977 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -1,34 +1,21 @@ """Tests for letsencrypt_apache.parser.""" import os import shutil -import sys import unittest import augeas import mock -import zope.component from letsencrypt import errors -from letsencrypt.display import util as display_util from letsencrypt_apache.tests import util -class ApacheParserTest(util.ApacheTest): +class BasicParserTest(util.ParserTest): """Apache Parser Test.""" - def setUp(self): - super(ApacheParserTest, self).setUp() - - zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) - - from letsencrypt_apache.parser import ApacheParser - self.aug = augeas.Augeas( - flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD) - with mock.patch("letsencrypt_apache.parser.ApacheParser." - "update_runtime_variables"): - self.parser = ApacheParser( - self.aug, self.config_path, self.ssl_options, "dummy_ctl_path") + def setUp(self): # pylint: disable=arguments-differ + super(BasicParserTest, self).setUp() def tearDown(self): shutil.rmtree(self.temp_dir) @@ -61,6 +48,10 @@ class ApacheParserTest(util.ApacheTest): self.assertEqual(len(test), 1) self.assertEqual(len(test2), 3) + def test_filter_args_num(self): + # TODO: TEST 2, TEST 1 + pass + def test_add_dir(self): aug_default = "/files" + self.parser.loc["default"] self.parser.add_dir(aug_default, "AddDirective", "test") @@ -115,7 +106,7 @@ class ApacheParserTest(util.ApacheTest): class ParserInitTest(util.ApacheTest): - def setUp(self): + def setUp(self): # pylint: disable=arguments-differ super(ParserInitTest, self).setUp() self.aug = augeas.Augeas( flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf index a1f4e05fc..b7b6a9be2 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf @@ -46,8 +46,8 @@ IncludeOptional sites-enabled/*.conf Define COMPLEX Define tls_port 1234 -Define example_path1 Documents/root +Define fnmatch_filename test_fnmatch.conf -Include test.conf +Include test_variables.conf # vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/conf-enabled/dummy.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/conf-enabled/dummy.conf new file mode 100644 index 000000000..1e5307780 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/conf-enabled/dummy.conf @@ -0,0 +1,9 @@ +# 3 - one arg directives +# 2 - two arg directives +# 1 - three arg directives +TestArgsDirective one_arg +TestArgsDirective one_arg two_arg +TestArgsDirective one_arg +TestArgsDirective one_arg two_arg +TestArgsDirective one_arg two_arg three_arg +TestArgsDirective one_arg diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf deleted file mode 100644 index f724756d5..000000000 --- a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test.conf +++ /dev/null @@ -1 +0,0 @@ -TESTDIRECTIVE ${tls_port} diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_fnmatch.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_fnmatch.conf new file mode 100644 index 000000000..4e6b84edf --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_fnmatch.conf @@ -0,0 +1 @@ +FNMATCH_DIRECTIVE Success diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_variables.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_variables.conf new file mode 100644 index 000000000..a38191837 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_variables.conf @@ -0,0 +1,65 @@ +TestVariablePort ${tls_port} + +LoadModule status_module modules/mod_status.so + +# Basic IfDefine + + VAR_DIRECTIVE success + LoadModule ssl_module modules/mod_ssl.so + + + + INVALID_VAR_DIRECTIVE failure + + + + INVALID_VAR_DIRECTIVE failure + + + + VAR_DIRECTIVE failure + + + +# Basic IfModule + + MOD_DIRECTIVE Success + + + + INVALID_MOD_DIRECTIVE failure + + + + INVALID_MOD_DIRECTIVE failure + + + + MOD_DIRECTIVE Success + + +# Nested Tests + + + NESTED_DIRECTIVE success + + + NESTED_DIRECTIVE success + + + + INVALID_NESTED_DIRECTIVE failure + + + + + INVALID_NESTED_DIRECTIVE failure + + + INVALID_NESTED_DIRECTIVE failure + + + + NESTED_DIRECTIVE success + + diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index ae2f25b9f..8b54b08a4 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -1,9 +1,14 @@ """Common utilities for letsencrypt_apache.""" import os import pkg_resources +import sys import unittest +import augeas import mock +import zope.component + +from letsencrypt.display import util as display_util from letsencrypt.plugins import common @@ -14,19 +19,20 @@ from letsencrypt_apache import obj class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods - def setUp(self): + def setUp(self, test_dir="debian_apache_2_4/two_vhost_80", + config_root="debian_apache_2_4/two_vhost_80/apache2"): + # pylint: disable=arguments-differ super(ApacheTest, self).setUp() self.temp_dir, self.config_dir, self.work_dir = common.dir_setup( - test_dir="debian_apache_2_4/two_vhost_80", + test_dir=test_dir, pkg="letsencrypt_apache.tests") self.ssl_options = common.setup_ssl_options( self.config_dir, constants.MOD_SSL_CONF_SRC, constants.MOD_SSL_CONF_DEST) - self.config_path = os.path.join( - self.temp_dir, "debian_apache_2_4/two_vhost_80/apache2") + self.config_path = os.path.join(self.temp_dir, config_root) self.rsa256_file = pkg_resources.resource_filename( "letsencrypt.tests", os.path.join("testdata", "rsa256_key.pem")) @@ -34,6 +40,23 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods "letsencrypt.tests", os.path.join("testdata", "rsa256_key.pem")) +class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods + + def setUp(self, test_dir="debian_apache_2_4/two_vhost_80", + config_root="debian_apache_2_4/two_vhost_80/apache2"): + super(ParserTest, self).setUp(test_dir, config_root) + + zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) + + from letsencrypt_apache.parser import ApacheParser + self.aug = augeas.Augeas( + flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD) + with mock.patch("letsencrypt_apache.parser.ApacheParser." + "update_runtime_variables"): + self.parser = ApacheParser( + self.aug, self.config_path, self.ssl_options, "dummy_ctl_path") + + def get_apache_configurator( config_path, config_dir, work_dir, version=(2, 4, 7), conf=None): """Create an Apache Configurator with the specified options. From 6a8590b4ccaafb375a449f4e6c1f6f6638f503a5 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Sun, 19 Jul 2015 19:49:44 -0700 Subject: [PATCH 20/52] Improved Apache and unittests --- .../letsencrypt_apache/configurator.py | 5 +-- .../letsencrypt_apache/dvsni.py | 2 +- letsencrypt-apache/letsencrypt_apache/obj.py | 4 +- .../letsencrypt_apache/parser.py | 42 ++++++------------- .../letsencrypt_apache/tests/obj_test.py | 32 ++++++++++++++ .../letsencrypt_apache/tests/parser_test.py | 21 +++------- .../letsencrypt_apache/tests/util.py | 2 +- 7 files changed, 56 insertions(+), 52 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index af6dad395..07e1bb2bc 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -137,8 +137,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.config_test() self.parser = parser.ApacheParser( - self.aug, self.conf("server-root"), self.mod_ssl_conf, - self.conf("ctl")) + self.aug, self.conf("server-root"), self.conf("ctl")) # Check for errors in parsing files with Augeas self.check_parsing_errors("httpd.aug") @@ -622,7 +621,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.parser.loc["ssl_options"]) + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_name_vhost_if_necessary(self, vhost): """Add NameVirtualHost Directives if necessary for new vhost. diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index e10c0cbf9..616a1a1ef 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -165,7 +165,7 @@ class ApacheDvsni(common.Dvsni): # https://docs.python.org/2.7/reference/lexical_analysis.html return self.VHOST_TEMPLATE.format( vhost=ips, server_name=achall.nonce_domain, - ssl_options_conf_path=self.configurator.parser.loc["ssl_options"], + ssl_options_conf_path=self.configurator.mod_ssl_conf, cert_path=self.get_cert_path(achall), key_path=self.get_key_path(achall), document_root=document_root).replace("\n", os.linesep) diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index ae84f7d26..a2df429c3 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -15,6 +15,9 @@ class Addr(common.Addr): and self.is_wildcard() and other.is_wildcard())) return False + def __ne__(self, other): + return not self.__eq__(other) + def is_wildcard(self): """Returns if address has a wildcard port.""" return self.tup[1] == "*" or not self.tup[1] @@ -35,7 +38,6 @@ class Addr(common.Addr): return self.get_addr_obj(port) - class VirtualHost(object): # pylint: disable=too-few-public-methods """Represents an Apache Virtualhost. diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 48234bfe5..8feb54fa8 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -24,7 +24,7 @@ class ApacheParser(object): arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") fnmatch_chars = set(["*", "?", "\\", "[", "]"]) - def __init__(self, aug, root, ssl_options, ctl): + def __init__(self, aug, root, ctl): # This uses the binary, so it can be done first. # https://httpd.apache.org/docs/2.4/mod/core.html#define # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine @@ -32,24 +32,27 @@ class ApacheParser(object): self.variables = {} self.update_runtime_variables(ctl) - # Find configuration root and make sure augeas can parse it. self.aug = aug + # Find configuration root and make sure augeas can parse it. self.root = os.path.abspath(root) - self.loc = self._set_locations(ssl_options) + self.loc = {"root": self._find_config_root()} self._parse_file(self.loc["root"]) - # Must also attempt to parse sites-available or equivalent - # Sites-available is not included naturally in configuration - self._parse_file(os.path.join(self.root, "sites-available") + "/*") - # This problem has been fixed in Augeas 1.0 self.standardize_excl() # Temporarily set modules to be empty, so that find_dirs can work # https://httpd.apache.org/docs/2.4/mod/core.html#ifmodule + # This needs to come before locations are set. self.modules = set() self._init_modules() + self.loc.update(self._set_locations(self.loc["root"])) + + # Must also attempt to parse sites-available or equivalent + # Sites-available is not included naturally in configuration + self._parse_file(os.path.join(self.root, "sites-available") + "/*") + def _init_modules(self): """Iterates on the configuration until no new modules are loaded. @@ -493,14 +496,13 @@ class ApacheParser(object): self.aug.load() - def _set_locations(self, ssl_options): + def _set_locations(self, root): """Set default location for directives. Locations are given as file_paths .. todo:: Make sure that files are included """ - root = self._find_config_root() default = self._set_user_config_file(root) temp = os.path.join(self.root, "ports.conf") @@ -511,8 +513,7 @@ class ApacheParser(object): listen = default name = default - return {"root": root, "default": default, "listen": listen, - "name": name, "ssl_options": ssl_options} + return {"default": default, "listen": listen, "name": name} def _find_config_root(self): """Find the Apache Configuration Root file.""" @@ -565,22 +566,3 @@ def get_aug_path(file_path): """ return "/files%s" % file_path - - -def strip_dir(path): - """Returns directory of file path. - - .. todo:: Replace this with Python standard function - - :param str path: path is a file path. not an augeas section or - directive path - - :returns: directory - :rtype: str - - """ - index = path.rfind("/") - if index > 0: - return path[:index+1] - # No directory - return "" diff --git a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py index c882588f6..624c86372 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py @@ -24,5 +24,37 @@ class VirtualHostTest(unittest.TestCase): self.assertFalse(vhost1b == 1234) +class AddrTest(unittest.TestCase): + """Test obj.Addr.""" + def setUp(self): + from letsencrypt_apache.obj import Addr + self.addr = Addr.fromstring("*:443") + + self.addr1 = Addr.fromstring("127.0.0.1") + self.addr2 = Addr.fromstring("127.0.0.1:*") + + def test_wildcard(self): + self.assertFalse(self.addr.is_wildcard()) + self.assertTrue(self.addr1.is_wildcard()) + self.assertTrue(self.addr2.is_wildcard()) + + def test_get_sni_addr(self): + from letsencrypt_apache.obj import Addr + self.assertEqual( + self.addr.get_sni_addr("443"), Addr.fromstring("*:443")) + self.assertEqual( + self.addr.get_sni_addr("225"), Addr.fromstring("*:225")) + self.assertEqual( + self.addr1.get_sni_addr("443"), Addr.fromstring("127.0.0.1")) + + def test_equal(self): + self.assertTrue(self.addr1 == self.addr2) + self.assertFalse(self.addr == self.addr1) + + def test_not_equal(self): + self.assertFalse(self.addr1 != self.addr2) + self.assertTrue(self.addr != self.addr1) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 49575e977..3075a42c3 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -6,8 +6,6 @@ import unittest import augeas import mock -from letsencrypt import errors - from letsencrypt_apache.tests import util @@ -48,10 +46,6 @@ class BasicParserTest(util.ParserTest): self.assertEqual(len(test), 1) self.assertEqual(len(test2), 3) - def test_filter_args_num(self): - # TODO: TEST 2, TEST 1 - pass - def test_add_dir(self): aug_default = "/files" + self.parser.loc["default"] self.parser.add_dir(aug_default, "AddDirective", "test") @@ -90,16 +84,10 @@ class BasicParserTest(util.ParserTest): def test_set_locations(self): with mock.patch("letsencrypt_apache.parser.os.path") as mock_path: - mock_path.isfile.return_value = False - - # pylint: disable=protected-access - self.assertRaises(errors.PluginError, - self.parser._set_locations, self.ssl_options) - mock_path.isfile.side_effect = [True, False, False] # pylint: disable=protected-access - results = self.parser._set_locations(self.ssl_options) + results = self.parser._set_locations("root") self.assertEqual(results["default"], results["listen"]) self.assertEqual(results["default"], results["name"]) @@ -124,7 +112,7 @@ class ParserInitTest(util.ApacheTest): path = os.path.join( self.temp_dir, "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2") - parser = ApacheParser(self.aug, path, None, "dummy_ctl") + parser = ApacheParser(self.aug, path, "dummy_ctl") self.assertEqual(parser.root, self.config_path) @@ -133,7 +121,7 @@ class ParserInitTest(util.ApacheTest): with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): parser = ApacheParser( - self.aug, os.path.relpath(self.config_path), None, "dummy_ctl") + self.aug, os.path.relpath(self.config_path), "dummy_ctl") self.assertEqual(parser.root, self.config_path) @@ -142,8 +130,9 @@ class ParserInitTest(util.ApacheTest): with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): parser = ApacheParser( - self.aug, self.config_path + os.path.sep, None, "dummy_ctl") + self.aug, self.config_path + os.path.sep, "dummy_ctl") self.assertEqual(parser.root, self.config_path) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 8b54b08a4..e0d2f177a 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -54,7 +54,7 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods with mock.patch("letsencrypt_apache.parser.ApacheParser." "update_runtime_variables"): self.parser = ApacheParser( - self.aug, self.config_path, self.ssl_options, "dummy_ctl_path") + self.aug, self.config_path, "dummy_ctl_path") def get_apache_configurator( From d25c7c36e7f2e468f93317ec08fd6719bf9430b8 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Tue, 21 Jul 2015 17:16:46 -0700 Subject: [PATCH 21/52] Convert to servername/serveralias --- .../letsencrypt_apache/configurator.py | 310 ++++++------------ .../letsencrypt_apache/constants.py | 2 +- letsencrypt-apache/letsencrypt_apache/obj.py | 104 +++++- .../letsencrypt_apache/parser.py | 3 +- .../tests/configurator_test.py | 2 +- .../letsencrypt_apache/tests/util.py | 6 +- 6 files changed, 206 insertions(+), 221 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 07e1bb2bc..6ef10baab 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -272,7 +272,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): best_points = 0 for vhost in self.vhosts: - if target_name in vhost.names: + if target_name in vhost.get_names(): points = 2 elif any(addr.get_addr() == target_name for addr in vhost.addrs): points = 1 @@ -312,7 +312,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): all_names = set() for vhost in self.vhosts: - all_names.update(vhost.names) + all_names.update(vhost.get_names()) for addr in vhost.addrs: name = self.get_name_from_ip(addr) if name: @@ -326,7 +326,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param addr: IP Address :type addr: ~.common.Addr - :returns: name + :returns: name or empty string if name cannot be determined :rtype: str """ @@ -347,17 +347,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :type host: :class:`~letsencrypt_apache.obj.VirtualHost` """ - name_match = self.aug.match(("%s//*[self::directive=~regexp('%s')] | " - "%s//*[self::directive=~regexp('%s')]" % - (host.path, - parser.case_i("ServerName"), - host.path, - parser.case_i("ServerAlias")))) + # Take the final ServerName as each overrides the previous + servername_match = self.parser.find_dir( + "ServerName", None, host.path, exclude=False) + serveralias_match = self.parser.find_dir( + "ServerAlias", None, host.path, exclude=False) - for name in name_match: - args = self.aug.match(name + "/*") - for arg in args: - host.add_name(self.parser.get_arg(arg)) + aliases = [] + for alias in serveralias_match: + aliases.append(self.parser.get_arg(alias)) + + if servername_match: + # Get last ServerName as each overwrites the previous + host.add_names(self.parser.get_arg(servername_match[-1]), aliases) + else: + host.add_names(None, aliases) def _create_vhost(self, path): """Used by get_virtual_hosts to create vhost objects @@ -700,28 +704,25 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if "rewrite_module" not in self.parser.modules: self.enable_mod("rewrite") - general_v = self._general_vhost(ssl_vhost) + general_v = self._get_http_vhost(ssl_vhost) if general_v is None: # Add virtual_server with redirect - logger.debug( - "Did not find http version of ssl virtual host... creating") - return self._create_redirect_vhost(ssl_vhost) + logger.debug("Did not find http version of ssl virtual host " + "attempting to create") + redirect_addrs = self._get_redirect_addrs(ssl_vhost) + for vhost in self.vhosts: + if vhost.enabled and vhost.conflicts(redirect_addrs): + raise errors.PluginError( + "Unable to find corresponding HTTP vhost; " + "Unable to create one as intended addresses conflict; " + "Current configuration does not support automated " + "redirection") + self.create_redirect_vhost(redirect_addrs) else: # Check if redirection already exists - exists, code = self._existing_redirect(general_v) - if exists: - if code == 0: - logger.debug("Redirect already added") - logger.info( - "Configuration is already redirecting traffic to HTTPS") - return - else: - logger.info("Unknown redirect exists for this vhost") - raise errors.PluginError( - "Unknown redirect already exists " - "in {}".format(general_v.filep)) + self._verify_no_redirects(vhost) # Add directives to server - self.parser.add_dir(general_v.path, "RewriteEngine", "On") + self.parser.add_dir(general_v.path, "RewriteEngine", "on") self.parser.add_dir(general_v.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % @@ -731,24 +732,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.info("Redirecting vhost in %s to ssl vhost in %s", general_v.filep, ssl_vhost.filep) - def _existing_redirect(self, vhost): + def _verify_no_redirects(self, vhost): """Checks to see if existing redirect is in place. Checks to see if virtualhost already contains a rewrite or redirect returns boolean, integer - The boolean indicates whether the redirection exists... - The integer has the following code: - 0 - Existing letsencrypt https rewrite rule is appropriate and in place - 1 - Virtual host contains a Redirect directive - 2 - Virtual host contains an unknown RewriteRule - - -1 is also returned in case of no redirection/rewrite directives :param vhost: vhost to check :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` - :returns: Success, code value... see documentation - :rtype: bool, int + :raises errors.PluginError: When another redirection exists """ rewrite_path = self.parser.find_dir("RewriteRule", None, vhost.path) @@ -756,20 +749,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if redirect_path: # "Existing Redirect directive for virtualhost" - return True, 1 - if not rewrite_path: + raise errors.PluginError("Existing Redirect present on HTTP vhost.") + if rewrite_path: # "No existing redirection for virtualhost" - return False, -1 - if len(rewrite_path) == len(constants.REWRITE_HTTPS_ARGS): - for idx, match in enumerate(rewrite_path): - if (self.aug.get(match) != - constants.REWRITE_HTTPS_ARGS[idx]): - # Not a letsencrypt https rewrite - return True, 2 - # Existing letsencrypt https rewrite rule is in place - return True, 0 - # Rewrite path exists but is not a letsencrypt https rule - return True, 2 + if len(rewrite_path) != len(constants.REWRITE_HTTPS_ARGS): + raise errors.PluginError("Unknown Existing RewriteRule") + for match, arg in itertools.izip( + rewrite_path, constants.REWRITE_HTTPS_ARGS): + if self.aug.get(match) != arg: + raise errors.PluginError("Unknown Existing RewriteRule") + raise errors.PluginError( + "Let's Encrypt has already enabled redirection") + def _create_redirect_vhost(self, ssl_vhost): """Creates an http_vhost specifically to redirect for the ssl_vhost. @@ -782,62 +773,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: tuple """ - # Consider changing this to a dictionary check - # Make sure adding the vhost will be safe - conflict, host_or_addrs = self._conflicting_host(ssl_vhost) - if conflict: - raise errors.PluginError( - "Unable to create a redirection vhost - {}".format( - host_or_addrs)) + text = self._get_redirect_config_str(ssl_vhost) - redirect_addrs = host_or_addrs - - # get servernames and serveraliases - serveralias = "" - servername = "" - size_n = len(ssl_vhost.names) - if size_n > 0: - servername = "ServerName " + ssl_vhost.names[0] - if size_n > 1: - serveralias = " ".join(ssl_vhost.names[1:size_n]) - serveralias = "ServerAlias " + serveralias - redirect_file = ("\n" - "%s \n" - "%s \n" - "ServerSignature Off\n" - "\n" - "RewriteEngine On\n" - "RewriteRule %s\n" - "\n" - "ErrorLog /var/log/apache2/redirect.error.log\n" - "LogLevel warn\n" - "\n" - % (servername, serveralias, - " ".join(constants.REWRITE_HTTPS_ARGS))) - - # Write out the file - # This is the default name - redirect_filename = "le-redirect.conf" - - # See if a more appropriate name can be applied - if len(ssl_vhost.names) > 0: - # Sanity check... - # make sure servername doesn't exceed filename length restriction - if ssl_vhost.names[0] < (255-23): - redirect_filename = "le-redirect-%s.conf" % ssl_vhost.names[0] - - redirect_filepath = os.path.join( - self.parser.root, "sites-available", redirect_filename) - - # Register the new file that will be created - # Note: always register the creation before writing to ensure file will - # be removed in case of unexpected program exit - self.reverter.register_file_creation(False, redirect_filepath) - - # Write out file - with open(redirect_filepath, "w") as redirect_fd: - redirect_fd.write(redirect_file) - logger.info("Created redirect file: %s", redirect_filename) + self._write_out_redirect(ssl_vhost, text) self.aug.load() # Make a new vhost data structure and add it to the lists @@ -849,85 +787,77 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ssl vhost %s\n" % (new_vhost.filep, ssl_vhost.filep)) - def _conflicting_host(self, ssl_vhost): - """Checks for conflicting HTTP vhost for ssl_vhost. + def _get_redirect_config_str(self, ssl_vhost): + # get servernames and serveraliases + serveralias = "" + servername = "" - Checks for a conflicting host, such that a new port 80 host could not - be created without ruining the apache config - Used with redirection + if ssl_vhost.name is not None: + servername = "ServerName " + ssl_vhost.name + if ssl_vhost.aliases: + serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - returns: conflict, host_or_addrs - boolean - if conflict: returns conflicting vhost - if not conflict: returns space separated list of new host addrs + return ("\n" + "%s \n" + "%s \n" + "ServerSignature Off\n" + "\n" + "RewriteEngine On\n" + "RewriteRule %s\n" + "\n" + "ErrorLog /var/log/apache2/redirect.error.log\n" + "LogLevel warn\n" + "\n" + % (" ".join(self._get_redirect_addrs(ssl_vhost)), + servername, serveralias, + " ".join(constants.REWRITE_HTTPS_ARGS))) - :param ssl_vhost: SSL Vhost to check for possible port 80 redirection - :type ssl_vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + def _write_out_redirect(self, ssl_vhost, text): + # This is the default name + redirect_filename = "le-redirect.conf" - :returns: TODO - :rtype: TODO + # See if a more appropriate name can be applied + if ssl_vhost.name is not None: + # make sure servername doesn't exceed filename length restriction + if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): + redirect_filename = "le-redirect-%s.conf" % ssl_vhost.servername - """ - # Consider changing this to a dictionary check - redirect_addrs = "" - for ssl_a in ssl_vhost.addrs: - # Add space on each new addr, combine "VirtualHost"+redirect_addrs - redirect_addrs = redirect_addrs + " " - ssl_a_vhttp = ssl_a.get_addr_obj("80") - # Search for a conflicting host... - for vhost in self.vhosts: - if vhost.enabled: - if (ssl_a_vhttp in vhost.addrs or - ssl_a.get_addr_obj("") in vhost.addrs or - ssl_a.get_addr_obj("*") in vhost.addrs): - # We have found a conflicting host... just return - return True, vhost + redirect_filepath = os.path.join( + self.parser.root, "sites-available", redirect_filename) - redirect_addrs = redirect_addrs + ssl_a_vhttp + # Register the new file that will be created + # Note: always register the creation before writing to ensure file will + # be removed in case of unexpected program exit + self.reverter.register_file_creation(False, redirect_filepath) - return False, redirect_addrs + # Write out file + with open(redirect_filepath, "w") as redirect_file: + redirect_file.write(text) + logger.info("Created redirect file: %s", redirect_filename) - def _general_vhost(self, ssl_vhost): - """Find appropriate HTTP vhost for ssl_vhost. + return redirect_filepath - Function needs to be thoroughly tested and perhaps improved - Will not do well with malformed configurations - Consider changing this into a dict check + def _get_http_vhost(self, ssl_vhost): + """Find appropriate HTTP vhost for ssl_vhost.""" + # First candidate vhosts filter + candidate_http_vhs = [ + vhost for vhost in self.vhosts if not vhost.ssl + ] - :param ssl_vhost: ssl vhost to check - :type ssl_vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + # Second filter - check addresses + for http_vh in candidate_http_vhs: + if http_vh.same_server(ssl_vhost): + return http_vh - :returns: HTTP vhost or None if unsuccessful - :rtype: :class:`~letsencrypt_apache.obj.VirtualHost` or ``None`` - - """ - # _default_:443 check - # Instead... should look for vhost of the form *:80 - # Should we prompt the user? - ssl_addrs = ssl_vhost.addrs - if ssl_addrs == obj.Addr.fromstring("_default_:443"): - ssl_addrs = [obj.Addr.fromstring("*:443")] - - for vhost in self.vhosts: - found = 0 - # Not the same vhost, and same number of addresses - if vhost != ssl_vhost and len(vhost.addrs) == len(ssl_vhost.addrs): - # Find each address in ssl_host in test_host - for ssl_a in ssl_addrs: - for test_a in vhost.addrs: - if test_a.get_addr() == ssl_a.get_addr(): - # Check if found... - if (test_a.get_port() == "80" or - test_a.get_port() == "" or - test_a.get_port() == "*"): - found += 1 - break - # Check to make sure all addresses were found - # and names are equal - if (found == len(ssl_vhost.addrs) and - vhost.names == ssl_vhost.names): - return vhost return None + def _get_redirect_addrs(self, ssl_vhost): + redirects = set() + for addr in vhost.addrs: + redirects.add(addr.get_addr_obj("80")) + + return redirects + def get_all_certs_keys(self): """Find all existing keys, certs from configuration. @@ -1075,38 +1005,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.reverter.register_file_creation(False, enabled_path) os.symlink(os.path.join(mods_available, filename), enabled_path) - def mod_loaded(self, module): - """Checks to see if mod_ssl is loaded - - Uses ``apache_ctl`` to get loaded module list. This also effectively - serves as a config_test. - - :returns: If module is loaded. - :rtype: bool - - """ - try: - proc = subprocess.Popen( - [self.conf("ctl"), "-M"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - - except (OSError, ValueError): - logger.error( - "Error accessing %s for loaded modules!", self.conf("ctl")) - raise errors.MisconfigurationError("Error accessing loaded modules") - # Small errors that do not impede - if proc.returncode != 0: - logger.warn("Error in checking loaded module list: %s", stderr) - raise errors.MisconfigurationError( - "Apache is unable to check whether or not the module is " - "loaded because Apache is misconfigured.") - - if module in stdout: - return True - return False - def restart(self): """Restarts apache server. diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index cb75276b2..7e7e127f5 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -20,5 +20,5 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename( distribution.""" REWRITE_HTTPS_ARGS = [ - "^.*$", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] """Apache rewrite rule arguments used for redirections to https vhost""" diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index a2df429c3..6895e9039 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -1,6 +1,9 @@ """Module contains classes used by the Apache Configurator.""" +import re + from letsencrypt.plugins import common + class Addr(common.Addr): """Represents an Apache address.""" def __eq__(self, other): @@ -45,39 +48,57 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods :ivar str path: Augeas path to virtual host :ivar set addrs: Virtual Host addresses (:class:`set` of :class:`common.Addr`) - :ivar set names: Server names/aliases of vhost + :ivar str name: ServerName of VHost + :ivar list aliases: Server aliases of vhost (:class:`list` of :class:`str`) :ivar bool ssl: SSLEngine on in vhost :ivar bool enabled: Virtual host is enabled + .. todo:: Handle ServerNames appropriately... + """ - def __init__(self, filep, path, addrs, ssl, enabled, names=None): + # ?: is used for not returning enclosed characters + strip_name = re.compile(r"^(?:.+://)?([^ :$]*)") + + def __init__(self, filep, path, addrs, ssl, enabled, name=None, aliases=[]): # pylint: disable=too-many-arguments """Initialize a VH.""" self.filep = filep self.path = path self.addrs = addrs - self.names = set() if names is None else set(names) + self.name = name + self.aliases = aliases self.ssl = ssl self.enabled = enabled - def add_name(self, name): + def add_names(self, servername, serveralias): """Add name to vhost.""" - self.names.add(name) + self.name = servername + self.aliases = serveralias + + def get_names(self): + all_names = set(self.aliases) + # Strip out any scheme:// and field from servername + if self.name is not None: + all_names.add(VirtualHost.strip_name.findall(self.name)[0]) + + return all_names def __str__(self): return ( "File: {filename}\n" "Vhost path: {vhpath}\n" "Addresses: {addrs}\n" - "Names: {names}\n" + "Name: {name}\n" + "Aliases: {aliases}\n" "TLS Enabled: {tls}\n" "Site Enabled: {active}".format( filename=self.filep, vhpath=self.path, addrs=", ".join(str(addr) for addr in self.addrs), - names=", ".join(name for name in self.names), + name=self.name if self.name is not None else "", + aliases=", ".join(name for name in self.aliases), tls="Yes" if self.ssl else "No", active="Yes" if self.enabled else "No")) @@ -85,7 +106,74 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and self.addrs == other.addrs and - self.names == other.names and + self.get_names() == other.get_names() and self.ssl == other.ssl and self.enabled == other.enabled) return False + + def __ne__(self, other): + return not self.__eq__(other) + + def conflicts(self, addrs): + """See if vhost conflicts with any of the addrs. + + This determines whether or not these addresses would/could overwrite + the vhost addresses. + + :param addrs: Iterable Addresses + :type addrs: Iterable :class:~obj.Addr + + :returns: If addresses conflict with vhost + :rtype: bool + + """ + # TODO: Handle domain name addrs... + for addr in addrs: + if (addr in self.addrs or addr.get_addr_obj("") in self.addrs or + addr.get_addr_obj("*") in self.addrs): + return True + + return False + + def same_server(self, vhost): + """Determines if the vhost is the same 'server'. + + Used in redirection - indicates whether or not the two virtual hosts + serve on the exact same IP combinations, but different ports. + + .. todo:: Handle _default_ + + """ + + if vhost.get_names() != self.get_names(): + return False + + # If equal and set is not empty... assume same server + if self.name is not None: + return True + + # Both sets of names are empty. + + # Make conservative educated guess... this is very restrictive + # Consider adding more safety checks. + if len(vhost.addrs) != len(self.addrs): + return False + + # already_found acts to keep everything very conservative. + # Don't allow multiple ip:ports in same set. + already_found = set() + + for addr in vhost.addrs: + for local_addr in self.addrs: + if (local_addr.get_addr() == addr.get_addr() and + local_addr != addr and + local_addr.get_addr() not in already_found): + + # This intends to make sure we aren't double counting... + # e.g. 127.0.0.1:* + already_found.add(local_addr.get_addr()) + break + else: + return False + + return True \ No newline at end of file diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 8feb54fa8..ada2db027 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -230,8 +230,7 @@ class ApacheParser(object): Directives should be in the form of a case insensitive regex currently .. todo:: arg should probably be a list - .. todo:: Check //* notation for including directories not intended - to be included. + .. todo:: Check //* notation for including directories Note: Augeas is inherently case sensitive while Apache is case insensitive. Augeas 1.0 allows case insensitive regexes like diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index e68b21945..101a53a97 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -185,7 +185,7 @@ class TwoVhost80Test(util.ApacheTest): "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") self.assertEqual(len(ssl_vhost.addrs), 1) self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) - self.assertEqual(ssl_vhost.names, set(["encryption-example.demo"])) + self.assertEqual(ssl_vhost.name, "encryption-example.demo") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index e0d2f177a..fcec3a6a5 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -105,7 +105,7 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "encryption-example.conf"), os.path.join(aug_pre, "encryption-example.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), - False, True, set(["encryption-example.demo"])), + False, True, "encryption-example.demo"), obj.VirtualHost( os.path.join(prefix, "default-ssl.conf"), os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), @@ -114,12 +114,12 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "000-default.conf"), os.path.join(aug_pre, "000-default.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - set(["ip-172-30-0-17"])), + "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "letsencrypt.conf"), os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - set(["letsencrypt.demo"])), + "letsencrypt.demo"), ] return vh_truth From 8b96264576741eee5b66fd5b012edd529ffa545c Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 22 Jul 2015 02:05:01 -0700 Subject: [PATCH 22/52] improved redirect, testcases --- .../letsencrypt_apache/configurator.py | 30 ++++++---- .../letsencrypt_apache/display_ops.py | 6 +- letsencrypt-apache/letsencrypt_apache/obj.py | 1 + .../letsencrypt_apache/parser.py | 6 +- .../tests/configurator_test.py | 56 +++++++++++++++++++ .../letsencrypt_apache/tests/dvsni_test.py | 5 +- .../apache2/mods-available/rewrite.load | 1 + 7 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/rewrite.load diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 6ef10baab..a27a5cb95 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1,5 +1,6 @@ """Apache Configuration based off of Augeas Configurator.""" # pylint: disable=too-many-lines +import itertools import logging import os import re @@ -60,6 +61,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): prone. .. todo:: Write a server protocol finder. Listen or Protocol . This can verify partial setups are correct + .. todo:: Add directives to sites-enabled... not sites-available. + sites-available doesn't allow immediate find_dir search even with save() + and load() :ivar config: Configuration. :type config: :class:`~letsencrypt.interfaces.IConfig` @@ -672,11 +676,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): try: return self._enhance_func[enhancement]( self.choose_vhost(domain), options) - except ValueError: + except KeyError: raise errors.PluginError( "Unsupported enhancement: {}".format(enhancement)) except errors.PluginError: logger.warn("Failed %s for %s", enhancement, domain) + raise def _enable_redirect(self, ssl_vhost, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -703,9 +708,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if "rewrite_module" not in self.parser.modules: self.enable_mod("rewrite") + general_vh = self._get_http_vhost(ssl_vhost) - general_v = self._get_http_vhost(ssl_vhost) - if general_v is None: + if general_vh is None: # Add virtual_server with redirect logger.debug("Did not find http version of ssl virtual host " "attempting to create") @@ -717,20 +722,23 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "Unable to create one as intended addresses conflict; " "Current configuration does not support automated " "redirection") - self.create_redirect_vhost(redirect_addrs) + self._create_redirect_vhost(redirect_addrs) else: # Check if redirection already exists - self._verify_no_redirects(vhost) + self._verify_no_redirects(general_vh) + # Add directives to server - self.parser.add_dir(general_v.path, "RewriteEngine", "on") - self.parser.add_dir(general_v.path, "RewriteRule", + # Note: These are not immediately searchable in sites-enabled + # even with save() and load() + self.parser.add_dir(general_vh.path, "RewriteEngine", "on") + self.parser.add_dir(general_vh.path, "RewriteRule", constants.REWRITE_HTTPS_ARGS) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % - (general_v.filep, ssl_vhost.filep)) + (general_vh.filep, ssl_vhost.filep)) self.save() logger.info("Redirecting vhost in %s to ssl vhost in %s", - general_v.filep, ssl_vhost.filep) + general_vh.filep, ssl_vhost.filep) def _verify_no_redirects(self, vhost): """Checks to see if existing redirect is in place. @@ -775,7 +783,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ text = self._get_redirect_config_str(ssl_vhost) - self._write_out_redirect(ssl_vhost, text) + redirect_filepath = self._write_out_redirect(ssl_vhost, text) self.aug.load() # Make a new vhost data structure and add it to the lists @@ -853,7 +861,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _get_redirect_addrs(self, ssl_vhost): redirects = set() - for addr in vhost.addrs: + for addr in ssl_vhost.addrs: redirects.add(addr.get_addr_obj("80")) return redirects diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index 352845376..45c55f49a 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -60,9 +60,9 @@ def _vhost_menu(domain, vhosts): choices = [] for vhost in vhosts: - if len(vhost.names) == 1: - disp_name = next(iter(vhost.names)) - elif len(vhost.names) == 0: + if len(vhost.get_names()) == 1: + disp_name = next(iter(vhost.get_names())) + elif len(vhost.get_names()) == 0: disp_name = "" else: disp_name = "Multiple Names" diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index 6895e9039..d453f34e3 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -78,6 +78,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods self.aliases = serveralias def get_names(self): + """Return a set of all names.""" all_names = set(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index ada2db027..17f6e8f28 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -287,9 +287,11 @@ class ApacheParser(object): dir_ = self.aug.get(match).lower() if dir_ == "include" or dir_ == "includeoptional": # start[6:] to strip off /files + #print self._get_include_path(self.get_arg(match +"/arg")), directive, arg ordered_matches.extend(self.find_dir( - directive, arg, self._get_include_path( - self.get_arg(match + "/arg")))) + directive, arg, + self._get_include_path(self.get_arg(match + "/arg")), + exclude)) # This additionally allows Include if dir_ == directive.lower(): ordered_matches.extend(self.aug.match(match + arg_suffix)) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 101a53a97..68b0433f1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -254,6 +254,62 @@ class TwoVhost80Test(util.ApacheTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.PluginError, self.config.get_version) + # TEST ENHANCEMENTS + def test_enhance_unknown_enhancement(self): + self.assertRaises( + errors.PluginError, + self.config.enhance, "letsencrypt.demo", "unknown_enhancement") + + @mock.patch("letsencrypt_apache.parser." + "ApacheParser.update_runtime_variables") + def test_redirect_well_formed_http(self, unused): + # This will create an ssl vhost for letsencrypt.demo + self.config.enhance("letsencrypt.demo", "redirect") + + # These are not immediately available in find_dir even with save() and + # load(). They must be found in sites-available + rw_engine = self.config.parser.find_dir( + "RewriteEngine", "on", self.vh_truth[3].path) + rw_rule = self.config.parser.find_dir( + "RewriteRule", None, self.vh_truth[3].path) + + self.assertEqual(len(rw_engine), 1) + # three args to rw_rule + self.assertEqual(len(rw_rule), 3) + + self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path)) + self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path)) + + self.assertTrue("rewrite_module" in self.config.parser.modules) + + def test_redirect_twice(self): + # Skip the enable mod + self.config.parser.modules.add("rewrite_module") + self.config.enhance("encryption-example.demo", "redirect") + self.assertRaises( + errors.PluginError, + self.config.enhance, "encryption-example.demo", "redirect") + + def test_unknown_rewrite(self): + # Skip the enable mod + self.config.parser.modules.add("rewrite_module") + self.config.parser.add_dir( + self.vh_truth[3].path, "RewriteRule", ["Unknown"]) + self.config.save() + self.assertRaises( + errors.PluginError, + self.config.enhance, "letsencrypt.demo", "redirect") + + def test_unknown_redirect(self): + # Skip the enable mod + self.config.parser.modules.add("rewrite_module") + self.config.parser.add_dir( + self.vh_truth[3].path, "Redirect", ["Unknown"]) + self.config.save() + self.assertRaises( + errors.PluginError, + self.config.enhance, "letsencrypt.demo", "redirect") + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py index b0aec4f8a..29b600ec3 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py @@ -109,9 +109,10 @@ class DvsniPerformTest(util.ApacheTest): self.assertEqual(len(vhs), 2) for vhost in vhs: self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")])) + names = vhost.get_names() self.assertTrue( - vhost.names == set([self.achalls[0].nonce_domain]) or - vhost.names == set([self.achalls[1].nonce_domain])) + names == set([self.achalls[0].nonce_domain]) or + names == set([self.achalls[1].nonce_domain])) if __name__ == "__main__": diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/rewrite.load b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/rewrite.load new file mode 100644 index 000000000..b32f16264 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/rewrite.load @@ -0,0 +1 @@ +LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so From c92714918578fbad648525f0759542b711b120d9 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 23 Jul 2015 01:34:51 -0700 Subject: [PATCH 23/52] Unittests and revisions --- .../letsencrypt_apache/configurator.py | 39 +++++--- .../letsencrypt_apache/dvsni.py | 1 - letsencrypt-apache/letsencrypt_apache/obj.py | 63 +++++++++--- .../letsencrypt_apache/parser.py | 24 +++-- .../tests/complex_parsing_test.py | 10 ++ .../tests/configurator_test.py | 49 +++++++++- .../letsencrypt_apache/tests/obj_test.py | 98 ++++++++++++++++--- .../letsencrypt_apache/tests/parser_test.py | 76 +++++++++++++- .../letsencrypt_apache/tests/util.py | 4 +- 9 files changed, 316 insertions(+), 48 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index a27a5cb95..915c129cf 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -117,7 +117,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Verify that all directories and files exist with proper permissions if os.geteuid() == 0: - self.verify_setup() + self.verify_setup() # pragma: no cover # Add name_server association dict self.assoc = dict() @@ -147,7 +147,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: - self.version = self.get_version() + self.version = self.get_version() # pragma: no cover # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() @@ -265,6 +265,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _find_best_vhost(self, target_name): """Finds the best vhost for a target_name. + This does not upgrade a vhost to HTTPS... it only finds the most + appropriate vhost for the given target_name. + :returns: VHost or None """ @@ -281,6 +284,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): elif any(addr.get_addr() == target_name for addr in vhost.addrs): points = 1 else: + # No points given if names can't be found. continue if vhost.ssl: @@ -290,8 +294,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): best_points = points best_candidate = vhost + # No winners here... is there only one reasonable vhost? + if best_candidate is None: + # reasonable == Not all _default_ addrs + reasonable_vhosts = self._non_default_vhosts() + if len(reasonable_vhosts) == 1: + best_candidate = reasonable_vhosts[0] + return best_candidate + def _non_default_vhosts(self): + """Return all non _default_ only vhosts.""" + return [vh for vh in self.vhosts if not all( + addr.get_addr() == "_default_" for addr in vh.addrs + )] + def create_dn_server_assoc(self, domain, vhost): """Create an association between a domain name and virtual host. @@ -887,17 +904,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): key_path = self.parser.find_dir( "SSLCertificateKeyFile", None, vhost.path) - # Can be removed once find directive can return ordered results - if len(cert_path) != 1 or len(key_path) != 1: - logger.error("Too many cert or key directives in vhost %s", - vhost.filep) - errors.MisconfigurationError( - "Too many cert/key directives in vhost") - - cert = os.path.abspath(self.parser.get_arg(cert_path[0])) - key = os.path.abspath(self.parser.get_arg(key_path[0])) - c_k.add((cert, key, get_file_path(cert_path[0]))) - + if cert_path and key_path: + cert = os.path.abspath(self.parser.get_arg(cert_path[-1])) + key = os.path.abspath(self.parser.get_arg(key_path[-1])) + c_k.add((cert, key, get_file_path(cert_path[-1]))) + else: + logger.warning( + "Invalid VirtualHost configuration - %s", vhost.filep) return c_k def is_site_enabled(self, avail_fp): diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index 616a1a1ef..113991c53 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -117,7 +117,6 @@ class ApacheDvsni(common.Dvsni): default_addr = obj.Addr(("*", self.configurator.config.dvsni_port)) for addr in vhost.addrs: - # I don't think there can be two _default_ namebasedvhosts if "_default_" == addr.get_addr(): dvsni_addrs.add(default_addr) else: diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index d453f34e3..f192ad6da 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -21,6 +21,44 @@ class Addr(common.Addr): def __ne__(self, other): return not self.__eq__(other) + def _addr_less_specific(self, addr): + """Returns if addr.get_addr() is more specific than self.get_addr().""" + return addr._rank_specific_addr() > self._rank_specific_addr() + + def _rank_specific_addr(self): + """Returns numerical rank for get_addr()""" + if self.get_addr() == "_default_": + return 0 + elif self.get_addr() == "*": + return 1 + else: + return 2 + + def conflicts(self, addr): + """Returns if address could conflict with correct function of self. + + Could addr take away service provided by self within Apache? + + .. note::IP Address is more important than wildcard. + Connection from 127.0.0.1:80 with choices of *:80 and 127.0.0.1:* + chooses 127.0.0.1:* + + .. todo:: Handle domain name addrs... + + Examples: + 127.0.0.1:*.conflicts(127.0.0.1:443) - True + 127.0.0.1:443.conflicts(127.0.0.1:*) - False + *:443.conflicts(*:80) - False + _default_:443.conflicts(*:443) - True + + """ + if self._addr_less_specific(addr): + return True + elif self.get_addr() == addr.get_addr(): + if self.is_wildcard() or self.get_port() == addr.get_port(): + return True + return False + def is_wildcard(self): """Returns if address has a wildcard port.""" return self.tup[1] == "*" or not self.tup[1] @@ -55,7 +93,9 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods :ivar bool ssl: SSLEngine on in vhost :ivar bool enabled: Virtual host is enabled - .. todo:: Handle ServerNames appropriately... + https://httpd.apache.org/docs/2.4/vhosts/details.html + .. todo:: Any vhost that includes the magic _default_ wildcard is given the + same ServerName as the main server. """ # ?: is used for not returning enclosed characters @@ -124,16 +164,14 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods :param addrs: Iterable Addresses :type addrs: Iterable :class:~obj.Addr - :returns: If addresses conflict with vhost + :returns: If addresses conflicts with vhost :rtype: bool """ - # TODO: Handle domain name addrs... - for addr in addrs: - if (addr in self.addrs or addr.get_addr_obj("") in self.addrs or - addr.get_addr_obj("*") in self.addrs): - return True - + for pot_addr in addrs: + for addr in self.addrs: + if addr.conflicts(pot_addr): + return True return False def same_server(self, vhost): @@ -150,7 +188,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False # If equal and set is not empty... assume same server - if self.name is not None: + if self.name is not None or self.aliases: return True # Both sets of names are empty. @@ -167,11 +205,12 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for addr in vhost.addrs: for local_addr in self.addrs: if (local_addr.get_addr() == addr.get_addr() and - local_addr != addr and - local_addr.get_addr() not in already_found): + local_addr != addr and + local_addr.get_addr() not in already_found): # This intends to make sure we aren't double counting... - # e.g. 127.0.0.1:* + # e.g. 127.0.0.1:* - We require same number of addrs + # currently already_found.add(local_addr.get_addr()) break else: diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 17f6e8f28..00af5d114 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -19,12 +19,18 @@ class ApacheParser(object): :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. + :ivar str root: Server root + :ivar set modules: All module names that are currently enabled. + :ivar dict loc: Location to place directives, root - configuration origin, + default - user config file, name - NameVirtualHost, """ arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") fnmatch_chars = set(["*", "?", "\\", "[", "]"]) def __init__(self, aug, root, ctl): + # Note: Order is important here. + # This uses the binary, so it can be done first. # https://httpd.apache.org/docs/2.4/mod/core.html#define # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine @@ -47,7 +53,8 @@ class ApacheParser(object): self.modules = set() self._init_modules() - self.loc.update(self._set_locations(self.loc["root"])) + # Set up rest of locations + self.loc.update(self._set_locations()) # Must also attempt to parse sites-available or equivalent # Sites-available is not included naturally in configuration @@ -89,7 +96,10 @@ class ApacheParser(object): variables = dict() matches = re.compile(r"Define: ([^ \n]*)").findall(stdout) - matches.remove("DUMP_RUN_CFG") + try: + matches.remove("DUMP_RUN_CFG") + except ValueError: + raise errors.PluginError("Unable to parse runtime variables") for match in matches: if match.count("=") > 1: @@ -183,7 +193,7 @@ class ApacheParser(object): self.aug.set(nvh_path + "/arg", args[0]) else: for i, arg in enumerate(args): - self.aug.set("%s/arg[%d]" % (nvh_path, i), arg) + self.aug.set("%s/arg[%d]" % (nvh_path, i+1), arg) def _get_ifmod(self, aug_conf_path, mod): @@ -497,14 +507,14 @@ class ApacheParser(object): self.aug.load() - def _set_locations(self, root): + def _set_locations(self): """Set default location for directives. Locations are given as file_paths .. todo:: Make sure that files are included """ - default = self._set_user_config_file(root) + default = self._set_user_config_file() temp = os.path.join(self.root, "ports.conf") if os.path.isfile(temp): @@ -526,7 +536,7 @@ class ApacheParser(object): raise errors.NoInstallationError("Could not find configuration root") - def _set_user_config_file(self, root): + def _set_user_config_file(self): """Set the appropriate user configuration file .. todo:: This will have to be updated for other distros versions @@ -538,7 +548,7 @@ class ApacheParser(object): # in hierarchy via direct include # httpd.conf was very common as a user file in Apache 2.2 if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and - self.find_dir("Include", "httpd.conf", root)): + self.find_dir("Include", "httpd.conf", self.loc["root"])): return os.path.join(self.root, "httpd.conf") else: return os.path.join(self.root, "apache2.conf") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py index 7281061e4..5e4478f67 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py @@ -3,6 +3,8 @@ import os import shutil import unittest +from letsencrypt import errors + from letsencrypt_apache.tests import util @@ -46,6 +48,14 @@ class ComplexParserTest(util.ParserTest): self.assertEqual(len(matches), 1) self.assertEqual(self.parser.get_arg(matches[0]), "1234") + def test_invalid_variable_parsing(self): + del self.parser.variables["tls_port"] + + matches = self.parser.find_dir("TestVariablePort") + self.assertRaises( + errors.PluginError, self.parser.get_arg, matches[0]) + + def test_basic_ifdefine(self): self.assertEqual(len(self.parser.find_dir("VAR_DIRECTIVE")), 2) self.assertEqual(len(self.parser.find_dir("INVALID_VAR_DIRECTIVE")), 0) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 68b0433f1..c2e7d2916 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -36,6 +36,11 @@ class TwoVhost80Test(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + def test_add_parser_arguments(self): + from letsencrypt_apache.configurator import ApacheConfigurator + # Weak test.. + ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_get_all_names(self): names = self.config.get_all_names() self.assertEqual(names, set( @@ -58,10 +63,44 @@ class TwoVhost80Test(util.ApacheTest): found += 1 break else: - raise Exception("Missed: %s" % vhost) + raise Exception("Missed: %s" % vhost) # pragma: no cover self.assertEqual(found, 4) + @mock.patch("letsencrypt_apache.display_ops.select_vhost") + def test_choose_vhost_none_avail(self, mock_select): + mock_select.return_value = None + self.assertRaises( + errors.PluginError, self.config.choose_vhost, "none.com") + + @mock.patch("letsencrypt_apache.display_ops.select_vhost") + def test_choose_vhost_select_vhost(self, mock_select): + mock_select.return_value = self.vh_truth[3] + self.assertEqual( + self.vh_truth[3], self.config.choose_vhost("none.com")) + + def test_find_best_vhost(self): + self.assertEqual( + self.vh_truth[3], self.config._find_best_vhost("letsencrypt.demo")) + self.assertEqual( + self.vh_truth[0], + self.config._find_best_vhost("encryption-example.demo")) + self.assertTrue( + self.config._find_best_vhost("does-not-exist.com") is None) + + def test_find_best_vhost_default(self): + # Assume only the two default vhosts. + self.config.vhosts = [vh for vh in self.config.vhosts + if vh.name not in + ["letsencrypt.demo", "encryption-example.demo"]] + + self.assertEqual( + self.config._find_best_vhost("example.demo"), self.vh_truth[2]) + + def test_non_default_vhosts(self): + # pylint: disable=protected-access + self.assertEqual(len(self.config._non_default_vhosts()), 3) + def test_is_site_enabled(self): """Test if site is enabled. @@ -137,6 +176,14 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(configurator.get_file_path(loc_chain[0]), self.vh_truth[1].filep) + # One more time for chain directive setting + self.config.deploy_cert( + "random.demo", + "two/cert.pem", "two/key.pem", "two/cert_chain.pem") + self.assertTrue(self.config.parser.find_dir( + "SSLCertificateChainFile", "two/cert_chain.pem", + self.vh_truth[1].path)) + def test_deploy_cert_invalid_vhost(self): self.config.parser.modules.add("ssl_module") mock_find = mock.MagicMock() diff --git a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py index 624c86372..13eddaddf 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py @@ -1,27 +1,73 @@ """Tests for letsencrypt_apache.obj.""" import unittest -from letsencrypt.plugins import common - class VirtualHostTest(unittest.TestCase): """Test the VirtualHost class.""" def setUp(self): + from letsencrypt_apache.obj import Addr from letsencrypt_apache.obj import VirtualHost + + self.addr1 = Addr.fromstring("127.0.0.1") + self.addr2 = Addr.fromstring("127.0.0.1:443") + self.addr_default = Addr.fromstring("_default_:443") + self.vhost1 = VirtualHost( - "filep", "vh_path", - set([common.Addr.fromstring("localhost")]), False, False) + "filep", "vh_path", set([self.addr1]), False, False, "localhost") + + self.vhost1b = VirtualHost( + "filep", "vh_path", set([self.addr1]), False, False, "localhost") + + self.vhost2 = VirtualHost( + "fp", "vhp", set([self.addr2]), False, False, "localhost") def test_eq(self): - from letsencrypt_apache.obj import VirtualHost - vhost1b = VirtualHost( - "filep", "vh_path", - set([common.Addr.fromstring("localhost")]), False, False) + self.assertTrue(self.vhost1b == self.vhost1) + self.assertFalse(self.vhost1 == self.vhost2) + self.assertEqual(str(self.vhost1b), str(self.vhost1)) + self.assertFalse(self.vhost1b == 1234) - self.assertEqual(vhost1b, self.vhost1) - self.assertEqual(str(vhost1b), str(self.vhost1)) - self.assertFalse(vhost1b == 1234) + def test_ne(self): + self.assertTrue(self.vhost1 != self.vhost2) + self.assertFalse(self.vhost1 != self.vhost1b) + + def test_conflicts(self): + from letsencrypt_apache.obj import Addr + from letsencrypt_apache.obj import VirtualHost + + complex_vh = VirtualHost( + "fp", "vhp", + set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]), + False, False) + self.assertTrue(complex_vh.conflicts([self.addr1])) + self.assertTrue(complex_vh.conflicts([self.addr2])) + self.assertFalse(complex_vh.conflicts([self.addr_default])) + + self.assertTrue(self.vhost1.conflicts([self.addr2])) + self.assertFalse(self.vhost1.conflicts([self.addr_default])) + + self.assertFalse(self.vhost2.conflicts([self.addr1, self.addr_default])) + + def test_same_server(self): + from letsencrypt_apache.obj import VirtualHost + no_name1 = VirtualHost( + "fp", "vhp", set([self.addr1]), False, False, None) + no_name2 = VirtualHost( + "fp", "vhp", set([self.addr2]), False, False, None) + no_name3 = VirtualHost( + "fp", "vhp", set([self.addr_default]), + False, False, None) + no_name4 = VirtualHost( + "fp", "vhp", set([self.addr2, self.addr_default]), + False, False, None) + + self.assertTrue(self.vhost1.same_server(self.vhost2)) + self.assertTrue(no_name1.same_server(no_name2)) + + self.assertFalse(self.vhost1.same_server(no_name1)) + self.assertFalse(no_name1.same_server(no_name3)) + self.assertFalse(no_name1.same_server(no_name4)) class AddrTest(unittest.TestCase): @@ -33,6 +79,9 @@ class AddrTest(unittest.TestCase): self.addr1 = Addr.fromstring("127.0.0.1") self.addr2 = Addr.fromstring("127.0.0.1:*") + self.addr_defined = Addr.fromstring("127.0.0.1:443") + self.addr_default = Addr.fromstring("_default_:443") + def test_wildcard(self): self.assertFalse(self.addr.is_wildcard()) self.assertTrue(self.addr1.is_wildcard()) @@ -47,9 +96,36 @@ class AddrTest(unittest.TestCase): self.assertEqual( self.addr1.get_sni_addr("443"), Addr.fromstring("127.0.0.1")) + def test_conflicts(self): + # Note: Defined IP is more important than defined port in match + self.assertTrue(self.addr.conflicts(self.addr1)) + self.assertTrue(self.addr.conflicts(self.addr2)) + self.assertTrue(self.addr.conflicts(self.addr_defined)) + self.assertFalse(self.addr.conflicts(self.addr_default)) + + self.assertFalse(self.addr1.conflicts(self.addr)) + self.assertTrue(self.addr1.conflicts(self.addr_defined)) + self.assertFalse(self.addr1.conflicts(self.addr_default)) + + self.assertFalse(self.addr_defined.conflicts(self.addr1)) + self.assertFalse(self.addr_defined.conflicts(self.addr2)) + self.assertFalse(self.addr_defined.conflicts(self.addr)) + self.assertFalse(self.addr_defined.conflicts(self.addr_default)) + + self.assertTrue(self.addr_default.conflicts(self.addr)) + self.assertTrue(self.addr_default.conflicts(self.addr1)) + self.assertTrue(self.addr_default.conflicts(self.addr_defined)) + + # Self test + self.assertTrue(self.addr.conflicts(self.addr)) + self.assertTrue(self.addr1.conflicts(self.addr1)) + # This is a tricky one... + self.assertTrue(self.addr1.conflicts(self.addr2)) + def test_equal(self): self.assertTrue(self.addr1 == self.addr2) self.assertFalse(self.addr == self.addr1) + self.assertFalse(self.addr == 123) def test_not_equal(self): self.assertFalse(self.addr1 != self.addr2) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 3075a42c3..a378d0b86 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -6,6 +6,8 @@ import unittest import augeas import mock +from letsencrypt import errors + from letsencrypt_apache.tests import util @@ -77,6 +79,20 @@ class BasicParserTest(util.ParserTest): self.assertEqual(len(matches), 1) self.assertTrue("IfModule" in matches[0]) + def test_add_dir_to_ifmodssl_multiple(self): + from letsencrypt_apache.parser import get_aug_path + # This makes sure that find_dir will work + self.parser.modules.add("mod_ssl.c") + + self.parser.add_dir_to_ifmodssl( + get_aug_path(self.parser.loc["default"]), + "FakeDirective", ["123", "456", "789"]) + + matches = self.parser.find_dir("FakeDirective") + + self.assertEqual(len(matches), 3) + self.assertTrue("IfModule" in matches[0]) + def test_get_aug_path(self): from letsencrypt_apache.parser import get_aug_path self.assertEqual("/files/etc/apache", get_aug_path("/etc/apache")) @@ -87,11 +103,69 @@ class BasicParserTest(util.ParserTest): mock_path.isfile.side_effect = [True, False, False] # pylint: disable=protected-access - results = self.parser._set_locations("root") + results = self.parser._set_locations() self.assertEqual(results["default"], results["listen"]) self.assertEqual(results["default"], results["name"]) + def test_set_user_config_file(self): + path = os.path.join(self.parser.root, "httpd.conf") + open(path, 'w').close() + self.parser.add_dir(self.parser.loc["default"], "Include", "httpd.conf") + + self.assertEqual( + path, self.parser._set_user_config_file()) + + @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") + def test_update_runtime_variables(self, mock_cfg): + mock_cfg.return_value = ( + 'ServerRoot: "/etc/apache2"\n' + 'Main DocumentRoot: "/var/www"\n' + 'Main ErrorLog: "/var/log/apache2/error.log"\n' + 'Mutex ssl-stapling: using_defaults\n' + 'Mutex ssl-cache: using_defaults\n' + 'Mutex default: dir="/var/lock/apache2" mechanism=fcntl\n' + 'Mutex watchdog-callback: using_defaults\n' + 'PidFile: "/var/run/apache2/apache2.pid"\n' + 'Define: TEST\n' + 'Define: DUMP_RUN_CFG\n' + 'Define: U_MICH\n' + 'Define: TLS=443\n' + 'Define: example_path=Documents/path\n' + 'User: name="www-data" id=33 not_used\n' + 'Group: name="www-data" id=33 not_used\n' + ) + expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", + "example_path":"Documents/path"} + + self.parser.update_runtime_variables("ctl") + self.assertEqual(self.parser.variables, expected_vars) + + @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") + def test_update_runtime_vars_bad_output(self, mock_cfg): + mock_cfg.return_value = "Define: TLS=443=24" + self.assertRaises( + errors.PluginError, self.parser.update_runtime_variables, "ctl") + + mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24" + self.assertRaises( + errors.PluginError, self.parser.update_runtime_variables, "ctl") + + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_update_runtime_vars_bad_ctl(self, mock_popen): + mock_popen.side_effect = OSError + self.assertRaises( + errors.MisconfigurationError, + self.parser.update_runtime_variables, "ctl") + + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") + def test_update_runtime_vars_bad_exit(self, mock_popen): + mock_popen().communicate.return_value = ("", "") + mock_popen.returncode = -1 + self.assertRaises( + errors.MisconfigurationError, + self.parser.update_runtime_variables, "ctl") + class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index fcec3a6a5..b1cb25050 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -87,7 +87,7 @@ def get_apache_configurator( version=version) # This allows testing scripts to set it a bit more quickly if conf is not None: - config.conf = conf + config.conf = conf # pragma: no cover config.prepare() @@ -123,4 +123,4 @@ def get_vh_truth(temp_dir, config_name): ] return vh_truth - return None + return None # pragma: no cover From aecb7b71d7db6d69d3158aa7fcca25a62d853d95 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 23 Jul 2015 14:51:07 -0700 Subject: [PATCH 24/52] 100% Augeas Configr unittests --- .../letsencrypt_apache/augeas_configurator.py | 3 +- .../tests/augeas_configurator_test.py | 81 +++++++++++++++++++ .../apache2/conf-available/bad_conf_file.conf | 5 ++ letsencrypt/reverter.py | 3 + letsencrypt/tests/reverter_test.py | 1 - 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/conf-available/bad_conf_file.conf diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index ea4bd0384..91b78c566 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -87,7 +87,8 @@ class AugeasConfigurator(common.Plugin): self._log_save_errors(ex_errs) # Erase Save Notes self.save_notes = "" - return False + raise errors.PluginError( + "Error saving files, check logs for more info.") # Retrieve list of modified files # Note: Noop saves can cause the file to be listed twice, I used a diff --git a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py new file mode 100644 index 000000000..b9b6712a7 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py @@ -0,0 +1,81 @@ +"""Test for letsencrypt_apache.augeas_configurator.""" +import os +import shutil +import unittest + +import mock + +from letsencrypt import errors + +from letsencrypt.tests import acme_util + +from letsencrypt_apache import configurator +from letsencrypt_apache import obj + +from letsencrypt_apache.tests import util + + +class AugeasConfiguratorTest(util.ApacheTest): + """Test for Augeas Configurator base class.""" + + def setUp(self): # pylint: disable=arguments-differ + super(AugeasConfiguratorTest, self).setUp() + + self.config = util.get_apache_configurator( + self.config_path, self.config_dir, self.work_dir) + + self.vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/two_vhost_80") + + def tearDown(self): + shutil.rmtree(self.temp_dir) + shutil.rmtree(self.config_dir) + shutil.rmtree(self.work_dir) + + def test_bad_parse(self): + self.config.parser._parse_file(os.path.join( + self.config.parser.root, "conf-available", "bad_conf_file.conf")) + self.assertRaises( + errors.PluginError, self.config.check_parsing_errors, "httpd.aug") + + def test_bad_save(self): + mock_save = mock.Mock() + mock_save.side_effect = IOError + self.config.aug.save = mock_save + + self.assertRaises(errors.PluginError, self.config.save) + + def test_finalize_save(self): + mock_finalize = mock.Mock() + self.config.reverter = mock_finalize + self.config.save("Example Title") + + self.assertTrue(mock_finalize.is_called) + + def test_recovery_routine(self): + mock_load = mock.Mock() + self.config.aug.load = mock_load + + self.config.recovery_routine() + self.assertEqual(mock_load.call_count, 1) + + def test_revert_challenge_config(self): + mock_load = mock.Mock() + self.config.aug.load = mock_load + + self.config.revert_challenge_config() + self.assertEqual(mock_load.call_count, 1) + + def test_rollback_checkpoints(self): + mock_load = mock.Mock() + self.config.aug.load = mock_load + + self.config.rollback_checkpoints() + self.assertEqual(mock_load.call_count, 1) + + def test_view_config_changes(self): + self.config.view_config_changes() + + +if __name__ == "__main__": + unittest.main() # pragma: no cover \ No newline at end of file diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/conf-available/bad_conf_file.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/conf-available/bad_conf_file.conf new file mode 100644 index 000000000..1aad6a9f4 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/conf-available/bad_conf_file.conf @@ -0,0 +1,5 @@ + + +ServerName invalid.net + + diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index a20ebb8dc..a31281a5b 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -27,6 +27,9 @@ class Reverter(object): def __init__(self, config): self.config = config + le_util.make_or_verify_dir( + config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid()) + def revert_temporary_config(self): """Reload users original configuration files after a temporary save. diff --git a/letsencrypt/tests/reverter_test.py b/letsencrypt/tests/reverter_test.py index f2615453a..da57bf8dc 100644 --- a/letsencrypt/tests/reverter_test.py +++ b/letsencrypt/tests/reverter_test.py @@ -377,7 +377,6 @@ def setup_work_direc(): """ work_dir = tempfile.mkdtemp("work") backup_dir = os.path.join(work_dir, "backup") - os.makedirs(backup_dir) return mock.MagicMock( work_dir=work_dir, backup_dir=backup_dir, From 647caba164a28c2b3bdb8d8fab18f4a357910fbd Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 03:22:35 -0700 Subject: [PATCH 25/52] 100% configurator coverage --- .../letsencrypt_apache/configurator.py | 131 ++++----- .../tests/configurator_test.py | 275 ++++++++++++++++-- letsencrypt/cli.py | 2 +- letsencrypt/errors.py | 3 + 4 files changed, 317 insertions(+), 94 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 915c129cf..59b000a46 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -88,6 +88,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): private_ips_regex = re.compile( r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") + hostname_regex = re.compile( + r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$", re.IGNORECASE) + @classmethod def add_parser_arguments(cls, add): @@ -121,8 +124,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add name_server association dict self.assoc = dict() - # Add number of outstanding challenges - self._chall_out = 0 + # Outstanding challenges + self._chall_out = set() # These will be set in the prepare function self.parser = None @@ -147,7 +150,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: - self.version = self.get_version() # pragma: no cover + self.version = self.get_version() + if self.version < (2, 2): + raise errors.NotSupportedError( + "Apache Version %s not supported.", str(self.version)) # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() @@ -208,11 +214,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug.set(path["chain_path"][-1], chain_path) # Save notes about the transaction that took place - self.save_notes += ("Changed vhost at %s with addresses of %s\n" % + self.save_notes += ("Changed vhost at %s with addresses of %s\n" + "\tSSLCertificateFile %s\n" + "\tSSLCertificateKeyFile %s\n" % (vhost.filep, - ", ".join(str(addr) for addr in vhost.addrs))) - self.save_notes += "\tSSLCertificateFile %s\n" % cert_path - self.save_notes += "\tSSLCertificateKeyFile %s\n" % key_path + ", ".join(str(addr) for addr in vhost.addrs), + cert_path, key_path)) if chain_path is not None: self.save_notes += "\tSSLCertificateChainFile %s\n" % chain_path @@ -285,7 +292,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): points = 1 else: # No points given if names can't be found. - continue + # This gets hit but doesn't register + continue # pragma: no cover if vhost.ssl: points += 2 @@ -309,19 +317,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addr.get_addr() == "_default_" for addr in vh.addrs )] - def create_dn_server_assoc(self, domain, vhost): - """Create an association between a domain name and virtual host. - - Helps to choose an appropriate vhost - - :param str domain: domain name to associate - - :param vhost: virtual host to associate with domain - :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` - - """ - self.assoc[domain] = vhost - def get_all_names(self): """Returns all names found in the Apache Configuration. @@ -334,10 +329,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for vhost in self.vhosts: all_names.update(vhost.get_names()) + for addr in vhost.addrs: - name = self.get_name_from_ip(addr) - if name: - all_names.add(name) + if ApacheConfigurator.hostname_regex.match(addr.get_addr()): + all_names.add(addr.get_addr()) + else: + name = self.get_name_from_ip(addr) + if name: + all_names.add(name) return all_names @@ -460,14 +459,17 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ loc = parser.get_aug_path(self.parser.loc["name"]) - if addr.get_port == "443": + + if addr.get_port() == "443": path = self.parser.add_dir_to_ifmodssl( loc, "NameVirtualHost", [str(addr)]) else: path = self.parser.add_dir(loc, "NameVirtualHost", [str(addr)]) - self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr - self.save_notes += "\tDirective added to %s\n" % path + msg = ("Setting %s to be NameBasedVirtualHost\n" + "\tDirective added to %s\n" % (addr, path)) + logger.debug(msg) + self.save_notes += msg def prepare_server_https(self, port): """Prepare the server for HTTPS. @@ -515,17 +517,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if self.version >= (2, 4): return - # TODO: Review this 3-year old demo code - # Check for NameVirtualHost - # First see if any of the vhost addresses is a _default_ addr - for addr in addrs: - if addr.get_addr() == "_default_": - if not self.is_name_vhost(default_addr): - logger.debug("Setting all VirtualHosts on %s to be " - "name based vhosts", default_addr) - self.add_name_vhost(default_addr) - - # No default addresses... so set each one individually for addr in addrs: if not self.is_name_vhost(addr): logger.debug("Setting VirtualHost at %s to be a name " @@ -661,9 +652,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): need_to_save = False # See if the exact address appears in any other vhost + # Remember 1.1.1.1:* == 1.1.1.1 -> hence any() for addr in vhost.addrs: for test_vh in self.vhosts: - if (vhost.filep != test_vh.filep and addr in test_vh.addrs and + if (vhost.filep != test_vh.filep and + any(test_addr == addr for test_addr in test_vh.addrs) and not self.is_name_vhost(addr)): self.add_name_vhost(addr) logger.info("Enabling NameVirtualHosts on %s", addr) @@ -739,7 +732,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "Unable to create one as intended addresses conflict; " "Current configuration does not support automated " "redirection") - self._create_redirect_vhost(redirect_addrs) + self._create_redirect_vhost(ssl_vhost) else: # Check if redirection already exists self._verify_no_redirects(general_vh) @@ -817,6 +810,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): serveralias = "" servername = "" + if ssl_vhost.name is not None: servername = "ServerName " + ssl_vhost.name if ssl_vhost.aliases: @@ -833,9 +827,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ErrorLog /var/log/apache2/redirect.error.log\n" "LogLevel warn\n" "\n" - % (" ".join(self._get_redirect_addrs(ssl_vhost)), - servername, serveralias, - " ".join(constants.REWRITE_HTTPS_ARGS))) + % ( + " ".join(str(addr) for addr in self._get_redirect_addrs(ssl_vhost)), + servername, serveralias, + " ".join(constants.REWRITE_HTTPS_ARGS))) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -845,7 +840,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if ssl_vhost.name is not None: # make sure servername doesn't exceed filename length restriction if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): - redirect_filename = "le-redirect-%s.conf" % ssl_vhost.servername + redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name redirect_filepath = os.path.join( self.parser.root, "sites-available", redirect_filename) @@ -900,9 +895,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for vhost in self.vhosts: if vhost.ssl: cert_path = self.parser.find_dir( - "SSLCertificateFile", None, vhost.path) + "SSLCertificateFile", None, vhost.path, exclude=False) key_path = self.parser.find_dir( - "SSLCertificateKeyFile", None, vhost.path) + "SSLCertificateKeyFile", None, vhost.path, exclude=False) if cert_path and key_path: cert = os.path.abspath(self.parser.get_arg(cert_path[-1])) @@ -940,12 +935,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param vhost: vhost to enable :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` - :returns: Success - :rtype: bool - """ if self.is_site_enabled(vhost.filep): - return True + return if vhost.ssl: # TODO: Make this based on addresses @@ -961,8 +953,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost.enabled = True logger.info("Enabling available site: %s", vhost.filep) self.save_notes += "Enabled site %s\n" % vhost.filep - return True - return False + else: + raise errors.MisconfigurationError( + "Unsupported filesystem layout. " + "sites-available/enabled expected.") def enable_mod(self, mod_name): """Enables module in Apache. @@ -976,7 +970,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if (not os.path.isdir(os.path.join(self.parser.root, "mods-available")) or not os.path.isdir( os.path.join(self.parser.root, "mods-enabled"))): - raise errors.MisconfigurationError( + raise errors.NotSupportedError( "Unsupported directory layout. You may try to enable mod %s " "and try again." % mod_name) @@ -1001,7 +995,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): elif mod_name == "rewrite": self._enable_mod_debian_files(["rewrite.load"], "rewrite_module") else: - raise NotImplementedError + raise errors.NotSupportedError def _enable_mod_debian_files(self, filenames, mod_name): """Move over all required files into mods-enabled.""" @@ -1011,7 +1005,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check to see all files are available. for filename in filenames: if not os.path.isfile(os.path.join(mods_available, filename)): - raise errors.MisconfigurationError( + raise errors.NoInstallationError( "Unable to enable module. Required files missing from " "mods-available. %s" % str(filenames)) @@ -1029,6 +1023,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def restart(self): """Restarts apache server. + .. todo:: This function will be converted to using reload + :returns: Success :rtype: bool @@ -1053,28 +1049,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError("Unable to run apache2ctl") if proc.returncode != 0: - print proc.returncode # Enter recovery routine... logger.error("Apache Configtest failed\n%s\n%s", stdout, stderr) raise errors.MisconfigurationError( "Apache Configtest failure:\n%s\n%s" % (stdout, stderr)) - def verify_setup(self): - """Verify the setup to ensure safe operating environment. - - Make sure that files/directories are setup with appropriate permissions - Aim for defensive coding... make sure all input files - have permissions of root - - """ - uid = os.geteuid() - le_util.make_or_verify_dir( - self.config.config_dir, core_constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( - self.config.backup_dir, core_constants.CONFIG_DIRS_MODE, uid) - def get_version(self): """Return version of Apache Server. @@ -1129,7 +1108,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): outstanding challenges will have to be designed better. """ - self._chall_out += len(achalls) + self._chall_out.update(achalls) responses = [None] * len(achalls) apache_dvsni = dvsni.ApacheDvsni(self) @@ -1157,10 +1136,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def cleanup(self, achalls): """Revert all challenges.""" - self._chall_out -= len(achalls) + self._chall_out.difference_update(achalls) # If all of the challenges have been finished, clean up everything - if self._chall_out <= 0: + if not self._chall_out: self.revert_challenge_config() self.restart() @@ -1192,7 +1171,7 @@ def apache_restart(apache_init_script): except (OSError, ValueError): logger.fatal( "Unable to restart the Apache process with %s", apache_init_script) - raise errors.PluginError( + raise errors.MisconfigurationError( "Unable to restart Apache process with %s" % apache_init_script) stdout, stderr = proc.communicate() diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index c2e7d2916..233b20b42 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -1,6 +1,7 @@ """Test for letsencrypt_apache.configurator.""" import os import shutil +import socket import unittest import mock @@ -36,6 +37,15 @@ class TwoVhost80Test(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + @mock.patch("letsencrypt_apache.parser.ApacheParser") + def test_prepare_version(self, mock_parser): + self.config.version = None + self.config.config_test = mock.Mock() + self.config.get_version = mock.Mock(return_value=(1, 1)) + + self.assertRaises( + errors.NotSupportedError, self.config.prepare) + def test_add_parser_arguments(self): from letsencrypt_apache.configurator import ApacheConfigurator # Weak test.. @@ -46,6 +56,31 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(names, set( ["letsencrypt.demo", "encryption-example.demo", "ip-172-30-0-17"])) + @mock.patch("letsencrypt_apache.configurator.socket.gethostbyaddr") + def test_get_all_names_addrs(self, mock_gethost): + mock_gethost.side_effect = [("google.com","",""), socket.error] + vh = obj.VirtualHost( + "fp", "ap", + set([obj.Addr(("8.8.8.8", "443")), + obj.Addr(("zombo.com",)), + obj.Addr(("192.168.1.2"))]), + True, False) + self.config.vhosts.append(vh) + + names = self.config.get_all_names() + self.assertEqual(len(names), 5) + self.assertTrue("zombo.com" in names) + self.assertTrue("google.com" in names) + self.assertTrue("letsencrypt.demo" in names) + + def test_add_servernames_alias(self): + self.config.parser.add_dir( + self.vh_truth[2].path, "ServerAlias", ["*.le.co"]) + self.config._add_servernames(self.vh_truth[2]) # pylint: disable=protected-access + + self.assertEqual( + self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) + def test_get_virtual_hosts(self): """Make sure all vhosts are being properly found. @@ -88,6 +123,13 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue( self.config._find_best_vhost("does-not-exist.com") is None) + def test_find_best_vhost_variety(self): + ssl_vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + True, False) + self.config.vhosts.append(ssl_vh) + self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) + def test_find_best_vhost_default(self): # Assume only the two default vhosts. self.config.vhosts = [vh for vh in self.config.vhosts @@ -129,6 +171,27 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("ssl_module" in self.config.parser.modules) self.assertTrue("mod_ssl.c" in self.config.parser.modules) + def test_enable_mod_unsupported_dirs(self): + os.removedirs(os.path.join(self.config.parser.root, "mods-enabled")) + self.assertRaises( + errors.NotSupportedError, self.config.enable_mod, "ssl") + + def test_enable_mod_unsupported_mod(self): + self.assertRaises( + errors.NotSupportedError, self.config.enable_mod, "unknown") + + def test_enable_mod_not_installed(self): + os.remove(os.path.join( + self.config.parser.root, "mods-available", "ssl.load")) + self.assertRaises( + errors.NoInstallationError, self.config.enable_mod, "ssl") + + def test_enable_mod_files_already_exist(self): + path = os.path.join(self.config.parser.root, "mods-enabled", "ssl.load") + open(path, "w").close() + self.assertRaises( + errors.PluginError, self.config.enable_mod, "ssl") + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") def test_enable_site(self, mock_popen): mock_popen().returncode = 0 @@ -139,6 +202,15 @@ class TwoVhost80Test(util.ApacheTest): self.config.enable_site(self.vh_truth[1]) self.assertTrue(self.vh_truth[1].enabled) + # Go again to make sure nothing fails + self.config.enable_site(self.vh_truth[1]) + + def test_enable_site_failure(self): + self.assertRaises( + errors.MisconfigurationError, + self.config.enable_site, + obj.VirtualHost("asdf", "afsaf", set(), False, False)) + @mock.patch("letsencrypt_apache.parser.subprocess.Popen") def test_deploy_cert(self, mock_popen): mock_popen().returncode = 0 @@ -204,8 +276,11 @@ class TwoVhost80Test(util.ApacheTest): def test_add_name_vhost(self): self.config.add_name_vhost(obj.Addr.fromstring("*:443")) + self.config.add_name_vhost(obj.Addr.fromstring("*:80")) self.assertTrue(self.config.parser.find_dir( - "NameVirtualHost", "*:443")) + "NameVirtualHost", "*:443", exclude=False)) + self.assertTrue(self.config.parser.find_dir( + "NameVirtualHost", "*:80")) def test_prepare_server_https(self): self.config.parser.modules.add("ssl_module") @@ -218,7 +293,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.parser.add_dir_to_ifmodssl = mock_add_dir self.config.prepare_server_https("443") - self.assertTrue(mock_add_dir.called) + self.config.prepare_server_https("8080") + self.assertEqual(mock_add_dir.call_count, 2) def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -246,26 +322,37 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(len(self.config.vhosts), 5) + def test_make_vhost_ssl_extra_vhs(self): + self.config.aug.match = mock.Mock(return_value=["p1", "p2"]) + self.assertRaises( + errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) + + def test_make_vhost_ssl_bad_write(self): + mock_open = mock.mock_open() + # This calls open + self.config.reverter.register_file_creation = mock.Mock() + mock_open.side_effect = IOError + with mock.patch("__builtin__.open", mock_open): + self.assertRaises( + errors.PluginError, + self.config.make_vhost_ssl, self.vh_truth[0]) + + def test_get_ssl_vhost_path(self): + self.assertTrue( + self.config._get_ssl_vhost_path("example_path").endswith(".conf")) + + def test_add_name_vhost_if_necessary(self): + self.config.save = mock.Mock() + self.config.version = (2, 2) + self.config._add_name_vhost_if_necessary(self.vh_truth[0]) + self.assertTrue(self.config.save.called) + @mock.patch("letsencrypt_apache.configurator.dvsni.ApacheDvsni.perform") @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") def test_perform(self, mock_restart, mock_dvsni_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - auth_key = le_util.Key(self.rsa256_file, self.rsa256_pem) - achall1 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI( - r="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q", - nonce="37bc5eb75d3e00a19b4f6355845e5a18"), - "pending"), - domain="encryption-example.demo", key=auth_key) - achall2 = achallenges.DVSNI( - challb=acme_util.chall_to_challb( - challenges.DVSNI( - r="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU", - nonce="59ed014cac95f77057b1d7a1b2c596ba"), - "pending"), - domain="letsencrypt.demo", key=auth_key) + auth_key, achall1, achall2 = self.get_achalls() dvsni_ret_val = [ challenges.DVSNIResponse(s="randomS1"), @@ -280,6 +367,31 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(mock_restart.call_count, 1) + @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") + def test_cleanup(self, mock_restart): + auth_key, achall1, achall2 = self.get_achalls() + + self.config._chall_out.add(achall1) + self.config._chall_out.add(achall2) + + self.config.cleanup([achall1]) + self.assertFalse(mock_restart.called) + + self.config.cleanup([achall2]) + self.assertTrue(mock_restart.called) + + @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") + def test_cleanup_no_errors(self, mock_restart): + auth_key, achall1, achall2 = self.get_achalls() + + self.config._chall_out.add(achall1) + + self.config.cleanup([achall2]) + self.assertFalse(mock_restart.called) + + self.config.cleanup([achall1, achall2]) + self.assertTrue(mock_restart.called) + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") def test_get_version(self, mock_popen): mock_popen().communicate.return_value = ( @@ -301,7 +413,79 @@ class TwoVhost80Test(util.ApacheTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.PluginError, self.config.get_version) + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_restart(self, mock_popen): + """These will be changed soon enough with reload.""" + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("", "") + + self.config.restart() + + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_restart_bad_process(self, mock_popen): + mock_popen.side_effect = OSError + + self.assertRaises(errors.MisconfigurationError, self.config.restart) + + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_restart_failure(self, mock_popen): + mock_popen().communicate.return_value = ("", "") + mock_popen.returncode=1 + + self.assertRaises(errors.MisconfigurationError, self.config.restart) + + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_config_test(self, mock_popen): + mock_popen().communicate.return_value = ("a", "b") + mock_popen().returncode = 0 + + self.config.config_test() + + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_config_test_bad_process(self, mock_popen): + mock_popen.side_effect = ValueError + + self.assertRaises(errors.PluginError, self.config.config_test) + + @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") + def test_config_test_failure(self, mock_popen): + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = -1 + + self.assertRaises(errors.MisconfigurationError, self.config.config_test) + + + def test_get_all_certs_keys(self): + c_k = self.config.get_all_certs_keys() + + self.assertEqual(len(c_k), 1) + cert, key, path = next(iter(c_k)) + self.assertTrue("cert" in cert) + self.assertTrue("key" in key) + self.assertTrue("default-ssl.conf" in path) + + def test_get_all_certs_keys_malformed_conf(self): + self.config.parser.find_dir = mock.Mock(side_effect=[["path"], []]) + c_k = self.config.get_all_certs_keys() + + self.assertFalse(c_k) + + def test_more_info(self): + self.assertTrue(self.config.more_info()) + + def test_get_chall_pref(self): + self.assertTrue(isinstance(self.config.get_chall_pref(""), list)) + + def test_temp_install(self): + from letsencrypt_apache.configurator import temp_install + path = os.path.join(self.work_dir, "test_it") + temp_install(path) + self.assertTrue(os.path.isfile(path)) + # TEST ENHANCEMENTS + def test_supported_enhancements(self): + self.assertTrue(isinstance(self.config.supported_enhancements(), list)) + def test_enhance_unknown_enhancement(self): self.assertRaises( errors.PluginError, @@ -329,6 +513,17 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("rewrite_module" in self.config.parser.modules) + def test_redirect_with_conflict(self): + self.config.parser.modules.add("rewrite_module") + ssl_vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + True, False) + # No names ^ this guy should conflict. + + # pylint: disable=protected-access + self.assertRaises( + errors.PluginError, self.config._enable_redirect, ssl_vh, "") + def test_redirect_twice(self): # Skip the enable mod self.config.parser.modules.add("rewrite_module") @@ -346,6 +541,15 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises( errors.PluginError, self.config.enhance, "letsencrypt.demo", "redirect") + def test_unknown_rewrite2(self): + # Skip the enable mod + self.config.parser.modules.add("rewrite_module") + self.config.parser.add_dir( + self.vh_truth[3].path, "RewriteRule", ["Unknown", "2", "3"]) + self.config.save() + self.assertRaises( + errors.PluginError, + self.config.enhance, "letsencrypt.demo", "redirect") def test_unknown_redirect(self): # Skip the enable mod @@ -357,6 +561,43 @@ class TwoVhost80Test(util.ApacheTest): errors.PluginError, self.config.enhance, "letsencrypt.demo", "redirect") + def test_create_own_redirect(self): + self.config.parser.modules.add("rewrite_module") + # For full testing... give names... + self.vh_truth[1].name = "default.com" + self.vh_truth[1].aliases = set(["yes.default.com"]) + + self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access + self.assertEqual(len(self.config.vhosts), 5) + + def get_achalls(self): + auth_key = le_util.Key(self.rsa256_file, self.rsa256_pem) + achall1 = achallenges.DVSNI( + challb=acme_util.chall_to_challb( + challenges.DVSNI( + r="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q", + nonce="37bc5eb75d3e00a19b4f6355845e5a18"), + "pending"), + domain="encryption-example.demo", key=auth_key) + achall2 = achallenges.DVSNI( + challb=acme_util.chall_to_challb( + challenges.DVSNI( + r="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU", + nonce="59ed014cac95f77057b1d7a1b2c596ba"), + "pending"), + domain="letsencrypt.demo", key=auth_key) + + return auth_key, achall1, achall2 + + def test_make_addrs_sni_ready(self): + self.config.version = (2, 2) + self.config.make_addrs_sni_ready( + set([obj.Addr.fromstring("*:443"), obj.Addr.fromstring("*:80")])) + self.assertTrue(self.config.parser.find_dir( + "NameVirtualHost", "*:80", exclude=False)) + self.assertTrue(self.config.parser.find_dir( + "NameVirtualHost", "*:443", exclude=False)) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0797f23b4..b25ef0760 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -172,7 +172,7 @@ def run(args, config, plugins): authenticator = display_ops.pick_authenticator( config, args.authenticator, plugins) else: - # TODO: this assume that user doesn't want to pick authenticator + # TODO: this assumes that user doesn't want to pick authenticator # and installer separately... authenticator = installer = display_ops.pick_configurator( config, args.configurator, plugins) diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index e1cae19c7..5cc45f000 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -65,6 +65,9 @@ class NoInstallationError(PluginError): class MisconfigurationError(PluginError): """Let's Encrypt Misconfiguration error.""" +class NotSupportedError(PluginError): + """Let's Encrypt Plugin function not supported error.""" + class RevokerError(Error): """Let's Encrypt Revoker error.""" From 9454995565d7e6a28e304df290018dd3d0a75e2d Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 03:29:16 -0700 Subject: [PATCH 26/52] remove verify_setup call --- letsencrypt-apache/letsencrypt_apache/configurator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 59b000a46..8bcdb1fbe 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -118,10 +118,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): version = kwargs.pop("version", None) super(ApacheConfigurator, self).__init__(*args, **kwargs) - # Verify that all directories and files exist with proper permissions - if os.geteuid() == 0: - self.verify_setup() # pragma: no cover - # Add name_server association dict self.assoc = dict() # Outstanding challenges From 1ff899ae3307cd460b2ba7995965ee4c6c532a67 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 15:47:38 -0700 Subject: [PATCH 27/52] pylint cleanup --- .../letsencrypt_apache/configurator.py | 24 ++++------- letsencrypt-apache/letsencrypt_apache/obj.py | 17 +++++--- .../tests/augeas_configurator_test.py | 10 ++--- .../tests/configurator_test.py | 41 +++++++++++-------- .../letsencrypt_apache/tests/parser_test.py | 1 + 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 8bcdb1fbe..c35922e21 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -13,10 +13,8 @@ import zope.interface from acme import challenges from letsencrypt import achallenges -from letsencrypt import constants as core_constants from letsencrypt import errors from letsencrypt import interfaces -from letsencrypt import le_util from letsencrypt_apache import augeas_configurator from letsencrypt_apache import constants @@ -369,15 +367,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): serveralias_match = self.parser.find_dir( "ServerAlias", None, host.path, exclude=False) - aliases = [] for alias in serveralias_match: - aliases.append(self.parser.get_arg(alias)) + host.aliases.add(self.parser.get_arg(alias)) if servername_match: # Get last ServerName as each overwrites the previous - host.add_names(self.parser.get_arg(servername_match[-1]), aliases) - else: - host.add_names(None, aliases) + host.name = self.parser.get_arg(servername_match[-1]) def _create_vhost(self, path): """Used by get_virtual_hosts to create vhost objects @@ -498,16 +493,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.save_notes += "Added Listen %s directive to %s\n" % ( port, self.parser.loc["listen"]) - def make_addrs_sni_ready( - self, addrs, default_addr=obj.Addr(("*", "443"))): + def make_addrs_sni_ready(self, addrs): """Checks to see if the server is ready for SNI challenges. :param addrs: Addresses to check SNI compatibility :type addrs: :class:`~letsencrypt_apache.obj.Addr` - :param default_addr: TODO - investigate function further - :type default_addr: :class:~letsencrypt_apache.obj.Addr - """ # Version 2.4 and later are automatically SNI ready. if self.version >= (2, 4): @@ -823,10 +814,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ErrorLog /var/log/apache2/redirect.error.log\n" "LogLevel warn\n" "\n" - % ( - " ".join(str(addr) for addr in self._get_redirect_addrs(ssl_vhost)), - servername, serveralias, - " ".join(constants.REWRITE_HTTPS_ARGS))) + % (" ".join(str(addr) for addr in self._get_redirect_addrs(ssl_vhost)), + servername, serveralias, + " ".join(constants.REWRITE_HTTPS_ARGS))) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -867,7 +857,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return None - def _get_redirect_addrs(self, ssl_vhost): + def _get_redirect_addrs(self, ssl_vhost): # pylint: disable=no-self-use redirects = set() for addr in ssl_vhost.addrs: redirects.add(addr.get_addr_obj("80")) diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index f192ad6da..040b69082 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -23,10 +23,16 @@ class Addr(common.Addr): def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" + # pylint: disable=protected-access return addr._rank_specific_addr() > self._rank_specific_addr() def _rank_specific_addr(self): - """Returns numerical rank for get_addr()""" + """Returns numerical rank for get_addr() + + :returns: 2 - FQ, 1 - wildcard, 0 - _default_ + :rtype: int + + """ if self.get_addr() == "_default_": return 0 elif self.get_addr() == "*": @@ -101,14 +107,14 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods # ?: is used for not returning enclosed characters strip_name = re.compile(r"^(?:.+://)?([^ :$]*)") - def __init__(self, filep, path, addrs, ssl, enabled, name=None, aliases=[]): + def __init__(self, filep, path, addrs, ssl, enabled, name=None, aliases=None): # pylint: disable=too-many-arguments """Initialize a VH.""" self.filep = filep self.path = path self.addrs = addrs self.name = name - self.aliases = aliases + self.aliases = aliases if aliases is not None else set() self.ssl = ssl self.enabled = enabled @@ -119,7 +125,8 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def get_names(self): """Return a set of all names.""" - all_names = set(self.aliases) + all_names = set() + all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: all_names.add(VirtualHost.strip_name.findall(self.name)[0]) @@ -216,4 +223,4 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods else: return False - return True \ No newline at end of file + return True diff --git a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py index b9b6712a7..8cb1fb3a8 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py @@ -7,11 +7,6 @@ import mock from letsencrypt import errors -from letsencrypt.tests import acme_util - -from letsencrypt_apache import configurator -from letsencrypt_apache import obj - from letsencrypt_apache.tests import util @@ -28,11 +23,12 @@ class AugeasConfiguratorTest(util.ApacheTest): self.temp_dir, "debian_apache_2_4/two_vhost_80") def tearDown(self): - shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + shutil.rmtree(self.temp_dir) def test_bad_parse(self): + # pylint: disable=protected-access self.config.parser._parse_file(os.path.join( self.config.parser.root, "conf-available", "bad_conf_file.conf")) self.assertRaises( @@ -78,4 +74,4 @@ class AugeasConfiguratorTest(util.ApacheTest): if __name__ == "__main__": - unittest.main() # pragma: no cover \ No newline at end of file + unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 233b20b42..f7c3d12f3 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-public-methods """Test for letsencrypt_apache.configurator.""" import os import shutil @@ -38,7 +39,7 @@ class TwoVhost80Test(util.ApacheTest): shutil.rmtree(self.work_dir) @mock.patch("letsencrypt_apache.parser.ApacheParser") - def test_prepare_version(self, mock_parser): + def test_prepare_version(self, _): self.config.version = None self.config.config_test = mock.Mock() self.config.get_version = mock.Mock(return_value=(1, 1)) @@ -46,7 +47,7 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises( errors.NotSupportedError, self.config.prepare) - def test_add_parser_arguments(self): + def test_add_parser_arguments(self): # pylint: disable=no-self-use from letsencrypt_apache.configurator import ApacheConfigurator # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) @@ -58,14 +59,14 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.configurator.socket.gethostbyaddr") def test_get_all_names_addrs(self, mock_gethost): - mock_gethost.side_effect = [("google.com","",""), socket.error] - vh = obj.VirtualHost( + mock_gethost.side_effect = [("google.com", "", ""), socket.error] + vhost = obj.VirtualHost( "fp", "ap", set([obj.Addr(("8.8.8.8", "443")), obj.Addr(("zombo.com",)), obj.Addr(("192.168.1.2"))]), True, False) - self.config.vhosts.append(vh) + self.config.vhosts.append(vhost) names = self.config.get_all_names() self.assertEqual(len(names), 5) @@ -115,6 +116,7 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[3], self.config.choose_vhost("none.com")) def test_find_best_vhost(self): + # pylint: disable=protected-access self.assertEqual( self.vh_truth[3], self.config._find_best_vhost("letsencrypt.demo")) self.assertEqual( @@ -124,6 +126,7 @@ class TwoVhost80Test(util.ApacheTest): self.config._find_best_vhost("does-not-exist.com") is None) def test_find_best_vhost_variety(self): + # pylint: disable=protected-access ssl_vh = obj.VirtualHost( "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), True, False) @@ -131,10 +134,12 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) def test_find_best_vhost_default(self): + # pylint: disable=protected-access # Assume only the two default vhosts. - self.config.vhosts = [vh for vh in self.config.vhosts - if vh.name not in - ["letsencrypt.demo", "encryption-example.demo"]] + self.config.vhosts = [ + vh for vh in self.config.vhosts + if vh.name not in ["letsencrypt.demo", "encryption-example.demo"] + ] self.assertEqual( self.config._find_best_vhost("example.demo"), self.vh_truth[2]) @@ -338,10 +343,12 @@ class TwoVhost80Test(util.ApacheTest): self.config.make_vhost_ssl, self.vh_truth[0]) def test_get_ssl_vhost_path(self): + # pylint: disable=protected-access self.assertTrue( self.config._get_ssl_vhost_path("example_path").endswith(".conf")) def test_add_name_vhost_if_necessary(self): + # pylint: disable=protected-access self.config.save = mock.Mock() self.config.version = (2, 2) self.config._add_name_vhost_if_necessary(self.vh_truth[0]) @@ -352,7 +359,7 @@ class TwoVhost80Test(util.ApacheTest): def test_perform(self, mock_restart, mock_dvsni_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - auth_key, achall1, achall2 = self.get_achalls() + _, achall1, achall2 = self.get_achalls() dvsni_ret_val = [ challenges.DVSNIResponse(s="randomS1"), @@ -369,10 +376,10 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") def test_cleanup(self, mock_restart): - auth_key, achall1, achall2 = self.get_achalls() + _, achall1, achall2 = self.get_achalls() - self.config._chall_out.add(achall1) - self.config._chall_out.add(achall2) + self.config._chall_out.add(achall1) # pylint: disable=protected-access + self.config._chall_out.add(achall2) # pylint: disable=protected-access self.config.cleanup([achall1]) self.assertFalse(mock_restart.called) @@ -382,9 +389,9 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") def test_cleanup_no_errors(self, mock_restart): - auth_key, achall1, achall2 = self.get_achalls() + _, achall1, achall2 = self.get_achalls() - self.config._chall_out.add(achall1) + self.config._chall_out.add(achall1) # pylint: disable=protected-access self.config.cleanup([achall2]) self.assertFalse(mock_restart.called) @@ -430,7 +437,7 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") def test_restart_failure(self, mock_popen): mock_popen().communicate.return_value = ("", "") - mock_popen.returncode=1 + mock_popen().returncode = 1 self.assertRaises(errors.MisconfigurationError, self.config.restart) @@ -454,7 +461,6 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises(errors.MisconfigurationError, self.config.config_test) - def test_get_all_certs_keys(self): c_k = self.config.get_all_certs_keys() @@ -493,7 +499,7 @@ class TwoVhost80Test(util.ApacheTest): @mock.patch("letsencrypt_apache.parser." "ApacheParser.update_runtime_variables") - def test_redirect_well_formed_http(self, unused): + def test_redirect_well_formed_http(self, _): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "redirect") @@ -571,6 +577,7 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(len(self.config.vhosts), 5) def get_achalls(self): + """Return testing achallenges.""" auth_key = le_util.Key(self.rsa256_file, self.rsa256_pem) achall1 = achallenges.DVSNI( challb=acme_util.chall_to_challb( diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index a378d0b86..45e77bdb9 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -109,6 +109,7 @@ class BasicParserTest(util.ParserTest): self.assertEqual(results["default"], results["name"]) def test_set_user_config_file(self): + # pylint: disable=protected-access path = os.path.join(self.parser.root, "httpd.conf") open(path, 'w').close() self.parser.add_dir(self.parser.loc["default"], "Include", "httpd.conf") From ab42f7fdfeb1f26faf7a9abf8bf35f65c08622c8 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 17:05:25 -0700 Subject: [PATCH 28/52] 100% unittests for apache plugin --- letsencrypt-apache/letsencrypt_apache/dvsni.py | 2 +- letsencrypt-apache/letsencrypt_apache/obj.py | 5 ----- .../letsencrypt_apache/parser.py | 9 +++++---- .../tests/complex_parsing_test.py | 1 + .../tests/display_ops_test.py | 18 ++++++++++++++++-- .../letsencrypt_apache/tests/dvsni_test.py | 11 +++++++++++ .../letsencrypt_apache/tests/parser_test.py | 6 ++++++ .../testdata/complex_parsing/apache2.conf | 2 +- 8 files changed, 41 insertions(+), 13 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/dvsni.py b/letsencrypt-apache/letsencrypt_apache/dvsni.py index 113991c53..fbe30b1a6 100644 --- a/letsencrypt-apache/letsencrypt_apache/dvsni.py +++ b/letsencrypt-apache/letsencrypt_apache/dvsni.py @@ -114,7 +114,7 @@ class ApacheDvsni(common.Dvsni): # TODO: Checkout _default_ rules. dvsni_addrs = set() - default_addr = obj.Addr(("*", self.configurator.config.dvsni_port)) + default_addr = obj.Addr(("*", str(self.configurator.config.dvsni_port))) for addr in vhost.addrs: if "_default_" == addr.get_addr(): diff --git a/letsencrypt-apache/letsencrypt_apache/obj.py b/letsencrypt-apache/letsencrypt_apache/obj.py index 040b69082..c0dcc6c43 100644 --- a/letsencrypt-apache/letsencrypt_apache/obj.py +++ b/letsencrypt-apache/letsencrypt_apache/obj.py @@ -118,11 +118,6 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods self.ssl = ssl self.enabled = enabled - def add_names(self, servername, serveralias): - """Add name to vhost.""" - self.name = servername - self.aliases = serveralias - def get_names(self): """Return a set of all names.""" all_names = set() diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 00af5d114..01ec4aa38 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -396,7 +396,10 @@ class ApacheParser(object): arg = os.path.normpath(os.path.join(self.root, arg)) # Attempts to add a transform to the file if one does not already exist - self._parse_file(arg) + if os.path.isdir(arg): + self._parse_file(os.path.join(arg, "*")) + else: + self._parse_file(arg) # Argument represents an fnmatch regular expression, convert it # Split up the path and convert each into an Augeas accepted regex @@ -409,11 +412,9 @@ class ApacheParser(object): split_arg[idx] = ("* [label()=~regexp('%s')]" % self.fnmatch_to_re(split)) # Reassemble the argument + # Note: This also normalizes the argument /serverroot/ -> /serverroot arg = "/".join(split_arg) - # If the include is a directory, just return the directory as a file - if arg.endswith("/"): - return get_aug_path(arg[:-1]) return get_aug_path(arg) def fnmatch_to_re(self, clean_fn_match): # pylint: disable=no-self-use diff --git a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py index 5e4478f67..d6112a486 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py @@ -36,6 +36,7 @@ class ComplexParserTest(util.ParserTest): ) def test_filter_args_num(self): + """Note: This may also fail do to Include conf-enabled/ syntax.""" matches = self.parser.find_dir("TestArgsDirective") self.assertEqual(len(self.parser.filter_args_num(matches, 1)), 3) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py index 5d08092ce..d7cfb09b3 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py @@ -5,10 +5,12 @@ import unittest import mock import zope.component -from letsencrypt_apache.tests import util - from letsencrypt.display import util as display_util +from letsencrypt_apache import obj + +from letsencrypt_apache.tests import util + class SelectVhostTest(unittest.TestCase): """Tests for letsencrypt_apache.display_ops.select_vhost.""" @@ -53,6 +55,18 @@ class SelectVhostTest(unittest.TestCase): self.assertEqual(mock_logger.debug.call_count, 1) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") + def test_multiple_names(self, mock_util): + mock_util().menu.return_value = (display_util.OK, 4) + + self.vhosts.append( + obj.VirtualHost( + "path", "aug_path", set([obj.Addr.fromstring("*:80")]), + False, False, + "wildcard.com", set(["*.wildcard.com"]))) + + self.assertEqual(self.vhosts[4], self._call(self.vhosts)) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py index 29b600ec3..ff13fef7b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py @@ -114,6 +114,17 @@ class DvsniPerformTest(util.ApacheTest): names == set([self.achalls[0].nonce_domain]) or names == set([self.achalls[1].nonce_domain])) + def test_get_dvsni_addrs_default(self): + self.sni.configurator.choose_vhost = mock.Mock( + return_value=obj.VirtualHost( + "path", "aug_path", set([obj.Addr.fromstring("_default_:443")]), + False, False) + ) + + self.assertEqual( + set([obj.Addr.fromstring("*:443")]), + self.sni.get_dvsni_addrs(self.achalls[0])) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 45e77bdb9..ce234bff7 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -22,6 +22,12 @@ class BasicParserTest(util.ParserTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + def test_find_config_root_no_root(self): + # pylint: disable=protected-access + os.remove(self.parser.loc["root"]) + self.assertRaises( + errors.NoInstallationError, self.parser._find_config_root) + def test_parse_file(self): """Test parse_file. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf index b7b6a9be2..26bf47263 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf +++ b/letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf @@ -38,7 +38,7 @@ IncludeOptional mods-enabled/*.conf # Include generic snippets of statements -IncludeOptional conf-enabled/*.conf +IncludeOptional conf-enabled/ # Include the virtual host configurations: IncludeOptional sites-enabled/*.conf From 6716d9f0b467f8e18ac30b88b3f44fecc6edaae4 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 17:06:27 -0700 Subject: [PATCH 29/52] Remove a2enmod option --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index c35922e21..9e9d606c2 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -98,8 +98,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): help="Path to the 'apache2ctl' binary, used for 'configtest', " "retrieving the Apache2 version number, and initialization " "parameters.") - add("enmod", default=constants.CLI_DEFAULTS["enmod"], - help="Path to the Apache 'a2enmod' binary.") add("init-script", default=constants.CLI_DEFAULTS["init_script"], help="Path to the Apache init script (used for server " "reload/restart).") From 87bf4969c0551d52172b6b4e8c674fd7b8a95c86 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 19:01:35 -0700 Subject: [PATCH 30/52] bump coverage --- tox.cover.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.cover.sh b/tox.cover.sh index f1d882cee..84bbf281a 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -22,5 +22,5 @@ rm -f .coverage # --cover-erase is off, make sure stats are correct # after_success) cover letsencrypt 97 && \ cover acme 100 && \ - cover letsencrypt_apache 78 && \ + cover letsencrypt_apache 100 && \ cover letsencrypt_nginx 96 From ba4a62f18547e2faabc66f460f7e4b06abfaaf5f Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 19:18:07 -0700 Subject: [PATCH 31/52] Add 'empty mods-enabled dir --- .../two_vhost_80/apache2/mods-enabled/.gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/.gitignore diff --git a/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/.gitignore b/letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-enabled/.gitignore new file mode 100644 index 000000000..e69de29bb From ccf678f146542e0a4630b6f9e040d19461c4493f Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 24 Jul 2015 19:23:12 -0700 Subject: [PATCH 32/52] rmdirs -> shutil.rmtree --- .../letsencrypt_apache/tests/configurator_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index f7c3d12f3..d318805a6 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -177,7 +177,7 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("mod_ssl.c" in self.config.parser.modules) def test_enable_mod_unsupported_dirs(self): - os.removedirs(os.path.join(self.config.parser.root, "mods-enabled")) + shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled")) self.assertRaises( errors.NotSupportedError, self.config.enable_mod, "ssl") From d86ade674d72e13d7772663c2ad79c88969fbfe7 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 29 Jul 2015 19:56:44 +0000 Subject: [PATCH 33/52] include_package_data in apache and nginx plugins ref https://github.com/letsencrypt/letsencrypt/issues/625 --- letsencrypt-apache/setup.py | 1 + letsencrypt-nginx/setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index fac5b6b88..39f4b68e1 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -20,4 +20,5 @@ setup( 'apache = letsencrypt_apache.configurator:ApacheConfigurator', ], }, + include_package_data=True, ) diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index bd8e8976d..92b974974 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -19,4 +19,5 @@ setup( 'nginx = letsencrypt_nginx.configurator:NginxConfigurator', ], }, + include_package_data=True, ) From fe237df2ccbd591c258c16554f8d7a67643bc168 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Wed, 29 Jul 2015 21:14:37 +0000 Subject: [PATCH 34/52] Update default server URI to staging. http://letsencrypt.status.io/pages/maintenance/55957a99e800baa4470002da/55b90ed12c1b61d83b0001eb --- letsencrypt/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index 07d1965fb..34abb1dce 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -16,7 +16,7 @@ CLI_DEFAULTS = dict( "letsencrypt", "cli.ini"), ], verbose_count=-(logging.WARNING / 10), - server="https://www.letsencrypt-demo.org/acme/new-reg", + server="https://acme-staging.api.letsencrypt.org/acme/new-reg", rsa_key_size=2048, rollback_checkpoints=0, config_dir="/etc/letsencrypt", From 6a90737bbb5baa49bed19aebfa268dda772aca37 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Wed, 29 Jul 2015 14:54:35 -0700 Subject: [PATCH 35/52] make mktemp in integration tests work on OS X --- tests/integration/_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index be4e75098..6fe06b1ee 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -2,7 +2,7 @@ if [ "xxx$root" = "xxx" ]; then - root="$(mktemp -d)" + root="$(mktemp -d -t leitXXXX)" echo "Root integration tests directory: $root" fi store_flags="--config-dir $root/conf --work-dir $root/work" From 584f19fef59c63c18ea6ba701661fbc48f31d969 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Wed, 29 Jul 2015 15:08:22 -0700 Subject: [PATCH 36/52] add comment for mktemp for @Kuba --- tests/integration/_common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 6fe06b1ee..8656b8518 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -2,6 +2,8 @@ if [ "xxx$root" = "xxx" ]; then + # The -t is required on OS X. It provides a template file path for + # the kernel to use. root="$(mktemp -d -t leitXXXX)" echo "Root integration tests directory: $root" fi From 06e21d3578b9b188100b745a210fe05975b0e5de Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 29 Jul 2015 15:17:37 -0700 Subject: [PATCH 37/52] Fix path for integration test. --- .travis.yml | 1 + tests/boulder-start.sh | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9076c52f5..69e53c4a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ before_install: env: global: - GOPATH=/tmp/go + - PATH=$GOPATH/bin:$PATH matrix: - TOXENV=py26 BOULDER_INTEGRATION=1 - TOXENV=py27 BOULDER_INTEGRATION=1 diff --git a/tests/boulder-start.sh b/tests/boulder-start.sh index 20f64bcce..d988d76c8 100755 --- a/tests/boulder-start.sh +++ b/tests/boulder-start.sh @@ -10,11 +10,5 @@ export GOPATH="${GOPATH:-/tmp/go}" go get -d github.com/letsencrypt/boulder/cmd/boulder cd $GOPATH/src/github.com/letsencrypt/boulder -make -j4 # Travis has 2 cores per build instance. -if [ "$1" = "amqp" ]; -then - ./start.py & -else - ./start.sh & -fi -# Hopefully start.py/start.sh bootstraps before integration test is started... +./start.py & +# Hopefully start.py bootstraps before integration test is started... From 84fe80a3477e6e555a6b4fa2e1e46820e087d27f Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Thu, 30 Jul 2015 04:13:14 +0000 Subject: [PATCH 38/52] "boulder-start.py amqp" is the default as of #637 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9076c52f5..95ce8ad86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ addons: - le.wtf install: "travis_retry pip install tox coveralls" -before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh amqp' +before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh' script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))' after_success: '[ "$TOXENV" == "cover" ] && coveralls' From ae4e1d50587070a56959acac16022bab30ff4c4e Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 29 Jul 2015 23:40:07 -0700 Subject: [PATCH 39/52] Use a2enmod and update reverter --- .../letsencrypt_apache/configurator.py | 84 ++++++------------ .../letsencrypt_apache/constants.py | 1 + .../tests/configurator_test.py | 86 ++++++++----------- .../letsencrypt_apache/tests/dvsni_test.py | 9 +- letsencrypt/errors.py | 4 + letsencrypt/le_util.py | 52 +++++++++++ letsencrypt/reverter.py | 76 ++++++++++++++-- letsencrypt/tests/le_util_test.py | 61 +++++++++++++ letsencrypt/tests/reverter_test.py | 61 +++++++++++-- 9 files changed, 306 insertions(+), 128 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 9e9d606c2..7b1144e2e 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -15,6 +15,7 @@ from acme import challenges from letsencrypt import achallenges from letsencrypt import errors from letsencrypt import interfaces +from letsencrypt import le_util from letsencrypt_apache import augeas_configurator from letsencrypt_apache import constants @@ -92,17 +93,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @classmethod def add_parser_arguments(cls, add): - add("server-root", default=constants.CLI_DEFAULTS["server_root"], - help="Apache server root directory.") add("ctl", default=constants.CLI_DEFAULTS["ctl"], help="Path to the 'apache2ctl' binary, used for 'configtest', " "retrieving the Apache2 version number, and initialization " "parameters.") + add("enmod", default=constants.CLI_DEFAULTS["enmod"], + help="Path to the Apache 'a2enmod' binary.") + add("dismod", default=constants.CLI_DEFAULTS["dismod"], + help="Path to the Apache 'a2enmod' binary.") add("init-script", default=constants.CLI_DEFAULTS["init_script"], help="Path to the Apache init script (used for server " "reload/restart).") add("le-vhost-ext", default=constants.CLI_DEFAULTS["le_vhost_ext"], help="SSL vhost configuration extension.") + add("server-root", default=constants.CLI_DEFAULTS["server_root"], + help="Apache server root directory.") def __init__(self, *args, **kwargs): """Initialize an Apache Configurator. @@ -942,12 +947,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "Unsupported filesystem layout. " "sites-available/enabled expected.") - def enable_mod(self, mod_name): + def enable_mod(self, mod_name, temp=False): """Enables module in Apache. Both enables and restarts Apache so module is active. :param str mod_name: Name of the module to enable. (e.g. 'ssl') + :param bool temp: Whether or not this is a temporary action. """ # Support Debian specific setup @@ -958,7 +964,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "Unsupported directory layout. You may try to enable mod %s " "and try again." % mod_name) - self._enable_mod_debian(mod_name) + self._enable_mod_debian(mod_name, temp) self.save_notes += "Enabled %s module in Apache" % mod_name logger.debug("Enabled Apache %s module", mod_name) @@ -970,39 +976,19 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.modules.add(mod_name + "_module") self.parser.modules.add("mod_" + mod_name + ".c") - def _enable_mod_debian(self, mod_name): + def _enable_mod_debian(self, mod_name, temp): """Assumes mods-available, mods-enabled layout.""" - # TODO: This can be further updated to not require all files. - if mod_name == "ssl": - self._enable_mod_debian_files( - ["ssl.conf", "ssl.load"], "ssl_module") - elif mod_name == "rewrite": - self._enable_mod_debian_files(["rewrite.load"], "rewrite_module") - else: - raise errors.NotSupportedError + # Generate reversal command. + # Try to be safe here... check that we can probably reverse before + # applying enmod command + if not le_util.exe_exists(self.conf("dismod")): + raise errors.MisconfigurationError( + "Unable to find a2dismod, please make sure a2enmod and " + "a2dismod are configured correctly for letsencrypt.") - def _enable_mod_debian_files(self, filenames, mod_name): - """Move over all required files into mods-enabled.""" - mods_available = os.path.join(self.parser.root, "mods-available") - mods_enabled = os.path.join(self.parser.root, "mods-enabled") - - # Check to see all files are available. - for filename in filenames: - if not os.path.isfile(os.path.join(mods_available, filename)): - raise errors.NoInstallationError( - "Unable to enable module. Required files missing from " - "mods-available. %s" % str(filenames)) - - # Register and symlink files - for filename in filenames: - enabled_path = os.path.join(mods_enabled, filename) - if os.path.isfile(enabled_path): - logger.debug( - "Error - enabling module %s, filepath already exists " - "%s", mod_name, enabled_path) - raise errors.PluginError("Error enabling module %s" % mod_name) - self.reverter.register_file_creation(False, enabled_path) - os.symlink(os.path.join(mods_available, filename), enabled_path) + self.reverter.register_undo_command( + temp, [self.conf("dismod"), mod_name]) + le_util.run_script([self.conf("enmod"), mod_name]) def restart(self): """Restarts apache server. @@ -1018,25 +1004,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def config_test(self): # pylint: disable=no-self-use """Check the configuration of Apache for errors. - :raises .errors.PluginError: If Unable to run apache2ctl :raises .errors.MisconfigurationError: If config_test fails """ try: - proc = subprocess.Popen( - [self.conf("ctl"), "configtest"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - except (OSError, ValueError): - logger.fatal("Unable to run /usr/sbin/apache2ctl configtest") - raise errors.PluginError("Unable to run apache2ctl") - - if proc.returncode != 0: - # Enter recovery routine... - logger.error("Apache Configtest failed\n%s\n%s", stdout, stderr) - raise errors.MisconfigurationError( - "Apache Configtest failure:\n%s\n%s" % (stdout, stderr)) + le_util.run_script([self.conf("ctl"), "configtest"]) + except errors.SubprocessError: + raise errors.MisconfigurationError("Config Test failed!") def get_version(self): """Return version of Apache Server. @@ -1050,17 +1024,13 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - proc = subprocess.Popen( - [self.conf("ctl"), "-v"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - text = proc.communicate()[0] - except (OSError, ValueError): + stdout, _ = le_util.run_script([self.conf("ctl"), "-v"]) + except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % self.conf("ctl")) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) - matches = regex.findall(text) + matches = regex.findall(stdout) if len(matches) != 1: raise errors.PluginError("Unable to find Apache version") diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 7e7e127f5..b38e898cf 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -6,6 +6,7 @@ CLI_DEFAULTS = dict( server_root="/etc/apache2", ctl="apache2ctl", enmod="a2enmod", + dismod="a2dismod", init_script="/etc/init.d/apache2", le_vhost_ext="-le-ssl.conf", ) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index d318805a6..46073619a 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -162,50 +162,45 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].filep)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].filep)) + @mock.patch("letsencrypt.le_util.run_script") + @mock.patch("letsencrypt.le_util.exe_exists") @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_enable_mod(self, mock_popen): + def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script): mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") mock_popen().returncode = 0 + mock_exe_exists.return_value = True self.config.enable_mod("ssl") - for filename in ["ssl.conf", "ssl.load"]: - self.assertTrue( - os.path.isfile(os.path.join( - self.config.conf("server-root"), "mods-enabled", filename))) - self.assertTrue("ssl_module" in self.config.parser.modules) self.assertTrue("mod_ssl.c" in self.config.parser.modules) + self.assertTrue(mock_run_script.called) + def test_enable_mod_unsupported_dirs(self): shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled")) self.assertRaises( errors.NotSupportedError, self.config.enable_mod, "ssl") - def test_enable_mod_unsupported_mod(self): + @mock.patch("letsencrypt.le_util.exe_exists") + def test_enable_mod_no_disable(self, mock_exe_exists): + mock_exe_exists.return_value = False self.assertRaises( - errors.NotSupportedError, self.config.enable_mod, "unknown") - - def test_enable_mod_not_installed(self): - os.remove(os.path.join( - self.config.parser.root, "mods-available", "ssl.load")) - self.assertRaises( - errors.NoInstallationError, self.config.enable_mod, "ssl") - - def test_enable_mod_files_already_exist(self): - path = os.path.join(self.config.parser.root, "mods-enabled", "ssl.load") - open(path, "w").close() - self.assertRaises( - errors.PluginError, self.config.enable_mod, "ssl") + errors.MisconfigurationError, self.config.enable_mod, "ssl") + @mock.patch("letsencrypt.le_util.run_script") + @mock.patch("letsencrypt.le_util.exe_exists") @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_enable_site(self, mock_popen): + def test_enable_site(self, mock_popen, mock_exe_exists, mock_run_script): mock_popen().returncode = 0 mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + mock_exe_exists.return_value = True # Default 443 vhost self.assertFalse(self.vh_truth[1].enabled) self.config.enable_site(self.vh_truth[1]) self.assertTrue(self.vh_truth[1].enabled) + # Mod enabled + self.assertTrue(mock_run_script.called) # Go again to make sure nothing fails self.config.enable_site(self.vh_truth[1]) @@ -216,10 +211,9 @@ class TwoVhost80Test(util.ApacheTest): self.config.enable_site, obj.VirtualHost("asdf", "afsaf", set(), False, False)) - @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_deploy_cert(self, mock_popen): - mock_popen().returncode = 0 - mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") + def test_deploy_cert(self): + self.config.parser.modules.add("ssl_module") + self.config.parser.modules.add("mod_ssl.c") # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -399,25 +393,25 @@ class TwoVhost80Test(util.ApacheTest): self.config.cleanup([achall1, achall2]) self.assertTrue(mock_restart.called) - @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") - def test_get_version(self, mock_popen): - mock_popen().communicate.return_value = ( + @mock.patch("letsencrypt.le_util.run_script") + def test_get_version(self, mock_script): + mock_script.return_value = ( "Server Version: Apache/2.4.2 (Debian)", "") self.assertEqual(self.config.get_version(), (2, 4, 2)) - mock_popen().communicate.return_value = ( + mock_script.return_value = ( "Server Version: Apache/2 (Linux)", "") self.assertEqual(self.config.get_version(), (2,)) - mock_popen().communicate.return_value = ( + mock_script.return_value = ( "Server Version: Apache (Debian)", "") self.assertRaises(errors.PluginError, self.config.get_version) - mock_popen().communicate.return_value = ( + mock_script.return_value = ( "Server Version: Apache/2.3{0} Apache/2.4.7".format(os.linesep), "") self.assertRaises(errors.PluginError, self.config.get_version) - mock_popen.side_effect = OSError("Can't find program") + mock_script.side_effect = errors.SubprocessError("Can't find program") self.assertRaises(errors.PluginError, self.config.get_version) @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") @@ -441,23 +435,13 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") - def test_config_test(self, mock_popen): - mock_popen().communicate.return_value = ("a", "b") - mock_popen().returncode = 0 - + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test(self, _): self.config.config_test() - @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") - def test_config_test_bad_process(self, mock_popen): - mock_popen.side_effect = ValueError - - self.assertRaises(errors.PluginError, self.config.config_test) - - @mock.patch("letsencrypt_apache.configurator.subprocess.Popen") - def test_config_test_failure(self, mock_popen): - mock_popen().communicate.return_value = ("", "") - mock_popen().returncode = -1 + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test_bad_process(self, mock_run_script): + mock_run_script.side_effect = errors.SubprocessError self.assertRaises(errors.MisconfigurationError, self.config.config_test) @@ -497,9 +481,11 @@ class TwoVhost80Test(util.ApacheTest): errors.PluginError, self.config.enhance, "letsencrypt.demo", "unknown_enhancement") - @mock.patch("letsencrypt_apache.parser." - "ApacheParser.update_runtime_variables") - def test_redirect_well_formed_http(self, _): + @mock.patch("letsencrypt.le_util.run_script") + @mock.patch("letsencrypt.le_util.exe_exists") + def test_redirect_well_formed_http(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + mock_exe.return_value = True # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "redirect") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py index ff13fef7b..329a5439b 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py @@ -36,10 +36,11 @@ class DvsniPerformTest(util.ApacheTest): resp = self.sni.perform() self.assertEqual(len(resp), 0) - @mock.patch("letsencrypt_apache.parser.subprocess.Popen") - def test_perform1(self, mock_popen): - mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") - mock_popen().returncode = 0 + @mock.patch("letsencrypt.le_util.exe_exists") + @mock.patch("letsencrypt.le_util.run_script") + def test_perform1(self, _, mock_exists): + mock_exists.return_value = True + self.sni.configurator.parser.update_runtime_variables = mock.Mock() achall = self.achalls[0] self.sni.add_chall(achall) diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index 5cc45f000..82331fced 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -5,6 +5,10 @@ class Error(Exception): """Generic Let's Encrypt client error.""" +class SubprocessError(Error): + """Subprocess handling error.""" + + class AccountStorageError(Error): """Generic `.AccountStorage` error.""" diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py index e525a333c..af8a56ef5 100644 --- a/letsencrypt/le_util.py +++ b/letsencrypt/le_util.py @@ -4,6 +4,7 @@ import errno import logging import os import re +import subprocess import stat from letsencrypt import errors @@ -17,6 +18,57 @@ Key = collections.namedtuple("Key", "file pem") CSR = collections.namedtuple("CSR", "file data form") +def run_script(params): + """Run the script with the given params. + + :param list params: List of parameters to pass to Popen + + """ + try: + proc = subprocess.Popen(params, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + except (OSError, ValueError): + msg = "Unable to run the command: %s" % " ".join(params) + logger.error(msg) + raise errors.SubprocessError(msg) + + stdout, stderr = proc.communicate() + + if proc.returncode != 0: + msg = "Error while running %s.\n%s\n%s" % ( + " ".join(params), stdout, stderr) + # Enter recovery routine... + logger.error(msg) + raise errors.SubprocessError(msg) + + return stdout, stderr + + +def exe_exists(exe): + """Determine whether path/name refers to an executable. + + :param str exe: Executable path or name + + :returns: If exe is a valid executable + :rtype: bool + + """ + def is_exe(path): + """Determine if path is an exe.""" + return os.path.isfile(path) and os.access(path, os.X_OK) + + path, _ = os.path.split(exe) + if path: + return is_exe(exe) + else: + for path in os.environ["PATH"].split(os.pathsep): + if is_exe(os.path.join(path, exe)): + return True + + return False + def make_or_verify_dir(directory, mode=0o755, uid=0): """Make sure directory exists with proper permissions. diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index a31281a5b..03d62ce13 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -1,4 +1,5 @@ """Reverter class saves configuration checkpoints and allows for recovery.""" +import csv import logging import os import shutil @@ -20,6 +21,8 @@ logger = logging.getLogger(__name__) class Reverter(object): """Reverter Class - save and revert configuration checkpoints. + .. note:: Consider moving everything over to CSV format. + :param config: Configuration. :type config: :class:`letsencrypt.interfaces.IConfig` @@ -101,6 +104,7 @@ class Reverter(object): if not backups: logger.info("The Let's Encrypt client has not saved any backups " "of your configuration") + return # Make sure there isn't anything unexpected in the backup folder # There should only be timestamped (float) directories @@ -204,7 +208,7 @@ class Reverter(object): notes_fd.write(save_notes) def _read_and_append(self, filepath): # pylint: disable=no-self-use - """Reads the file lines and returns a fd. + """Reads the file lines and returns a file obj. Read the file returning the lines, and a pointer to the end of the file. @@ -230,6 +234,10 @@ class Reverter(object): :raises errors.ReverterError: If unable to recover checkpoint """ + # Undo all commands + if os.path.isfile(os.path.join(cp_dir, "COMMANDS")): + self._run_undo_commands(os.path.join(cp_dir, "COMMANDS")) + # Revert all changed files if os.path.isfile(os.path.join(cp_dir, "FILEPATHS")): try: with open(os.path.join(cp_dir, "FILEPATHS")) as paths_fd: @@ -254,6 +262,17 @@ class Reverter(object): raise errors.ReverterError( "Unable to remove directory: %s" % cp_dir) + def _run_undo_commands(self, filepath): # pylint: disable=no-self-use + """Run all commands in a file.""" + with open(filepath, 'rb') as csvfile: + csvreader = csv.reader(csvfile) + for command in reversed(list(csvreader)): + try: + le_util.run_script(command) + except errors.SubprocessError: + logger.error( + "Unable to run undo command: %s", " ".join(command)) + def _check_tempfile_saves(self, save_files): """Verify save isn't overwriting any temporary files. @@ -306,13 +325,7 @@ class Reverter(object): raise errors.ReverterError( "Forgot to provide files to registration call") - if temporary: - cp_dir = self.config.temp_checkpoint_dir - else: - cp_dir = self.config.in_progress_dir - - le_util.make_or_verify_dir( - cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid()) + cp_dir = self._get_cp_dir(temporary) # Append all new files (that aren't already registered) new_fd = None @@ -331,6 +344,53 @@ class Reverter(object): if new_fd is not None: new_fd.close() + def register_undo_command(self, temporary, command): + """Register a command to be run to undo actions taken. + + .. warning:: This function does not enforce order of operations in terms + of file modification vs. command registration. All undo commands + are run first before all normal files are reverted to their previous + state. If you need to maintain strict order, you may create + checkpoints before and after the the command registration. This + function may be improved in the future based on demand. + + :param bool temporary: Whether the command should be saved in the + IN_PROGRESS or TEMPORARY checkpoints. + :param command: Command to be run. + :type command: list of str + + """ + commands_fp = os.path.join(self._get_cp_dir(temporary), "COMMANDS") + command_file = None + try: + if os.path.isfile(commands_fp): + command_file = open(commands_fp, "ab") + else: + command_file = open(commands_fp, "wb") + + csvwriter = csv.writer(command_file) + csvwriter.writerow(command) + + except (IOError, OSError): + logger.error("Unable to register undo command") + raise errors.ReverterError( + "Unable to register undo command.") + finally: + if command_file is not None: + command_file.close() + + def _get_cp_dir(self, temporary): + """Return the proper reverter directory.""" + if temporary: + cp_dir = self.config.temp_checkpoint_dir + else: + cp_dir = self.config.in_progress_dir + + le_util.make_or_verify_dir( + cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid()) + + return cp_dir + def recovery_routine(self): """Revert configuration to most recent finalized checkpoint. diff --git a/letsencrypt/tests/le_util_test.py b/letsencrypt/tests/le_util_test.py index 1ecc1ea16..6a6ad3a54 100644 --- a/letsencrypt/tests/le_util_test.py +++ b/letsencrypt/tests/le_util_test.py @@ -11,6 +11,67 @@ import mock from letsencrypt import errors +class RunScriptTest(unittest.TestCase): + """Tests for letsencrypt.le_util.run_script.""" + @classmethod + def _call(cls, params): + from letsencrypt.le_util import run_script + return run_script(params) + + @mock.patch("letsencrypt.le_util.subprocess.Popen") + def test_default(self, mock_popen): + """These will be changed soon enough with reload.""" + mock_popen().returncode = 0 + mock_popen().communicate.return_value = ("stdout", "stderr") + + out, err = self._call(["test"]) + self.assertEqual(out, "stdout") + self.assertEqual(err, "stderr") + + @mock.patch("letsencrypt.le_util.subprocess.Popen") + def test_bad_process(self, mock_popen): + mock_popen.side_effect = OSError + + self.assertRaises(errors.SubprocessError, self._call, ["test"]) + + @mock.patch("letsencrypt.le_util.subprocess.Popen") + def test_failure(self, mock_popen): + mock_popen().communicate.return_value = ("", "") + mock_popen().returncode = 1 + + self.assertRaises(errors.SubprocessError, self._call, ["test"]) + + +class ExeExistsTest(unittest.TestCase): + """Tests for letsencrypt.le_util.exe_exists.""" + + @classmethod + def _call(cls, exe): + from letsencrypt.le_util import exe_exists + return exe_exists(exe) + + @mock.patch("letsencrypt.le_util.os.path.isfile") + @mock.patch("letsencrypt.le_util.os.access") + def test_full_path(self, mock_access, mock_isfile): + mock_access.return_value = True + mock_isfile.return_value = True + self.assertTrue(self._call("/path/to/exe")) + + @mock.patch("letsencrypt.le_util.os.path.isfile") + @mock.patch("letsencrypt.le_util.os.access") + def test_on_path(self, mock_access, mock_isfile): + mock_access.return_value = True + mock_isfile.return_value = True + self.assertTrue(self._call("exe")) + + @mock.patch("letsencrypt.le_util.os.path.isfile") + @mock.patch("letsencrypt.le_util.os.access") + def test_not_found(self, mock_access, mock_isfile): + mock_access.return_value = False + mock_isfile.return_value = True + self.assertFalse(self._call("exe")) + + class MakeOrVerifyDirTest(unittest.TestCase): """Tests for letsencrypt.le_util.make_or_verify_dir. diff --git a/letsencrypt/tests/reverter_test.py b/letsencrypt/tests/reverter_test.py index da57bf8dc..d568d2aef 100644 --- a/letsencrypt/tests/reverter_test.py +++ b/letsencrypt/tests/reverter_test.py @@ -1,4 +1,6 @@ """Test letsencrypt.reverter.""" +import csv +import itertools import logging import os import shutil @@ -11,7 +13,7 @@ from letsencrypt import errors class ReverterCheckpointLocalTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, too-many-public-methods """Test the Reverter Class.""" def setUp(self): from letsencrypt.reverter import Reverter @@ -126,6 +128,42 @@ class ReverterCheckpointLocalTest(unittest.TestCase): errors.ReverterError, self.reverter.register_file_creation, "filepath") + def test_register_undo_command(self): + coms = [ + ["a2dismod", "ssl"], + ["a2dismod", "rewrite"], + ["cleanslate"] + ] + for com in coms: + self.reverter.register_undo_command(True, com) + + act_coms = get_undo_commands(self.config.temp_checkpoint_dir) + + for a_com, com in itertools.izip(act_coms, coms): + self.assertEqual(a_com, com) + + def test_bad_register_undo_command(self): + m_open = mock.mock_open() + with mock.patch("letsencrypt.reverter.open", m_open, create=True): + m_open.side_effect = OSError("bad open") + self.assertRaises( + errors.ReverterError, self.reverter.register_undo_command, + True, ["command"]) + + @mock.patch("letsencrypt.le_util.run_script") + def test_run_undo_commands(self, mock_run): + mock_run.side_effect = ["", errors.SubprocessError] + coms = [ + ["invalid_command"], + ["a2dismod", "ssl"], + ] + for com in coms: + self.reverter.register_undo_command(True, com) + + self.reverter.revert_temporary_config() + + self.assertEqual(mock_run.call_count, 2) + def test_recovery_routine_in_progress_failure(self): self.reverter.add_to_checkpoint(self.sets[0], "perm save") @@ -390,9 +428,9 @@ def setup_test_files(): dir2 = tempfile.mkdtemp("dir2") config1 = os.path.join(dir1, "config.txt") config2 = os.path.join(dir2, "config.txt") - with open(config1, 'w') as file_fd: + with open(config1, "w") as file_fd: file_fd.write("directive-dir1") - with open(config2, 'w') as file_fd: + with open(config2, "w") as file_fd: file_fd.write("directive-dir2") sets = [set([config1]), @@ -404,30 +442,35 @@ def setup_test_files(): def get_save_notes(dire): """Read save notes""" - return read_in(os.path.join(dire, 'CHANGES_SINCE')) + return read_in(os.path.join(dire, "CHANGES_SINCE")) def get_filepaths(dire): """Get Filepaths""" - return read_in(os.path.join(dire, 'FILEPATHS')) + return read_in(os.path.join(dire, "FILEPATHS")) def get_new_files(dire): """Get new files.""" - return read_in(os.path.join(dire, 'NEW_FILES')).splitlines() + return read_in(os.path.join(dire, "NEW_FILES")).splitlines() + + +def get_undo_commands(dire): + """Get new files.""" + return csv.reader(open(os.path.join(dire, "COMMANDS"))) def read_in(path): """Read in a file, return the str""" - with open(path, 'r') as file_fd: + with open(path, "r") as file_fd: return file_fd.read() def update_file(filename, string): """Update a file with a new value.""" - with open(filename, 'w') as file_fd: + with open(filename, "w") as file_fd: file_fd.write(string) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() # pragma: no cover From 7390a39a4d093c611fd9b33da7995d540b7790a2 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 29 Jul 2015 23:49:02 -0700 Subject: [PATCH 40/52] edit spacing --- letsencrypt/le_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py index af8a56ef5..f8c911d99 100644 --- a/letsencrypt/le_util.py +++ b/letsencrypt/le_util.py @@ -69,6 +69,7 @@ def exe_exists(exe): return False + def make_or_verify_dir(directory, mode=0o755, uid=0): """Make sure directory exists with proper permissions. From 15708733120b3984c7f60eaa4d63030c875098f4 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 00:29:23 -0700 Subject: [PATCH 41/52] Fix single candidate plugin misconfiguration bug --- letsencrypt/display/ops.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index afb7d6688..a566031a1 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -82,6 +82,11 @@ def pick_plugin(config, default, plugins, question, ifaces): elif len(prepared) == 1: plugin_ep = prepared.values()[0] logger.debug("Single candidate plugin: %s", plugin_ep) + if plugin_ep.misconfigured: + logger.warning( + "Only candidate plugin, %s, is misconfigured." + "Please fix the configuration before proceeding.", plugin_ep) + return None return plugin_ep.init() else: logger.debug("No candidate plugin") @@ -90,7 +95,7 @@ def pick_plugin(config, default, plugins, question, ifaces): def pick_authenticator( config, default, plugins, question="How would you " - "like to authenticate with Let's Encrypt CA?"): + "like to authenticate with the Let's Encrypt CA?"): """Pick authentication plugin.""" return pick_plugin( config, default, plugins, question, (interfaces.IAuthenticator,)) From e64e3ceab0f4718d6e215e1710133e886ac67922 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 01:19:02 -0700 Subject: [PATCH 42/52] Proper misconfiguration message --- letsencrypt-apache/letsencrypt_apache/configurator.py | 4 ++-- letsencrypt/display/ops.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 7b1144e2e..f2c43f8bf 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -1009,8 +1009,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: le_util.run_script([self.conf("ctl"), "configtest"]) - except errors.SubprocessError: - raise errors.MisconfigurationError("Config Test failed!") + except errors.SubprocessError as err: + raise errors.MisconfigurationError(str(err)) def get_version(self): """Return version of Apache Server. diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index a566031a1..de5af2e0d 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -84,8 +84,9 @@ def pick_plugin(config, default, plugins, question, ifaces): logger.debug("Single candidate plugin: %s", plugin_ep) if plugin_ep.misconfigured: logger.warning( - "Only candidate plugin, %s, is misconfigured." - "Please fix the configuration before proceeding.", plugin_ep) + "Only candidate plugin, %s, is misconfigured. " + "Please fix the configuration before proceeding.", + plugin_ep.name) return None return plugin_ep.init() else: From 3bcf29be5148506e345f5e1627c16438483b291c Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 01:22:59 -0700 Subject: [PATCH 43/52] Change default rollback to 1 --- letsencrypt/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index 07d1965fb..d0c2a3af4 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -18,7 +18,7 @@ CLI_DEFAULTS = dict( verbose_count=-(logging.WARNING / 10), server="https://www.letsencrypt-demo.org/acme/new-reg", rsa_key_size=2048, - rollback_checkpoints=0, + rollback_checkpoints=1, config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", From 47be104e2b49036d7df9b58fb9abe8dca9d9d722 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 01:37:05 -0700 Subject: [PATCH 44/52] Update pick_plugin tests based on misconfigured single plugin --- letsencrypt/tests/display/ops_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/letsencrypt/tests/display/ops_test.py b/letsencrypt/tests/display/ops_test.py index 3a0c627ce..fc4013bed 100644 --- a/letsencrypt/tests/display/ops_test.py +++ b/letsencrypt/tests/display/ops_test.py @@ -92,9 +92,19 @@ class PickPluginTest(unittest.TestCase): def test_single(self): plugin_ep = mock.MagicMock() plugin_ep.init.return_value = "foo" + plugin_ep.misconfigured = False + self.reg.ifaces().verify().available.return_value = {"bar": plugin_ep} self.assertEqual("foo", self._call()) + def test_single_misconfigured(self): + plugin_ep = mock.MagicMock() + plugin_ep.init.return_value = "foo" + plugin_ep.misconfigured = True + + self.reg.ifaces().verify().available.return_value = {"bar": plugin_ep} + self.assertTrue(self._call() is None) + def test_multiple(self): plugin_ep = mock.MagicMock() plugin_ep.init.return_value = "foo" From b37fc95386fe3c0734b4c179eda25a117925f344 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 01:49:04 -0700 Subject: [PATCH 45/52] py26 compat --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index f2c43f8bf..ce52b8467 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -678,7 +678,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.choose_vhost(domain), options) except KeyError: raise errors.PluginError( - "Unsupported enhancement: {}".format(enhancement)) + "Unsupported enhancement: {0}".format(enhancement)) except errors.PluginError: logger.warn("Failed %s for %s", enhancement, domain) raise From f71119681cfc73a84459ee68051108e298d0154d Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 30 Jul 2015 23:14:58 -0700 Subject: [PATCH 46/52] Address most of first round of comments --- .../letsencrypt_apache/configurator.py | 138 +++++++++++------- .../letsencrypt_apache/parser.py | 1 - .../tests/configurator_test.py | 29 +++- .../letsencrypt_nginx/configurator.py | 11 +- letsencrypt/display/ops.py | 4 - letsencrypt/plugins/common.py | 7 + letsencrypt/tests/reverter_test.py | 3 +- 7 files changed, 125 insertions(+), 68 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index ce52b8467..7cda8066c 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -17,6 +17,8 @@ from letsencrypt import errors from letsencrypt import interfaces from letsencrypt import le_util +from letsencrypt.plugins import common + from letsencrypt_apache import augeas_configurator from letsencrypt_apache import constants from letsencrypt_apache import display_ops @@ -48,6 +50,15 @@ logger = logging.getLogger(__name__) # transactional due to the use of register_file_creation() +# TODO: Verify permissions on configuration root... it is easier than +# checking permissions on each of the relative directories and less error +# prone. +# TODO: Write a server protocol finder. Listen or +# Protocol . This can verify partial setups are correct +# TODO: Add directives to sites-enabled... not sites-available. +# sites-available doesn't allow immediate find_dir search even with save() +# and load() + class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # pylint: disable=too-many-instance-attributes,too-many-public-methods """Apache configurator. @@ -55,15 +66,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): State of Configurator: This code has been been tested and built for Ubuntu 14.04 Apache 2.4 and it works for Ubuntu 12.04 Apache 2.2 - .. todo:: Verify permissions on configuration root... it is easier than - checking permissions on each of the relative directories and less error - prone. - .. todo:: Write a server protocol finder. Listen or - Protocol . This can verify partial setups are correct - .. todo:: Add directives to sites-enabled... not sites-available. - sites-available doesn't allow immediate find_dir search even with save() - and load() - :ivar config: Configuration. :type config: :class:`~letsencrypt.interfaces.IConfig` @@ -82,14 +84,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): description = "Apache Web Server - Alpha" - # Kept in same function to avoid multiple compilations of the regex - - private_ips_regex = re.compile( - r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" - r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") - hostname_regex = re.compile( - r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$", re.IGNORECASE) - @classmethod def add_parser_arguments(cls, add): @@ -136,7 +130,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST) def prepare(self): - """Prepare the authenticator/installer.""" + """Prepare the authenticator/installer. + + :raises .errors.NoInstallationError: If Apache configs cannot be found + :raises .errors.MisconfigurationError: If Apache is misconfigured + :raises .errors.NotSupportedError: If Apache version is not supported + :raises .errors.PluginError: If there is any other error + + """ # Make sure configuration is valid self.config_test() @@ -170,6 +171,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): .. todo:: Might be nice to remove chain directive if none exists This shouldn't happen within letsencrypt though + :raises errors.PluginError: When unable to deploy certificate due to + a lack of directives + """ vhost = self.choose_vhost(domain) @@ -235,7 +239,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :returns: ssl vhost associated with name :rtype: :class:`~letsencrypt_apache.obj.VirtualHost` - :raises .errors.PluginError: If no vhost is available + :raises .errors.PluginError: If no vhost is available or chosen """ # Allows for domain names to be associated with a virtual host @@ -251,21 +255,34 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc[target_name] = vhost return vhost + return self._choose_vhost_from_list(target_name) + + def _choose_vhost_from_list(self, target_name): # Select a vhost from a list vhost = display_ops.select_vhost(target_name, self.vhosts) - if vhost is not None: - self.assoc[target_name] = vhost - else: + if vhost is None: logger.error( "No vhost exists with servername or alias of: %s. " "No vhost was selected. Please specify servernames " "in the Apache config", target_name) raise errors.PluginError("No vhost selected") - # TODO: Ask the user if they would like to add ServerName/Alias to VH - + if not vhost.ssl: + addrs = self._get_proposed_addrs(vhost, "443") + # TODO: Conflicts is too conservative + if not any(vhost.enabled and vhost.conflicts(addrs) for vhost in self.vhosts): + vhost = self.make_vhost_ssl(vhost) + self.assoc[target_name] = vhost + else: + logger.error( + "The selected vhost would conflict with other HTTPS " + "VirtualHosts within Apache. Please select another " + "vhost or add ServerNames to your configuration.") + raise errors.PluginError( + "VirtualHost not able to be selected.") return vhost + def _find_best_vhost(self, target_name): """Finds the best vhost for a target_name. @@ -328,7 +345,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): all_names.update(vhost.get_names()) for addr in vhost.addrs: - if ApacheConfigurator.hostname_regex.match(addr.get_addr()): + if common.hostname_regex.match(addr.get_addr()): all_names.add(addr.get_addr()) else: name = self.get_name_from_ip(addr) @@ -348,7 +365,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # If it isn't a private IP, do a reverse DNS lookup - if not ApacheConfigurator.private_ips_regex.match(addr.get_addr()): + if not common.private_ips_regex.match(addr.get_addr()): try: socket.inet_aton(addr.get_addr()) return socket.gethostbyaddr(addr.get_addr())[0] @@ -366,9 +383,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Take the final ServerName as each overrides the previous servername_match = self.parser.find_dir( - "ServerName", None, host.path, exclude=False) + "ServerName", None, start=host.path, exclude=False) serveralias_match = self.parser.find_dir( - "ServerAlias", None, host.path, exclude=False) + "ServerAlias", None, start=host.path, exclude=False) for alias in serveralias_match: host.aliases.add(self.parser.get_arg(alias)) @@ -392,7 +409,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): addrs.add(obj.Addr.fromstring(self.parser.get_arg(arg))) is_ssl = False - if self.parser.find_dir("SSLEngine", "on", path, exclude=False): + if self.parser.find_dir("SSLEngine", "on", start=path, exclude=False): is_ssl = True filename = get_file_path(path) @@ -430,8 +447,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): now NameVirtualHosts. If version is earlier than 2.4, check if addr has a NameVirtualHost directive in the Apache config - :param target_addr: vhost address - :type target_addr: :class:~letsencrypt_apache.obj.Addr + :param letsencrypt_apache.obj.Addr target_addr: vhost address :returns: Success :rtype: bool @@ -449,7 +465,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """Adds NameVirtualHost directive for given address. :param addr: Address that will be added as NameVirtualHost directive - :type addr: :class:~letsencrypt_apache.obj.Addr + :type addr: :class:`~letsencrypt_apache.obj.Addr` """ loc = parser.get_aug_path(self.parser.loc["name"]) @@ -672,13 +688,17 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): See :const:`~letsencrypt.constants.ENHANCEMENTS` documentation for appropriate parameter. + :raises .errors.PluginError: If Enhancement is not supported, or if + there is any other problem with the enhancement. + """ try: - return self._enhance_func[enhancement]( - self.choose_vhost(domain), options) + func = self._enhance_func[enhancement] except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + try: + func(self.choose_vhost(domain), options) except errors.PluginError: logger.warn("Failed %s for %s", enhancement, domain) raise @@ -705,6 +725,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :returns: Success, general_vhost (HTTP vhost) :rtype: (bool, :class:`~letsencrypt_apache.obj.VirtualHost`) + :raises .errors.PluginError: If no viable HTTP host can be created or + used for the redirect. + """ if "rewrite_module" not in self.parser.modules: self.enable_mod("rewrite") @@ -714,7 +737,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add virtual_server with redirect logger.debug("Did not find http version of ssl virtual host " "attempting to create") - redirect_addrs = self._get_redirect_addrs(ssl_vhost) + redirect_addrs = self._get_proposed_addrs(ssl_vhost) for vhost in self.vhosts: if vhost.enabled and vhost.conflicts(redirect_addrs): raise errors.PluginError( @@ -752,8 +775,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :raises errors.PluginError: When another redirection exists """ - rewrite_path = self.parser.find_dir("RewriteRule", None, vhost.path) - redirect_path = self.parser.find_dir("Redirect", None, vhost.path) + rewrite_path = self.parser.find_dir( + "RewriteRule", None, start=vhost.path) + redirect_path = self.parser.find_dir("Redirect", None, start=vhost.path) if redirect_path: # "Existing Redirect directive for virtualhost" @@ -800,7 +824,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): serveralias = "" servername = "" - if ssl_vhost.name is not None: servername = "ServerName " + ssl_vhost.name if ssl_vhost.aliases: @@ -817,7 +840,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ErrorLog /var/log/apache2/redirect.error.log\n" "LogLevel warn\n" "\n" - % (" ".join(str(addr) for addr in self._get_redirect_addrs(ssl_vhost)), + % (" ".join(str(addr) for addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(constants.REWRITE_HTTPS_ARGS))) @@ -860,10 +883,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return None - def _get_redirect_addrs(self, ssl_vhost): # pylint: disable=no-self-use + def _get_proposed_addrs(self, vhost, port="80"): # pylint: disable=no-self-use + """Return all addrs of vhost with the port replaced with the specified. + + :param obj.VirtualHost ssl_vhost: Original Vhost + :param str port: Desired port for new addresses + + :returns: `set` of :class:`~obj.Addr` + + """ redirects = set() - for addr in ssl_vhost.addrs: - redirects.add(addr.get_addr_obj("80")) + for addr in vhost.addrs: + redirects.add(addr.get_addr_obj(port)) return redirects @@ -884,9 +915,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for vhost in self.vhosts: if vhost.ssl: cert_path = self.parser.find_dir( - "SSLCertificateFile", None, vhost.path, exclude=False) + "SSLCertificateFile", None, + start=vhost.path, exclude=False) key_path = self.parser.find_dir( - "SSLCertificateKeyFile", None, vhost.path, exclude=False) + "SSLCertificateKeyFile", None, + start=vhost.path, exclude=False) if cert_path and key_path: cert = os.path.abspath(self.parser.get_arg(cert_path[-1])) @@ -924,6 +957,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param vhost: vhost to enable :type vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + :raises .errors.NotSupportedError: If filesystem layout is not + supported. + """ if self.is_site_enabled(vhost.filep): return @@ -943,7 +979,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.info("Enabling available site: %s", vhost.filep) self.save_notes += "Enabled site %s\n" % vhost.filep else: - raise errors.MisconfigurationError( + raise errors.NotSupportedError( "Unsupported filesystem layout. " "sites-available/enabled expected.") @@ -955,6 +991,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str mod_name: Name of the module to enable. (e.g. 'ssl') :param bool temp: Whether or not this is a temporary action. + :raises .errors.NotSupportedError: If the filesystem layout is not + supported. + :raises .errors.MisconfigurationError: If a2enmod or a2dismod cannot be + run. + """ # Support Debian specific setup if (not os.path.isdir(os.path.join(self.parser.root, "mods-available")) @@ -995,8 +1036,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): .. todo:: This function will be converted to using reload - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If unable to restart due to a + configuration problem, or if the restart subprocess cannot be run. """ return apache_restart(self.conf("init-script")) @@ -1112,9 +1153,8 @@ def apache_restart(apache_init_script): need to be moved into the class again. Perhaps this version can live on... for testing purposes. - :raises .errors.PluginError: If unable to restart with apache_init_script :raises .errors.MisconfigurationError: If unable to restart due to a - configuration problem. + configuration problem, or if the restart subprocess cannot be run. """ try: diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 01ec4aa38..e14569abc 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -240,7 +240,6 @@ class ApacheParser(object): Directives should be in the form of a case insensitive regex currently .. todo:: arg should probably be a list - .. todo:: Check //* notation for including directories Note: Augeas is inherently case sensitive while Apache is case insensitive. Augeas 1.0 allows case insensitive regexes like diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 46073619a..8c59147a3 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -110,10 +110,31 @@ class TwoVhost80Test(util.ApacheTest): errors.PluginError, self.config.choose_vhost, "none.com") @mock.patch("letsencrypt_apache.display_ops.select_vhost") - def test_choose_vhost_select_vhost(self, mock_select): - mock_select.return_value = self.vh_truth[3] + def test_choose_vhost_select_vhost_ssl(self, mock_select): + mock_select.return_value = self.vh_truth[1] self.assertEqual( - self.vh_truth[3], self.config.choose_vhost("none.com")) + self.vh_truth[1], self.config.choose_vhost("none.com")) + + @mock.patch("letsencrypt_apache.display_ops.select_vhost") + def test_choose_vhost_select_vhost_non_ssl(self, mock_select): + mock_select.return_value = self.vh_truth[0] + chosen_vhost = self.config.choose_vhost("none.com") + self.assertEqual( + self.vh_truth[0].get_names(), chosen_vhost.get_names()) + + # Make sure we go from HTTP -> HTTPS + self.assertFalse(self.vh_truth[0].ssl) + self.assertTrue(chosen_vhost.ssl) + + @mock.patch("letsencrypt_apache.display_ops.select_vhost") + def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): + mock_select.return_value = self.vh_truth[3] + conflicting_vhost = obj.VirtualHost( + "path", "aug_path", set([obj.Addr.fromstring("*:443")]), True, True) + self.config.vhosts.append(conflicting_vhost) + + self.assertRaises( + errors.PluginError, self.config.choose_vhost, "none.com") def test_find_best_vhost(self): # pylint: disable=protected-access @@ -207,7 +228,7 @@ class TwoVhost80Test(util.ApacheTest): def test_enable_site_failure(self): self.assertRaises( - errors.MisconfigurationError, + errors.NotSupportedError, self.config.enable_site, obj.VirtualHost("asdf", "afsaf", set(), False, False)) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 18b5d719a..f759dc7c1 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -244,22 +244,15 @@ class NginxConfigurator(common.Plugin): """ all_names = set() - # Kept in same function to avoid multiple compilations of the regex - priv_ip_regex = (r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" - r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") - private_ips = re.compile(priv_ip_regex) - hostname_regex = r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$" - hostnames = re.compile(hostname_regex, re.IGNORECASE) - for vhost in self.parser.get_vhosts(): all_names.update(vhost.names) for addr in vhost.addrs: host = addr.get_addr() - if hostnames.match(host): + if common.hostname_regex.match(host): # If it's a hostname, add it to the names. all_names.add(host) - elif not private_ips.match(host): + elif not common.private_ips_regex.match(host): # If it isn't a private IP, do a reverse DNS lookup # TODO: IPv6 support try: diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index de5af2e0d..a220d07d9 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -83,10 +83,6 @@ def pick_plugin(config, default, plugins, question, ifaces): plugin_ep = prepared.values()[0] logger.debug("Single candidate plugin: %s", plugin_ep) if plugin_ep.misconfigured: - logger.warning( - "Only candidate plugin, %s, is misconfigured. " - "Please fix the configuration before proceeding.", - plugin_ep.name) return None return plugin_ep.init() else: diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index 90daa685f..3e7596c1f 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -1,6 +1,7 @@ """Plugin common functions.""" import os import pkg_resources +import re import shutil import tempfile @@ -22,6 +23,12 @@ def dest_namespace(name): """ArgumentParser dest namespace (prefix of all destinations).""" return name + "_" +private_ips_regex = re.compile( # pylint: disable=invalid-name + r"(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|" + r"(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)") +hostname_regex = re.compile( # pylint: disable=invalid-name + r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*[a-z]+$", re.IGNORECASE) + class Plugin(object): """Generic plugin.""" diff --git a/letsencrypt/tests/reverter_test.py b/letsencrypt/tests/reverter_test.py index d568d2aef..59a7e4d9a 100644 --- a/letsencrypt/tests/reverter_test.py +++ b/letsencrypt/tests/reverter_test.py @@ -457,7 +457,8 @@ def get_new_files(dire): def get_undo_commands(dire): """Get new files.""" - return csv.reader(open(os.path.join(dire, "COMMANDS"))) + with open(os.path.join(dire, "COMMANDS")) as csvfile: + return list(csv.reader(csvfile)) def read_in(path): From 92dbc4f4eb3560b735e8f7668ef4d3a3de8302e3 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 09:55:02 -0700 Subject: [PATCH 47/52] Finish merge of #17 --- .../letsencrypt_apache/constants.py | 2 +- letsencrypt/client/CONFIG.py | 59 ------------------- 2 files changed, 1 insertion(+), 60 deletions(-) delete mode 100644 letsencrypt/client/CONFIG.py diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index cb75276b2..7e7e127f5 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -20,5 +20,5 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename( distribution.""" REWRITE_HTTPS_ARGS = [ - "^.*$", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] """Apache rewrite rule arguments used for redirections to https vhost""" diff --git a/letsencrypt/client/CONFIG.py b/letsencrypt/client/CONFIG.py deleted file mode 100644 index 910e5c881..000000000 --- a/letsencrypt/client/CONFIG.py +++ /dev/null @@ -1,59 +0,0 @@ -# CA hostname -# If you create your own server... change this line -# Note: the server certificate must be trusted in order to avoid -# further modifications to the client -ACME_SERVER = "letsencrypt-demo.org" -# Apache server root directory -SERVER_ROOT = "/etc/apache2/" -# Configuration file directory for letsencrypt -CONFIG_DIR = "/etc/letsencrypt/" -# Working directory for letsencrypt -WORK_DIR = "/var/lib/letsencrypt/" -# Directory where configuration backups are stored -BACKUP_DIR = WORK_DIR + "backups/" -# Replaces MODIFIED_FILES, directory where temp checkpoint is created -TEMP_CHECKPOINT_DIR = WORK_DIR + "temp_checkpoint/" -# Directory used before a permanent checkpoint is finalized -IN_PROGRESS_DIR = BACKUP_DIR + "IN_PROGRESS/" -# Directory where all certificates/keys are stored - used for easy revocation -CERT_KEY_BACKUP = WORK_DIR + "keys-certs/" -# Where all keys should be stored -KEY_DIR = SERVER_ROOT + "ssl/" -# Certificate storage -CERT_DIR = SERVER_ROOT + "certs/" - -# Used by openssl to sign challenge certificate with letsencrypt extension -# No longer used -#CHOC_CERT_CONF = CONFIG_DIR + "choc_cert_extensions.cnf" -# Contains standard Apache SSL directives -OPTIONS_SSL_CONF = CONFIG_DIR + "options-ssl.conf" -# Let's Encrypt SSL vhost configuration extension -LE_VHOST_EXT = "-le-ssl.conf" -# Temporary file for challenge virtual hosts -APACHE_CHALLENGE_CONF = CONFIG_DIR + "le_dvsni_cert_challenge.conf" - -# Byte size of S and Nonce -S_SIZE = 32 -NONCE_SIZE = 16 - -# Key Sizes -RSA_KEY_SIZE = 2048 - -# bits of hashcash to generate -difficulty = 23 - -# Let's Encrypt cert and chain files -CERT_PATH = CERT_DIR + "cert-letsencrypt.pem" -CHAIN_PATH = CERT_DIR + "chain-letsencrypt.pem" - -#Invalid Extension -INVALID_EXT = ".acme.invalid" - -# Challenge Preferences Dict for currently supported challenges -CHALLENGE_PREFERENCES = ["dvsni", "recoveryToken"] - -# Mutually Exclusive Challenges - only solve 1 -EXCLUSIVE_CHALLENGES = [set(["dvsni", "simpleHttps"])] - -# Rewrite rule arguments used for redirections to https vhost -REWRITE_HTTPS_ARGS = ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] From 06a014de09ded69f89ccbd5da235480899d7e110 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 10:29:04 -0700 Subject: [PATCH 48/52] Update documentation, reduce spurious logging --- .../letsencrypt_apache/augeas_configurator.py | 26 ++++++++++++++++--- letsencrypt/reverter.py | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 91b78c566..4b61b9cc4 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -45,6 +45,9 @@ class AugeasConfigurator(common.Plugin): :param str lens: lens to check for errors + :raises .errors.PluginError: If there has been an error in parsing with + the specified lens. + """ error_files = self.aug.match("/augeas//error") @@ -58,7 +61,6 @@ class AugeasConfigurator(common.Plugin): "There has been an error in parsing the file (%s): %s", # Strip off /augeas/files and /error path[13:len(path) - 6], self.aug.get(path + "/message")) - logger.error(msg) raise errors.PluginError(msg) def save(self, title=None, temporary=False): @@ -75,6 +77,10 @@ class AugeasConfigurator(common.Plugin): :param bool temporary: Indicates whether the changes made will be quickly reversed in the future (ie. challenges) + :raises .errors.PluginError: If there was an error in Augeas, in an + attempt to save the configuration. + :raises .errors.ReverterError: If unable to create the checkpoint + """ save_state = self.aug.get("/augeas/save") self.aug.set("/augeas/save", "noop") @@ -138,13 +144,19 @@ class AugeasConfigurator(common.Plugin): Reverts all modified files that have not been saved as a checkpoint + :raises .errors.ReverterError: If unable to recover the configuration + """ self.reverter.recovery_routine() # Need to reload configuration after these changes take effect self.aug.load() def revert_challenge_config(self): - """Used to cleanup challenge configurations.""" + """Used to cleanup challenge configurations. + + :raises .errors.ReverterError: If unable to revert the challenge config. + + """ self.reverter.revert_temporary_config() self.aug.load() @@ -153,10 +165,18 @@ class AugeasConfigurator(common.Plugin): :param int rollback: Number of checkpoints to revert + :raises .errors.ReverterError: If there is a problem with the input or + the function is unable to correctly revert the configuration + """ self.reverter.rollback_checkpoints(rollback) self.aug.load() def view_config_changes(self): - """Show all of the configuration changes that have taken place.""" + """Show all of the configuration changes that have taken place. + + :raises .errors.ReverterError: If there is a problem while processing + the checkpoints directories. + + """ self.reverter.view_config_changes() diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index 03d62ce13..bddf52b8d 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -398,6 +398,8 @@ class Reverter(object): finalized. This is useful to protect against crashes and other execution interruptions. + :raises .errors.ReverterError: If unable to recover the configuration + """ # First, any changes found in IConfig.temp_checkpoint_dir are removed, # then IN_PROGRESS changes are removed The order is important. From 33111e1ce6149ab0940a024491b8b9e449fec9aa Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 10:44:21 -0700 Subject: [PATCH 49/52] Added additional errors to IPlugin.prepare interface --- .../letsencrypt_nginx/configurator.py | 2 +- .../tests/configurator_test.py | 2 +- letsencrypt/interfaces.py | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index f759dc7c1..2899e1f76 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -444,7 +444,7 @@ class NginxConfigurator(common.Plugin): # nginx < 0.8.48 uses machine hostname as default server_name instead of # the empty string if nginx_version < (0, 8, 48): - raise errors.PluginError("Nginx version must be 0.8.48+") + raise errors.NotSupportedError("Nginx version must be 0.8.48+") return nginx_version diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index d8f82bd05..3703a8201 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -249,7 +249,7 @@ class NginxConfiguratorTest(util.NginxTest): " (based on LLVM 3.5svn)", "TLS SNI support enabled", "configure arguments: --with-http_ssl_module"])) - self.assertRaises(errors.PluginError, self.config.get_version) + self.assertRaises(errors.NotSupportedError, self.config.get_version) mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.PluginError, self.config.get_version) diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index b07e64894..3a0732b64 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -102,14 +102,19 @@ class IPlugin(zope.interface.Interface): def prepare(): """Prepare the plugin. - Finish up any additional initialization. + Finish up any additional initialization. - :raises .MisconfigurationError: - when full initialization cannot be completed. Plugin will - be displayed on a list of available plugins. - :raises .NoInstallationError: - when the necessary programs/files cannot be located. Plugin - will NOT be displayed on a list of available plugins. + :raises .PluginError: + when full initialization cannot be completed. + :raises .MisconfigurationError: + when full initialization cannot be completed. Plugin will + be displayed on a list of available plugins. + :raises .NoInstallationError: + when the necessary programs/files cannot be located. Plugin + will NOT be displayed on a list of available plugins. + :raises .NotSupportedError: + when the installation is recognized, but the version is not + currently supported. """ From 65f4332798a8a32c60ff55817454264a49565187 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 12:49:39 -0700 Subject: [PATCH 50/52] Update IPlugin docs, make augeas conform --- .../letsencrypt_apache/augeas_configurator.py | 56 ++++++++++++------- .../tests/augeas_configurator_test.py | 38 +++++++++++++ letsencrypt/interfaces.py | 35 ++++++++++-- letsencrypt/reverter.py | 2 + 4 files changed, 107 insertions(+), 24 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 4b61b9cc4..7557a27c6 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -38,7 +38,7 @@ class AugeasConfigurator(common.Plugin): # because this will change the underlying configuration and potential # vhosts self.reverter = reverter.Reverter(self.config) - self.reverter.recovery_routine() + self.recovery_routine() def check_parsing_errors(self, lens): """Verify Augeas can parse all of the lens files. @@ -63,6 +63,7 @@ class AugeasConfigurator(common.Plugin): path[13:len(path) - 6], self.aug.get(path + "/message")) raise errors.PluginError(msg) + # TODO: Cleanup this function def save(self, title=None, temporary=False): """Saves all changes to the configuration files. @@ -78,8 +79,7 @@ class AugeasConfigurator(common.Plugin): be quickly reversed in the future (ie. challenges) :raises .errors.PluginError: If there was an error in Augeas, in an - attempt to save the configuration. - :raises .errors.ReverterError: If unable to create the checkpoint + attempt to save the configuration, or an error creating a checkpoint """ save_state = self.aug.get("/augeas/save") @@ -108,22 +108,26 @@ class AugeasConfigurator(common.Plugin): for path in save_paths: save_files.add(self.aug.get(path)[6:]) - # Create Checkpoint - if temporary: - self.reverter.add_to_temp_checkpoint( - save_files, self.save_notes) - else: - self.reverter.add_to_checkpoint(save_files, self.save_notes) + try: + # Create Checkpoint + if temporary: + self.reverter.add_to_temp_checkpoint( + save_files, self.save_notes) + else: + self.reverter.add_to_checkpoint(save_files, self.save_notes) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) if title and not temporary: - self.reverter.finalize_checkpoint(title) + try: + self.reverter.finalize_checkpoint(title) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.aug.set("/augeas/save", save_state) self.save_notes = "" self.aug.save() - return True - def _log_save_errors(self, ex_errs): """Log errors due to bad Augeas save. @@ -144,20 +148,26 @@ class AugeasConfigurator(common.Plugin): Reverts all modified files that have not been saved as a checkpoint - :raises .errors.ReverterError: If unable to recover the configuration + :raises .errors.PluginError: If unable to recover the configuration """ - self.reverter.recovery_routine() + try: + self.reverter.recovery_routine() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) # Need to reload configuration after these changes take effect self.aug.load() def revert_challenge_config(self): """Used to cleanup challenge configurations. - :raises .errors.ReverterError: If unable to revert the challenge config. + :raises .errors.PluginError: If unable to revert the challenge config. """ - self.reverter.revert_temporary_config() + try: + self.reverter.revert_temporary_config() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.aug.load() def rollback_checkpoints(self, rollback=1): @@ -165,18 +175,24 @@ class AugeasConfigurator(common.Plugin): :param int rollback: Number of checkpoints to revert - :raises .errors.ReverterError: If there is a problem with the input or + :raises .errors.PluginError: If there is a problem with the input or the function is unable to correctly revert the configuration """ - self.reverter.rollback_checkpoints(rollback) + try: + self.reverter.rollback_checkpoints(rollback) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.aug.load() def view_config_changes(self): """Show all of the configuration changes that have taken place. - :raises .errors.ReverterError: If there is a problem while processing + :raises .errors.PluginError: If there is a problem while processing the checkpoints directories. """ - self.reverter.view_config_changes() + try: + self.reverter.view_config_changes() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) diff --git a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py index 8cb1fb3a8..815e6fc44 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py @@ -41,6 +41,20 @@ class AugeasConfiguratorTest(util.ApacheTest): self.assertRaises(errors.PluginError, self.config.save) + def test_bad_save_checkpoint(self): + self.config.reverter.add_to_checkpoint = mock.Mock( + side_effect=errors.ReverterError) + self.config.parser.add_dir( + self.vh_truth[0].path, "Test", "bad_save_ckpt") + self.assertRaises(errors.PluginError, self.config.save) + + def test_bad_save_finalize_checkpoint(self): + self.config.reverter.finalize_checkpoint = mock.Mock( + side_effect=errors.ReverterError) + self.config.parser.add_dir( + self.vh_truth[0].path, "Test", "bad_save_ckpt") + self.assertRaises(errors.PluginError, self.config.save, "Title") + def test_finalize_save(self): mock_finalize = mock.Mock() self.config.reverter = mock_finalize @@ -55,6 +69,13 @@ class AugeasConfiguratorTest(util.ApacheTest): self.config.recovery_routine() self.assertEqual(mock_load.call_count, 1) + def test_recovery_routine_error(self): + self.config.reverter.recovery_routine = mock.Mock( + side_effect=errors.ReverterError) + + self.assertRaises( + errors.PluginError, self.config.recovery_routine) + def test_revert_challenge_config(self): mock_load = mock.Mock() self.config.aug.load = mock_load @@ -62,6 +83,13 @@ class AugeasConfiguratorTest(util.ApacheTest): self.config.revert_challenge_config() self.assertEqual(mock_load.call_count, 1) + def test_revert_challenge_config_error(self): + self.config.reverter.revert_temporary_config = mock.Mock( + side_effect=errors.ReverterError) + + self.assertRaises( + errors.PluginError, self.config.revert_challenge_config) + def test_rollback_checkpoints(self): mock_load = mock.Mock() self.config.aug.load = mock_load @@ -69,9 +97,19 @@ class AugeasConfiguratorTest(util.ApacheTest): self.config.rollback_checkpoints() self.assertEqual(mock_load.call_count, 1) + def test_rollback_error(self): + self.config.reverter.rollback_checkpoints = mock.Mock( + side_effect=errors.ReverterError) + self.assertRaises(errors.PluginError, self.config.rollback_checkpoints) + def test_view_config_changes(self): self.config.view_config_changes() + def test_view_config_changes_error(self): + self.config.reverter.view_config_changes = mock.Mock( + side_effect=errors.ReverterError) + self.assertRaises(errors.PluginError, self.config.view_config_changes) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index 3a0732b64..3cb7270b4 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -171,6 +171,8 @@ class IAuthenticator(IPlugin): :rtype: :class:`list` of :class:`acme.challenges.ChallengeResponse` + :raises .PluginError: If challenges cannot be performed + """ def cleanup(achalls): @@ -180,6 +182,8 @@ class IAuthenticator(IPlugin): :class:`~letsencrypt.achallenges.AnnotatedChallenge` instances, a subset of those previously passed to :func:`perform`. + :raises PluginError: if original configuration cannot be restored + """ @@ -253,6 +257,8 @@ class IInstaller(IPlugin): :param str key_path: absolute path to the private key file :param str chain_path: absolute path to the certificate chain file + :raises .PluginError: when cert cannot be deployed + """ def enhance(domain, enhancement, options=None): @@ -266,6 +272,9 @@ class IInstaller(IPlugin): :const:`~letsencrypt.constants.ENHANCEMENTS` for expected options for each enhancement. + :raises .PluginError: If Enhancement is not supported, or if + an error occurs during the enhancement. + """ def supported_enhancements(): @@ -304,19 +313,37 @@ class IInstaller(IPlugin): :param bool temporary: Indicates whether the changes made will be quickly reversed in the future (challenges) + :raises .PluginError: when save is unsuccessful + """ def rollback_checkpoints(rollback=1): - """Revert `rollback` number of configuration checkpoints.""" + """Revert `rollback` number of configuration checkpoints. + + :raises .PluginError: when configuration cannot be fully reverted + + """ def view_config_changes(): - """Display all of the LE config changes.""" + """Display all of the LE config changes. + + :raises .PluginError: when config changes cannot be parsed + + """ def config_test(): - """Make sure the configuration is valid.""" + """Make sure the configuration is valid. + + :raises .MisconfigurationError: when the config is not in a usable state + + """ def restart(): - """Restart or refresh the server content.""" + """Restart or refresh the server content. + + :raises .PluginError: when server cannot be restarted + + """ class IDisplay(zope.interface.Interface): diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index bddf52b8d..87b3301d9 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -97,6 +97,8 @@ class Reverter(object): .. todo:: Decide on a policy for error handling, OSError IOError... + :raises .errors.ReverterError: If invalid directory structure. + """ backups = os.listdir(self.config.backup_dir) backups.sort(reverse=True) From 2953d3b2ee8f961489379a1be49136d8b9d3b974 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 13:23:31 -0700 Subject: [PATCH 51/52] Associate vhost with name after selection --- letsencrypt-apache/letsencrypt_apache/configurator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 7cda8066c..963945d8a 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -267,12 +267,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "in the Apache config", target_name) raise errors.PluginError("No vhost selected") - if not vhost.ssl: + elif not vhost.ssl: addrs = self._get_proposed_addrs(vhost, "443") # TODO: Conflicts is too conservative if not any(vhost.enabled and vhost.conflicts(addrs) for vhost in self.vhosts): vhost = self.make_vhost_ssl(vhost) - self.assoc[target_name] = vhost else: logger.error( "The selected vhost would conflict with other HTTPS " @@ -280,6 +279,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "vhost or add ServerNames to your configuration.") raise errors.PluginError( "VirtualHost not able to be selected.") + + self.assoc[target_name] = vhost return vhost From 1587653366b444005ce8b795a334ce7807b313d0 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 31 Jul 2015 15:38:43 -0700 Subject: [PATCH 52/52] Final cleanup --- letsencrypt-apache/letsencrypt_apache/configurator.py | 2 +- letsencrypt/reverter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 963945d8a..5653c7949 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -279,7 +279,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "vhost or add ServerNames to your configuration.") raise errors.PluginError( "VirtualHost not able to be selected.") - + self.assoc[target_name] = vhost return vhost diff --git a/letsencrypt/reverter.py b/letsencrypt/reverter.py index 87b3301d9..363de65f4 100644 --- a/letsencrypt/reverter.py +++ b/letsencrypt/reverter.py @@ -447,7 +447,7 @@ class Reverter(object): os.remove(path) else: logger.warning( - "File: %s - Could not be found to be deleted%s" + "File: %s - Could not be found to be deleted %s - " "LE probably shut down unexpectedly", os.linesep, path) except (IOError, OSError):