From 0b57daf473f5f742eacbad7c999631987b4f9064 Mon Sep 17 00:00:00 2001 From: Jakub Warmuz Date: Tue, 2 Jun 2015 12:10:22 +0000 Subject: [PATCH] Renewer dynamic dirs based on --config-dir/--work-dir (fixes #469). --- letsencrypt/cli.py | 4 +- letsencrypt/client.py | 17 +++-- letsencrypt/configuration.py | 28 +++++++++ letsencrypt/constants.py | 11 ++-- letsencrypt/renewer.py | 20 ++++-- letsencrypt/storage.py | 37 +++++------ letsencrypt/tests/renewer_test.py | 101 +++++++++++++++--------------- 7 files changed, 137 insertions(+), 81 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1a87f0c60..5978c4d21 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -283,7 +283,7 @@ def create_parser(plugins): help="Automatically redirect all HTTP traffic to HTTPS for the newly " "authenticated vhost.") - _paths_parser(parser.add_argument_group("paths")) + _paths_parser(parser) # _plugins_parsing should be the last thing to act upon the main # parser (--help should display plugin-specific options last) _plugins_parsing(parser, plugins) @@ -342,7 +342,7 @@ def _create_subparsers(parser): def _paths_parser(parser): - add = parser.add_argument + add = parser.add_argument_group("paths").add_argument add("--config-dir", default=flag_default("config_dir"), help=config_help("config_dir")) add("--work-dir", default=flag_default("work_dir"), diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 02159f5d2..54077d644 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -11,6 +11,8 @@ from acme.jose import jwk from letsencrypt import account from letsencrypt import auth_handler +from letsencrypt import configuration +from letsencrypt import constants from letsencrypt import continuity_auth from letsencrypt import crypto_util from letsencrypt import errors @@ -193,10 +195,17 @@ class Client(object): # ideally should be a ConfigObj, but in this case a dict will be # accepted in practice.) params = vars(self.config.namespace) - config = {"renewer_config_file": - params["renewer_config_file"]} if "renewer_config_file" in params else None - return storage.RenewableCert.new_lineage(domains[0], cert, privkey, - chain, params, config) + config = {} + cli_config = configuration.RenewerConfiguration(self.config.namespace) + + if (cli_config.config_dir != constants.CLI_DEFAULTS["config_dir"] or + cli_config.work_dir != constants.CLI_DEFAULTS["work_dir"]): + logging.warning( + "Non-standard path(s), might not work with crontab installed " + "by your operating system package manager") + + return storage.RenewableCert.new_lineage( + domains[0], cert, privkey, chain, params, config, cli_config) def save_certificate(self, certr, cert_path, chain_path): diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 00b45040a..6f05b2b49 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -90,3 +90,31 @@ class NamespaceConfig(object): def temp_checkpoint_dir(self): # pylint: disable=missing-docstring return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + + +class RenewerConfiguration(object): + """Configuration wrapper for renewer.""" + + def __init__(self, namespace): + self.namespace = namespace + + def __getattr__(self, name): + return getattr(self.namespace, name) + + @property + def archive_dir(self): # pylint: disable=missing-docstring + return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR) + + @property + def live_dir(self): # pylint: disable=missing-docstring + return os.path.join(self.namespace.config_dir, constants.LIVE_DIR) + + @property + def renewal_configs_dir(self): # pylint: disable=missing-docstring + return os.path.join( + self.namespace.config_dir, constants.RENEWAL_CONFIGS_DIR) + + @property + def renewer_config_file(self): # pylint: disable=missing-docstring + return os.path.join( + self.namespace.config_dir, constants.RENEWER_CONFIG_FILENAME) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index 9d04fb4c2..56d91f0c9 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -22,10 +22,6 @@ CLI_DEFAULTS = dict( RENEWER_DEFAULTS = dict( - renewer_config_file="/etc/letsencrypt/renewer.conf", - renewal_configs_dir="/etc/letsencrypt/configs", - archive_dir="/etc/letsencrypt/archive", - live_dir="/etc/letsencrypt/live", renewer_enabled="yes", renew_before_expiry="30 days", deploy_before_expiry="20 days", @@ -50,6 +46,8 @@ List of expected options parameters: """ +ARCHIVE_DIR = "archive" +"""TODO relative to `IConfig.config_dir`.""" CONFIG_DIRS_MODE = 0o755 """Directory mode for ``.IConfig.config_dir`` et al.""" @@ -77,6 +75,9 @@ IN_PROGRESS_DIR = "IN_PROGRESS" KEY_DIR = "keys" """Directory (relative to `IConfig.config_dir`) where keys are saved.""" +LIVE_DIR = "live" +"""TODO relative to `IConfig.config_dir`.""" + TEMP_CHECKPOINT_DIR = "temp_checkpoint" """Temporary checkpoint directory (relative to `IConfig.work_dir`).""" @@ -84,6 +85,8 @@ REC_TOKEN_DIR = "recovery_tokens" """Directory where all recovery tokens are saved (relative to `IConfig.work_dir`).""" +RENEWAL_CONFIGS_DIR = "configs" +"""TODO relative to `IConfig.config_dir`.""" RENEWER_CONFIG_FILENAME = "renewer.conf" """Renewer config file name (relative to `IConfig.config_dir`).""" diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 6e61fd893..b27f5fa4c 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -7,15 +7,19 @@ within lineages of successor certificates, according to configuration. .. todo:: Call new installer API to restart servers after deployment """ +import argparse import os +import sys import configobj from letsencrypt import configuration +from letsencrypt import cli from letsencrypt import client from letsencrypt import crypto_util from letsencrypt import notify from letsencrypt import storage + from letsencrypt.plugins import disco as plugins_disco @@ -92,7 +96,12 @@ def renew(cert, old_version): # (where fewer than all names were renewed) -def main(config=None): +def _create_parser(): + parser = argparse.ArgumentParser() + #parser.add_argument("--cron", action="store_true", help="Run as cronjob.") + return cli._paths_parser(parser) # pylint: disable=protected-access + +def main(config=None, args=sys.argv[1:]): """Main function for autorenewer script.""" # TODO: Distinguish automated invocation from manual invocation, # perhaps by looking at sys.argv[0] and inhibiting automated @@ -100,6 +109,9 @@ def main(config=None): # turned it off. (The boolean parameter should probably be # called renewer_enabled.) + cli_config = configuration.RenewerConfiguration( + _create_parser().parse_args(args)) + config = storage.config_with_defaults(config) # Now attempt to read the renewer config file and augment or replace # the renewer defaults with any options contained in that file. If @@ -108,14 +120,14 @@ def main(config=None): # elaborate renewer command line, we will presumably also be able to # specify a config file on the command line, which, if provided, should # take precedence over this one. - config.merge(configobj.ConfigObj(config.get("renewer_config_file", ""))) + config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) - for i in os.listdir(config["renewal_configs_dir"]): + for i in os.listdir(cli_config.renewal_configs_dir): print "Processing", i if not i.endswith(".conf"): continue rc_config = configobj.ConfigObj( - os.path.join(config["renewal_configs_dir"], i)) + os.path.join(cli_config.renewal_configs_dir, i)) try: # TODO: Before trying to initialize the RenewableCert object, # we could check here whether the combination of the config diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 2648be3ba..c314e3b00 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -78,14 +78,16 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes renewal configuration file and/or systemwide defaults. """ - def __init__(self, configfile, config_opts=None): + def __init__(self, configfile, config_opts=None, cli_config=None): """Instantiate a RenewableCert object from an existing lineage. :param configobj.ConfigObj configfile: an already-parsed - ConfigObj object made from reading the renewal config file - that defines this lineage. :param configobj.ConfigObj - config_opts: systemwide defaults for renewal properties not - otherwise specified in the individual renewal config file. + ConfigObj object made from reading the renewal config file + that defines this lineage. + + :param configobj.ConfigObj config_opts: systemwide defaults for + renewal properties not otherwise specified in the individual + renewal config file. :raises ValueError: if the configuration file's name didn't end in ".conf", or the file is missing or broken. @@ -93,6 +95,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes ConfigObj object. """ + self.cli_config = cli_config if isinstance(configfile, configobj.ConfigObj): if not os.path.basename(configfile.filename).endswith(".conf"): raise ValueError("renewal config file name must end in .conf") @@ -149,7 +152,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # Each element's link must point within the cert lineage's # directory within the official archive directory desired_directory = os.path.join( - self.configuration["archive_dir"], self.lineagename) + self.cli_config.archive_dir, self.lineagename) if not os.path.samefile(os.path.dirname(target), desired_directory): return False @@ -499,7 +502,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes @classmethod def new_lineage(cls, lineagename, cert, privkey, chain, - renewalparams=None, config=None): + renewalparams=None, config=None, cli_config=None): # pylint: disable=too-many-locals,too-many-arguments """Create a new certificate lineage. @@ -536,17 +539,15 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # the renewer defaults with any options contained in that file. If # renewer_config_file is undefined or if the file is nonexistent or # empty, this .merge() will have no effect. - config.merge(configobj.ConfigObj(config.get("renewer_config_file", ""))) + config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) # Examine the configuration and find the new lineage's name - configs_dir = config["renewal_configs_dir"] - archive_dir = config["archive_dir"] - live_dir = config["live_dir"] - for i in (configs_dir, archive_dir, live_dir): + for i in (cli_config.renewal_configs_dir, cli_config.archive_dir, + cli_config.live_dir): if not os.path.exists(i): os.makedirs(i, 0700) - config_file, config_filename = le_util.unique_lineage_name(configs_dir, - lineagename) + config_file, config_filename = le_util.unique_lineage_name( + cli_config.renewal_configs_dir, lineagename) if not config_filename.endswith(".conf"): raise ValueError("renewal config file name must end in .conf") @@ -554,8 +555,8 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # lineagename will now potentially be modified based on which # renewal configuration file could actually be created lineagename = os.path.basename(config_filename)[:-len(".conf")] - archive = os.path.join(archive_dir, lineagename) - live_dir = os.path.join(live_dir, lineagename) + archive = os.path.join(cli_config.archive_dir, lineagename) + live_dir = os.path.join(cli_config.live_dir, lineagename) if os.path.exists(archive): raise ValueError("archive directory exists for " + lineagename) if os.path.exists(live_dir): @@ -593,7 +594,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # TODO: add human-readable comments explaining other available # parameters new_config.write() - return cls(new_config, config) + return cls(new_config, config, cli_config) def save_successor(self, prior_version, new_cert, new_privkey, new_chain): @@ -624,7 +625,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # Figure out what the new version is and hence where to save things target_version = self.next_free_version() - archive = self.configuration["archive_dir"] + archive = self.cli_config.archive_dir prefix = os.path.join(archive, self.lineagename) target = dict( [(kind, diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 0f85674d4..d68078c18 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -10,6 +10,7 @@ import configobj import mock import pytz +from letsencrypt import configuration from letsencrypt.storage import ALL_FOUR @@ -31,22 +32,24 @@ class RenewableCertTests(unittest.TestCase): def setUp(self): from letsencrypt import storage self.tempdir = tempfile.mkdtemp() + + self.cli_config = configuration.RenewerConfiguration( + namespace=mock.MagicMock(config_dir=self.tempdir)) + # TODO: maybe provide RenewerConfiguration.make_dirs? os.makedirs(os.path.join(self.tempdir, "live", "example.org")) os.makedirs(os.path.join(self.tempdir, "archive", "example.org")) os.makedirs(os.path.join(self.tempdir, "configs")) - defaults = configobj.ConfigObj() - defaults["live_dir"] = os.path.join(self.tempdir, "live") - defaults["archive_dir"] = os.path.join(self.tempdir, "archive") - defaults["renewal_configs_dir"] = os.path.join(self.tempdir, - "configs") + config = configobj.ConfigObj() for kind in ALL_FOUR: config[kind] = os.path.join(self.tempdir, "live", "example.org", kind + ".pem") config.filename = os.path.join(self.tempdir, "configs", "example.org.conf") - self.defaults = defaults # for main() test - self.test_rc = storage.RenewableCert(config, defaults) + + self.defaults = configobj.ConfigObj() + self.test_rc = storage.RenewableCert( + config, self.defaults, self.cli_config) def tearDown(self): shutil.rmtree(self.tempdir) @@ -457,60 +460,57 @@ class RenewableCertTests(unittest.TestCase): def test_new_lineage(self): """Test for new_lineage() class method.""" from letsencrypt import storage - config_dir = self.defaults["renewal_configs_dir"] - archive_dir = self.defaults["archive_dir"] - live_dir = self.defaults["live_dir"] - result = storage.RenewableCert.new_lineage("the-lineage.com", "cert", - "privkey", "chain", None, - self.defaults) + result = storage.RenewableCert.new_lineage( + "the-lineage.com", "cert", "privkey", "chain", None, + self.defaults, self.cli_config) # This consistency check tests most relevant properties about the # newly created cert lineage. self.assertTrue(result.consistent()) - self.assertTrue(os.path.exists(os.path.join(config_dir, - "the-lineage.com.conf"))) + self.assertTrue(os.path.exists(os.path.join( + self.cli_config.renewal_configs_dir, "the-lineage.com.conf"))) with open(result.fullchain) as f: self.assertEqual(f.read(), "cert" + "chain") # Let's do it again and make sure it makes a different lineage - result = storage.RenewableCert.new_lineage("the-lineage.com", "cert2", - "privkey2", "chain2", None, - self.defaults) - self.assertTrue(os.path.exists( - os.path.join(config_dir, "the-lineage.com-0001.conf"))) + result = storage.RenewableCert.new_lineage( + "the-lineage.com", "cert2", "privkey2", "chain2", None, + self.defaults, self.cli_config) + self.assertTrue(os.path.exists(os.path.join( + self.cli_config.renewal_configs_dir, "the-lineage.com-0001.conf"))) # Now trigger the detection of already existing files - os.mkdir(os.path.join(live_dir, "the-lineage.com-0002")) + os.mkdir(os.path.join( + self.cli_config.live_dir, "the-lineage.com-0002")) self.assertRaises(ValueError, storage.RenewableCert.new_lineage, "the-lineage.com", "cert3", "privkey3", "chain3", - None, self.defaults) - os.mkdir(os.path.join(archive_dir, "other-example.com")) + None, self.defaults, self.cli_config) + os.mkdir(os.path.join(self.cli_config.archive_dir, "other-example.com")) self.assertRaises(ValueError, storage.RenewableCert.new_lineage, "other-example.com", "cert4", "privkey4", "chain4", - None, self.defaults) + None, self.defaults, self.cli_config) # Make sure it can accept renewal parameters params = {"stuff": "properties of stuff", "great": "awesome"} - result = storage.RenewableCert.new_lineage("the-lineage.com", "cert2", - "privkey2", "chain2", - params, self.defaults) + result = storage.RenewableCert.new_lineage( + "the-lineage.com", "cert2", "privkey2", "chain2", + params, self.defaults, self.cli_config) # TODO: Conceivably we could test that the renewal parameters actually # got saved def test_new_lineage_nonexistent_dirs(self): """Test that directories can be created if they don't exist.""" from letsencrypt import storage - config_dir = self.defaults["renewal_configs_dir"] - archive_dir = self.defaults["archive_dir"] - live_dir = self.defaults["live_dir"] - shutil.rmtree(config_dir) - shutil.rmtree(archive_dir) - shutil.rmtree(live_dir) - storage.RenewableCert.new_lineage("the-lineage.com", "cert2", - "privkey2", "chain2", - None, self.defaults) + shutil.rmtree(self.cli_config.renewal_configs_dir) + shutil.rmtree(self.cli_config.archive_dir) + shutil.rmtree(self.cli_config.live_dir) + + storage.RenewableCert.new_lineage( + "the-lineage.com", "cert2", "privkey2", "chain2", + None, self.defaults, self.cli_config) self.assertTrue(os.path.exists( - os.path.join(config_dir, "the-lineage.com.conf"))) - self.assertTrue(os.path.exists( - os.path.join(live_dir, "the-lineage.com", "privkey.pem"))) - self.assertTrue(os.path.exists( - os.path.join(archive_dir, "the-lineage.com", "privkey1.pem"))) + os.path.join( + self.cli_config.renewal_configs_dir, "the-lineage.com.conf"))) + self.assertTrue(os.path.exists(os.path.join( + self.cli_config.live_dir, "the-lineage.com", "privkey.pem"))) + self.assertTrue(os.path.exists(os.path.join( + self.cli_config.archive_dir, "the-lineage.com", "privkey1.pem"))) @mock.patch("letsencrypt.storage.le_util.unique_lineage_name") def test_invalid_config_filename(self, mock_uln): @@ -518,7 +518,7 @@ class RenewableCertTests(unittest.TestCase): mock_uln.return_value = "this_does_not_end_with_dot_conf", "yikes" self.assertRaises(ValueError, storage.RenewableCert.new_lineage, "example.com", "cert", "privkey", "chain", - None, self.defaults) + None, self.defaults, self.cli_config) def test_bad_kind(self): self.assertRaises(ValueError, self.test_rc.current_target, "elephant") @@ -602,22 +602,23 @@ class RenewableCertTests(unittest.TestCase): mock_rc_instance.should_autorenew.return_value = True mock_rc_instance.latest_common_version.return_value = 10 mock_rc.return_value = mock_rc_instance - with open(os.path.join(self.defaults["renewal_configs_dir"], + with open(os.path.join(self.cli_config.renewal_configs_dir, "README"), "w") as f: f.write("This is a README file to make sure that the renewer is") f.write("able to correctly ignore files that don't end in .conf.") - with open(os.path.join(self.defaults["renewal_configs_dir"], + with open(os.path.join(self.cli_config.renewal_configs_dir, "example.org.conf"), "w") as f: # This isn't actually parsed in this test; we have a separate # test_initialization that tests the initialization, assuming # that configobj can correctly parse the config file. f.write("cert = cert.pem\nprivkey = privkey.pem\n") f.write("chain = chain.pem\nfullchain = fullchain.pem\n") - with open(os.path.join(self.defaults["renewal_configs_dir"], + with open(os.path.join(self.cli_config.renewal_configs_dir, "example.com.conf"), "w") as f: f.write("cert = cert.pem\nprivkey = privkey.pem\n") f.write("chain = chain.pem\nfullchain = fullchain.pem\n") - renewer.main(self.defaults) + renewer.main(self.defaults, args=[ + '--config-dir', self.cli_config.config_dir]) self.assertEqual(mock_rc.call_count, 2) self.assertEqual(mock_rc_instance.update_all_links_to.call_count, 2) self.assertEqual(mock_notify.notify.call_count, 4) @@ -630,7 +631,8 @@ class RenewableCertTests(unittest.TestCase): mock_happy_instance.should_autorenew.return_value = False mock_happy_instance.latest_common_version.return_value = 10 mock_rc.return_value = mock_happy_instance - renewer.main(self.defaults) + renewer.main(self.defaults, args=[ + '--config-dir', self.cli_config.config_dir]) self.assertEqual(mock_rc.call_count, 4) self.assertEqual(mock_happy_instance.update_all_links_to.call_count, 0) self.assertEqual(mock_notify.notify.call_count, 4) @@ -638,10 +640,11 @@ class RenewableCertTests(unittest.TestCase): def test_bad_config_file(self): from letsencrypt import renewer - with open(os.path.join(self.defaults["renewal_configs_dir"], + with open(os.path.join(self.cli_config.renewal_configs_dir, "bad.conf"), "w") as f: f.write("incomplete = configfile\n") - renewer.main(self.defaults) + renewer.main(self.defaults, args=[ + '--config-dir', self.cli_config.config_dir]) # The ValueError is caught inside and nothing happens.