From 177e42e9d67601e860db45df1c0121f4905b536d Mon Sep 17 00:00:00 2001 From: Eugene Kazakov Date: Sun, 20 Dec 2015 12:43:57 +0600 Subject: [PATCH 001/208] Remove warning "Root (sudo) is required" --- letsencrypt/cli.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 29519d430..f614e13f8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1384,17 +1384,6 @@ def main(cli_args=sys.argv[1:]): zope.component.provideUtility(report) atexit.register(report.atexit_print_messages) - if not os.geteuid() == 0: - logger.warning( - "Root (sudo) is required to run most of letsencrypt functionality.") - # check must be done after arg parsing as --help should work - # w/o root; on the other hand, e.g. "letsencrypt run - # --authenticator dns" or "letsencrypt plugins" does not - # require root as well - #return ( - # "{0}Root is required to run letsencrypt. Please use sudo.{0}" - # .format(os.linesep)) - return args.func(args, config, plugins) if __name__ == "__main__": From d3fddc351920ffce2a55c55f08b99899942dbf44 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Fri, 8 Jan 2016 13:38:57 +0100 Subject: [PATCH 002/208] Prevent recording of deps in world set --- bootstrap/_gentoo_common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh index f49dc00f0..aa0de650c 100755 --- a/bootstrap/_gentoo_common.sh +++ b/bootstrap/_gentoo_common.sh @@ -12,12 +12,12 @@ PACKAGES=" case "$PACKAGE_MANAGER" in (paludis) - cave resolve --keep-targets if-possible $PACKAGES -x + cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x ;; (pkgcore) - pmerge --noreplace $PACKAGES + pmerge --noreplace --oneshot $PACKAGES ;; (portage|*) - emerge --noreplace $PACKAGES + emerge --noreplace --oneshot $PACKAGES ;; esac From aba11814cb666d7d6e1e55fb641d93cc9dad45e4 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Fri, 8 Jan 2016 13:47:55 +0100 Subject: [PATCH 003/208] Add Gentoo documentation --- docs/using.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 5da13f02c..78967b90c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -64,7 +64,7 @@ or for full help, type: ``letsencrypt-auto`` is the recommended method of running the Let's Encrypt client beta releases on systems that don't have a packaged version. Debian, -Arch linux, FreeBSD, and OpenBSD now have native packages, so on those +Arch linux, Gentoo, FreeBSD, and OpenBSD now have native packages, so on those systems you can just install ``letsencrypt`` (and perhaps ``letsencrypt-apache``). If you'd like to run the latest copy from Git, or run your own locally modified copy of the client, follow the instructions in @@ -376,6 +376,23 @@ If you don't want to use the Apache plugin, you can omit the Packages for Debian Jessie are coming in the next few weeks. +**Gentoo** + +.. code-block:: shell + + emerge -av app-crypt/letsencrypt + +Currently, the Apache and nginx plugins are not included in Portage. You can +however use Layman to add the mrueg overlay which does include the plugin +packages: + +.. code-block:: shell + + emerge -av app-portage/layman + layman -S + layman -a mrueg + emerge -av app-crypt/letsencrypt-apache app-crypt/letsencrypt-nginx + **Other Operating Systems** OS packaging is an ongoing effort. If you'd like to package From 15f492946893a6b460ad8ea40cffed323c9de7bd Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Mon, 25 Jan 2016 22:23:30 +0100 Subject: [PATCH 004/208] Update and expansion of Gentoo documentation --- docs/using.rst | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 78967b90c..6370de963 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -378,20 +378,49 @@ Packages for Debian Jessie are coming in the next few weeks. **Gentoo** +The official Let's Encrypt client is available in Gentoo Portage. If you +want to use the Apache plugin, it has to be installed separately: + .. code-block:: shell emerge -av app-crypt/letsencrypt + emerge -av app-crypt/letsencrypt-apache -Currently, the Apache and nginx plugins are not included in Portage. You can -however use Layman to add the mrueg overlay which does include the plugin -packages: +Currently, only the Apache plugin is included in Portage. However, if you +want the nginx plugin, you can use Layman to add the mrueg overlay which +does include the nginx plugin package: .. code-block:: shell emerge -av app-portage/layman layman -S layman -a mrueg - emerge -av app-crypt/letsencrypt-apache app-crypt/letsencrypt-nginx + emerge -av app-crypt/letsencrypt-nginx + +When using the Apache plugin, you will run into a "cannot find a cert or key +directive" error if you're sporting the default Gentoo ``httpd.conf``. +You can fix this by commenting out two lines in ``/etc/apache2/httpd.conf`` +as follows: + +Change + +.. code-block:: shell + + + LoadModule ssl_module modules/mod_ssl.so + + +to + +.. code-block:: shell + + # + LoadModule ssl_module modules/mod_ssl.so + # + +For the time being, this is the only way for the Apache plugin to recognise +the appropriate directives when installing the certificate. +Note: this change is not required for the other plugins. **Other Operating Systems** From c82ae7e755aee9eb11e5f17623b7002c77c11d9d Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 25 Jan 2016 17:17:24 -0800 Subject: [PATCH 005/208] Rename --renew-by-default to --force-renewal --- letsencrypt/cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 9fe821dd3..d2acb03bc 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -257,7 +257,7 @@ def _handle_identical_cert_request(config, cert): """ if config.renew_by_default: - logger.info("Auto-renewal forced with --renew-by-default...") + logger.info("Auto-renewal forced with --force-renewal or --renew-by-default...") return "renew", cert if cert.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") @@ -986,10 +986,12 @@ def prepare_and_parse_args(plugins, args): version="%(prog)s {0}".format(letsencrypt.__version__), help="show program's version number and exit") helpful.add( - "automation", "--renew-by-default", action="store_true", - help="Select renewal by default when domains are a superset of a " - "previously attained cert (often --keep-until-expiring is " - "more appropriate). Implies --expand.") + "automation", "--force-renewal", "--renew-by-default", + action="store_true", dest="renew_by_default", help="If a certificate " + "already exists for the requested domains, renew it now, " + "regardless of whether it is near expiry. (Often " + "--keep-until-expiring is more appropriate). Also implies " + "--expand.") helpful.add( "automation", "--agree-tos", dest="tos", action="store_true", help="Agree to the Let's Encrypt Subscriber Agreement") From 83afb58a9a6099ad1e3c54097c2bb509e98f38f8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 1 Feb 2016 17:20:20 -0800 Subject: [PATCH 006/208] Avoid dangerous and mysterious behaviour if someone tries to modify a config --- letsencrypt/configuration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index afd5edbe4..d6a016cea 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -43,6 +43,12 @@ class NamespaceConfig(object): # Check command line parameters sanity, and error out in case of problem. check_config_sanity(self) + # We're done setting up the attic. Now pull up the ladder after ourselves... + self.__setattr__ = self.__setattr_implementation__ + + def __setattr_implementation__(self, var, value): + return self.namespace.__setattr__(var, value) + def __getattr__(self, name): return getattr(self.namespace, name) From 5337fdec2394a132f5a69d29964229afa543708e Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 1 Feb 2016 17:24:05 -0800 Subject: [PATCH 007/208] Work in progress - make renew verb/main loop --- letsencrypt/cli.py | 78 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 61bc85e72..032248611 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -69,6 +69,7 @@ the cert. Major SUBCOMMANDS are: (default) run Obtain & install a cert in your current webserver certonly Obtain cert, but do not install it (aka "auth") install Install a previously obtained cert in a server + renew Renew previously obtained certs that are near expiry revoke Revoke a previously obtained certificate rollback Rollback server configuration changes made during install config_changes Show changes made to server config during installation @@ -259,7 +260,7 @@ def _treat_as_renewal(config, domains): return _handle_subset_cert_request(config, domains, subset_names_cert) def _handle_identical_cert_request(config, cert): - """Figure out what to do if a cert has the same names as a perviously obtained one + """Figure out what to do if a cert has the same names as a previously obtained one :param storage.RenewableCert cert: @@ -663,6 +664,79 @@ def install(args, config, plugins): le_client.enhance_config(domains, config) +def renew(args, config, plugins): + """Renew previously-obtained certificates.""" + print("Welcome to the renew verb!") + plugins = plugins_disco.PluginsRegistry.find_all() + cli_config = configuration.RenewerConfiguration(config) + configs_dir = cli_config.renewal_configs_dir + for renewal_file in os.listdir(configs_dir): + if not renewal_file.endswith(".conf"): + continue + print("Processing " + renewal_file) + # XXX: does this succeed in making a fully independent config object + # each time? + cli_config = configuration.RenewerConfiguration(config) + full_path = os.path.join(configs_dir, renewal_file) + try: + renewal_candidate = storage.RenewableCert(full_path, cli_config) + except (errors.CertStorageError, IOError): + logger.warning("Renewal configuration file %s is broken. " + "Skipping.", full_path) + continue + print(renewal_candidate.names(), renewal_candidate.should_autorenew()) + print("We should make a decision about whether to renew...!") + if "renewalparams" not in renewal_candidate.configuration: + logger.warning("Renewal configuration file %s lacks " + "renewalparams. Skipping.", full_path) + continue + renewalparams = renewal_candidate.configuration["renewalparams"] + if "authenticator" not in renewalparams: + logger.warning("Renewal configuration file %s does not specify " + "an authenticator. Skipping.", full_path) + continue + # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) + # XXX: also need: webroot_map + # XXX: also need: nginx_ and apache_ items + # string-valued items to add if they're present + for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", + "server", "standalone_supported_challenges"]: + if config_item in renewalparams: + print("setting", config_item, renewalparams[config_item]) + cli_config.namespace.__setattr__(config_item, + renewalparams[config_item]) + # int-valued items to add if they're present + for config_item in ["rsa_key_size", "tls_sni_01_port", "http01_port"]: + if config_item in renewalparams: + try: + value = int(renewalparams[config_item]) + cli_config.namespace.__setattr__(config_item, value) + except ValueError: + logger.warning("Renewal configuration file %s specifies " + "a non-numeric value for %s. Skipping.", + full_path, config_item) + continue + # XXX: what does this do? + zope.component.provideUtility(cli_config) + try: + authenticator = plugins[renewalparams["authenticator"]] + except KeyError: + if "authenticator" in renewal_params: + logger.warning("Renewal configuration file %s specifies an " + "authenticator plugin (%s) that could not be " + "found. Skipping.", full_path, + renewal_params["authenticator"]) + else: + logger.warning("Renewal configuration file %s specifies no " + "authenticator plugin. Skipping.", full_path) + continue + authenticator = authenticator.init(cli_config) + le_client = _init_le_client(args, cli_config, authenticator, + authenticator) + # TODO: How do we handle the separate installer vs. authenticator + # the same as installer issue? + import code; code.interact(local=locals()) + def revoke(args, config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction @@ -781,7 +855,7 @@ class HelpfulArgumentParser(object): # Maps verbs/subcommands to the functions that implement them VERBS = {"auth": obtain_cert, "certonly": obtain_cert, "config_changes": config_changes, "everything": run, - "install": install, "plugins": plugins_cmd, + "install": install, "plugins": plugins_cmd, "renew": renew, "revoke": revoke, "rollback": rollback, "run": run} # List of topics for which additional help can be provided From 42cee297b8094e23fd7c690c39ce4c280a243a52 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 1 Feb 2016 17:31:00 -0800 Subject: [PATCH 008/208] Don't parse the cli twice --- letsencrypt/cli.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 032248611..6980cca4c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -664,11 +664,11 @@ def install(args, config, plugins): le_client.enhance_config(domains, config) -def renew(args, config, plugins): +def renew(args, cli_config, plugins): """Renew previously-obtained certificates.""" print("Welcome to the renew verb!") plugins = plugins_disco.PluginsRegistry.find_all() - cli_config = configuration.RenewerConfiguration(config) + # cli_config = configuration.RenewerConfiguration(config) configs_dir = cli_config.renewal_configs_dir for renewal_file in os.listdir(configs_dir): if not renewal_file.endswith(".conf"): @@ -676,10 +676,10 @@ def renew(args, config, plugins): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - cli_config = configuration.RenewerConfiguration(config) + config = configuration.RenewerConfiguration(config) full_path = os.path.join(configs_dir, renewal_file) try: - renewal_candidate = storage.RenewableCert(full_path, cli_config) + renewal_candidate = storage.RenewableCert(full_path, config) except (errors.CertStorageError, IOError): logger.warning("Renewal configuration file %s is broken. " "Skipping.", full_path) @@ -703,21 +703,21 @@ def renew(args, config, plugins): "server", "standalone_supported_challenges"]: if config_item in renewalparams: print("setting", config_item, renewalparams[config_item]) - cli_config.namespace.__setattr__(config_item, - renewalparams[config_item]) + config.namespace.__setattr__(config_item, + renewalparams[config_item]) # int-valued items to add if they're present for config_item in ["rsa_key_size", "tls_sni_01_port", "http01_port"]: if config_item in renewalparams: try: value = int(renewalparams[config_item]) - cli_config.namespace.__setattr__(config_item, value) + config.namespace.__setattr__(config_item, value) except ValueError: logger.warning("Renewal configuration file %s specifies " "a non-numeric value for %s. Skipping.", full_path, config_item) continue # XXX: what does this do? - zope.component.provideUtility(cli_config) + zope.component.provideUtility(config) try: authenticator = plugins[renewalparams["authenticator"]] except KeyError: @@ -730,9 +730,8 @@ def renew(args, config, plugins): logger.warning("Renewal configuration file %s specifies no " "authenticator plugin. Skipping.", full_path) continue - authenticator = authenticator.init(cli_config) - le_client = _init_le_client(args, cli_config, authenticator, - authenticator) + authenticator = authenticator.init(config) + le_client = _init_le_client(args, config, authenticator, authenticator) # TODO: How do we handle the separate installer vs. authenticator # the same as installer issue? import code; code.interact(local=locals()) From 1488a3c2b495bd67d287bf99f23ac498d43c6521 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 1 Feb 2016 18:10:59 -0800 Subject: [PATCH 009/208] Work in progress --- letsencrypt/cli.py | 19 +++++++++++-------- letsencrypt/configuration.py | 6 ++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ff2c916e5..602543225 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -672,8 +672,7 @@ def install(args, config, plugins): def renew(args, cli_config, plugins): """Renew previously-obtained certificates.""" print("Welcome to the renew verb!") - plugins = plugins_disco.PluginsRegistry.find_all() - # cli_config = configuration.RenewerConfiguration(config) + cli_config = configuration.RenewerConfiguration(cli_config) configs_dir = cli_config.renewal_configs_dir for renewal_file in os.listdir(configs_dir): if not renewal_file.endswith(".conf"): @@ -681,7 +680,7 @@ def renew(args, cli_config, plugins): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - config = configuration.RenewerConfiguration(config) + config = configuration.RenewerConfiguration(cli_config) full_path = os.path.join(configs_dir, renewal_file) try: renewal_candidate = storage.RenewableCert(full_path, config) @@ -702,20 +701,24 @@ def renew(args, cli_config, plugins): continue # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) # XXX: also need: webroot_map - # XXX: also need: nginx_ and apache_ items + # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", "server", "standalone_supported_challenges"]: if config_item in renewalparams: - print("setting", config_item, renewalparams[config_item]) - config.namespace.__setattr__(config_item, - renewalparams[config_item]) + value = renewalparams[config_item] + # Unfortunately, we've lost type information from ConfigObj, + # so we don't know if the original was NoneType or str! + if value == "None": + value = None + print("setting", config_item, value) + config.__setattr__(config_item, value) # int-valued items to add if they're present for config_item in ["rsa_key_size", "tls_sni_01_port", "http01_port"]: if config_item in renewalparams: try: value = int(renewalparams[config_item]) - config.namespace.__setattr__(config_item, value) + config.__setattr__(config_item, value) except ValueError: logger.warning("Renewal configuration file %s specifies " "a non-numeric value for %s. Skipping.", diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index d6a016cea..5e54649e6 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -90,10 +90,16 @@ class RenewerConfiguration(object): def __init__(self, namespace): self.namespace = namespace + # We're done setting up the attic. Now pull up the ladder after ourselves... + self.__setattr__ = self.__setattr_implementation__ def __getattr__(self, name): return getattr(self.namespace, name) + def __setattr_implementation__(self, var, value): + print("in __setattr_implementation__, setting", var, value) + return self.namespace.__setattr__(var, value) + @property def archive_dir(self): # pylint: disable=missing-docstring return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR) From 71bd458494d8290c95571a69104cc81f3b9537a0 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 1 Feb 2016 19:18:27 -0800 Subject: [PATCH 010/208] Further work in progress --- letsencrypt/cli.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 602543225..d6b7ab683 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -6,6 +6,7 @@ from __future__ import print_function # (TODO: split this file into main.py and cli.py) import argparse import atexit +import copy import functools import json import logging @@ -388,7 +389,7 @@ def _suggest_donate(): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) -def _auth_from_domains(le_client, config, domains): +def _auth_from_domains(le_client, config, domains, lineage=None): """Authenticate and enroll certificate.""" # Note: This can raise errors... caught above us though. This is now # a three-way case: reinstall (which results in a no-op here because @@ -397,7 +398,16 @@ def _auth_from_domains(le_client, config, domains): # (which results in treating the request as a renewal), or newcert # (which results in treating the request as a new certificate request). - action, lineage = _treat_as_renewal(config, domains) + # If lineage is specified, use that one instead of looking around for + # a matching one. + if lineage is None: + # This will find a relevant matching lineage that exists + action, lineage = _treat_as_renewal(config, domains) + else: + # Renewal, where we already know the specific lineage we're + # interested in + action = "renew" if lineage.should_autorenew() else "reinstall" + if action == "reinstall": # The lineage already exists; allow the caller to try installing # it without getting a new certificate at all. @@ -674,13 +684,13 @@ def renew(args, cli_config, plugins): print("Welcome to the renew verb!") cli_config = configuration.RenewerConfiguration(cli_config) configs_dir = cli_config.renewal_configs_dir - for renewal_file in os.listdir(configs_dir): + for renewal_file in reversed(os.listdir(configs_dir)): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - config = configuration.RenewerConfiguration(cli_config) + config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) full_path = os.path.join(configs_dir, renewal_file) try: renewal_candidate = storage.RenewableCert(full_path, config) @@ -704,7 +714,8 @@ def renew(args, cli_config, plugins): # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", - "server", "standalone_supported_challenges"]: + "server", "account", + "standalone_supported_challenges"]: if config_item in renewalparams: value = renewalparams[config_item] # Unfortunately, we've lost type information from ConfigObj, @@ -724,7 +735,7 @@ def renew(args, cli_config, plugins): "a non-numeric value for %s. Skipping.", full_path, config_item) continue - # XXX: what does this do? + # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(config) try: authenticator = plugins[renewalparams["authenticator"]] @@ -739,7 +750,11 @@ def renew(args, cli_config, plugins): "authenticator plugin. Skipping.", full_path) continue authenticator = authenticator.init(config) - le_client = _init_le_client(args, config, authenticator, authenticator) + print(config) + le_client = _init_le_client(config, config, authenticator, authenticator) + print("Trying...") + print(_auth_from_domains(le_client, config, renewal_candidate.names(), + renewal_candidate)) # TODO: How do we handle the separate installer vs. authenticator # the same as installer issue? import code; code.interact(local=locals()) From 7a7cd3d4f7015e13f46bf2c5b69f06a486bfd3bc Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 1 Feb 2016 20:35:43 -0800 Subject: [PATCH 011/208] Work in progress (renewal succeeded) --- letsencrypt/cli.py | 59 ++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ccb295e07..b91f6df28 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -626,7 +626,7 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo _suggest_donate() -def obtain_cert(args, config, plugins): +def obtain_cert(args, config, plugins, lineage=None): """Implements "certonly": authenticate & obtain cert, but do not install it.""" if args.domains and args.csr is not None: @@ -645,6 +645,7 @@ def obtain_cert(args, config, plugins): # This is a special case; cert and chain are simply saved if args.csr is not None: + assert lineage is None, "Did not expect a CSR with a RenewableCert" certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( file=args.csr[0], data=args.csr[1], form="der")) cert_path, _, cert_fullchain = le_client.save_certificate( @@ -652,7 +653,7 @@ def obtain_cert(args, config, plugins): _report_new_cert(cert_path, cert_fullchain) else: domains = _find_domains(config, installer) - _auth_from_domains(le_client, config, domains) + _auth_from_domains(le_client, config, domains, lineage) _suggest_donate() @@ -681,7 +682,6 @@ def install(args, config, plugins): def renew(args, cli_config, plugins): """Renew previously-obtained certificates.""" - print("Welcome to the renew verb!") cli_config = configuration.RenewerConfiguration(cli_config) configs_dir = cli_config.renewal_configs_dir for renewal_file in reversed(os.listdir(configs_dir)): @@ -699,7 +699,6 @@ def renew(args, cli_config, plugins): "Skipping.", full_path) continue print(renewal_candidate.names(), renewal_candidate.should_autorenew()) - print("We should make a decision about whether to renew...!") if "renewalparams" not in renewal_candidate.configuration: logger.warning("Renewal configuration file %s lacks " "renewalparams. Skipping.", full_path) @@ -714,7 +713,7 @@ def renew(args, cli_config, plugins): # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", - "server", "account", + "server", "account", "authenticator", "installer", "standalone_supported_challenges"]: if config_item in renewalparams: value = renewalparams[config_item] @@ -722,7 +721,6 @@ def renew(args, cli_config, plugins): # so we don't know if the original was NoneType or str! if value == "None": value = None - print("setting", config_item, value) config.__setattr__(config_item, value) # int-valued items to add if they're present for config_item in ["rsa_key_size", "tls_sni_01_port", "http01_port"]: @@ -737,27 +735,38 @@ def renew(args, cli_config, plugins): continue # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(config) - try: - authenticator = plugins[renewalparams["authenticator"]] - except KeyError: - if "authenticator" in renewal_params: - logger.warning("Renewal configuration file %s specifies an " - "authenticator plugin (%s) that could not be " - "found. Skipping.", full_path, - renewal_params["authenticator"]) - else: - logger.warning("Renewal configuration file %s specifies no " - "authenticator plugin. Skipping.", full_path) - continue - authenticator = authenticator.init(config) + # try: + # authenticator = plugins[renewalparams["authenticator"]] + # if "installer" in renewalparams and renewalparams["installer"] != "None": + # installer = plugins[renewalparams["installer"]] + # except KeyError: + # if "authenticator" in renewal_params: + # logger.warning("Renewal configuration file %s specifies an " + # "authenticator plugin (%s) that could not be " + # "found. Skipping.", full_path, + # renewal_params["authenticator"]) + # else: + # logger.warning("Renewal configuration file %s specifies no " + # "authenticator plugin. Skipping.", full_path) + # continue + #authenticator = authenticator.init(config) + #installer = installer.init(config) print(config) - le_client = _init_le_client(config, config, authenticator, authenticator) + #le_client = _init_le_client(config, config, authenticator, installer) + try: + domains = [le_util.enforce_domain_sanity(x) for x in + renewal_candidate.names()] + except UnicodeError, ValueError: + logger.warning("Renewal configuration file %s references a cert " + "that mentions a domain name that we regarded as " + "invalid. Skipping.", full_path) + continue + + config.__setattr__("domains", domains) + print("Trying...") - print(_auth_from_domains(le_client, config, renewal_candidate.names(), - renewal_candidate)) - # TODO: How do we handle the separate installer vs. authenticator - # the same as installer issue? - import code; code.interact(local=locals()) + print(obtain_cert(config, config, plugins, renewal_candidate)) + def revoke(args, config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" From 273a78a5c63cdb7337c218aeb5fdd8173387b91b Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 15:02:15 -0800 Subject: [PATCH 012/208] use webroot_map if present (it's already a dict when deserialized!) --- letsencrypt/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ef939e426..272f97f83 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -734,7 +734,10 @@ def renew(args, cli_config, plugins): "an authenticator. Skipping.", full_path) continue # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) - # XXX: also need: webroot_map + if "webroot_map" in renewalparams: + config.__setattr__("webroot_map", renewalparams["webroot_map"]) + print ("webroot_map", renewalparams["webroot_map"]) + raw_input() # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", From 2c200b1e4316fd0d1dab7a175eb70b03c34a5b77 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 2 Feb 2016 15:05:08 -0800 Subject: [PATCH 013/208] Stray merge fix --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 2717581a7..e4a72682d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -707,7 +707,7 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def renew(args, cli_config, plugins): +def renew(cli_config, plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) configs_dir = cli_config.renewal_configs_dir From 3ea1c499f43f79d66ebd5473793889be3bd633d4 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 15:20:49 -0800 Subject: [PATCH 014/208] Cleanup after removal of "args" --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index b40583b46..788209c54 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -795,7 +795,7 @@ def renew(cli_config, plugins): config.__setattr__("domains", domains) print("Trying...") - print(obtain_cert(config, config, plugins, renewal_candidate)) + print(obtain_cert(config, plugins, renewal_candidate)) def revoke(config, unused_plugins): # TODO: coop with renewal config From 30bdbbfb823777cd5c8f3e51e544add54a7192f6 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 15:49:01 -0800 Subject: [PATCH 015/208] It's an error to use -d with renew now --- letsencrypt/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 788209c54..12430ae57 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -710,6 +710,14 @@ def install(config, plugins): def renew(cli_config, plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) + if cli_config.domains != []: + raise errors.Error("Currently, the renew verb is only capable of " + "renewing all installed certificates that are due " + "to be renewed; individual domains cannot be " + "specified with this action. If you would like to " + "renew specific certificates, use the certonly " + "command. The renew verb may provide other options " + "for selecting certificates to renew in the future.") configs_dir = cli_config.renewal_configs_dir for renewal_file in reversed(os.listdir(configs_dir)): if not renewal_file.endswith(".conf"): From 9e36c5b36d9e12d63359b1eac4120fbc5cea2765 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 16:03:48 -0800 Subject: [PATCH 016/208] Default deploy_before_expiry is now 99 years --- letsencrypt/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index a1dccd1ea..402f5e9a1 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -37,7 +37,9 @@ STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" RENEWER_DEFAULTS = dict( renewer_enabled="yes", renew_before_expiry="30 days", - deploy_before_expiry="20 days", + # This value should ensure that there is never a deployment delay by + # default. + deploy_before_expiry="99 years", ) """Defaults for renewer script.""" From ccd58dea5bebd095fcd04030d296448af195b11d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 2 Feb 2016 16:47:42 -0800 Subject: [PATCH 017/208] More helpful error when renewing with standalone --- letsencrypt/plugins/standalone.py | 3 ++- letsencrypt/plugins/standalone_test.py | 2 +- letsencrypt/plugins/util.py | 11 +++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cde7041d8..6f4f4f6a7 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -200,7 +200,8 @@ class Authenticator(common.Plugin): return self.supported_challenges def perform(self, achalls): # pylint: disable=missing-docstring - if any(util.already_listening(port) for port in self._necessary_ports): + renewer = self.config.verb == "renew" + if any(util.already_listening(port, renewer) for port in self._necessary_ports): raise errors.MisconfigurationError( "At least one of the (possibly) required ports is " "already taken.") diff --git a/letsencrypt/plugins/standalone_test.py b/letsencrypt/plugins/standalone_test.py index 1e39dee57..80f9c8a74 100644 --- a/letsencrypt/plugins/standalone_test.py +++ b/letsencrypt/plugins/standalone_test.py @@ -125,7 +125,7 @@ class AuthenticatorTest(unittest.TestCase): self.config.standalone_supported_challenges = chall self.assertRaises( errors.MisconfigurationError, self.auth.perform, []) - mock_util.already_listening.assert_called_once_with(port) + mock_util.already_listening.assert_called_once_with(port, False) mock_util.already_listening.reset_mock() @mock.patch("letsencrypt.plugins.standalone.zope.component.getUtility") diff --git a/letsencrypt/plugins/util.py b/letsencrypt/plugins/util.py index d50c7d61c..53d2f439a 100644 --- a/letsencrypt/plugins/util.py +++ b/letsencrypt/plugins/util.py @@ -11,7 +11,7 @@ from letsencrypt import interfaces logger = logging.getLogger(__name__) -def already_listening(port): +def already_listening(port, renewer=False): """Check if a process is already listening on the port. If so, also tell the user via a display notification. @@ -49,11 +49,18 @@ def already_listening(port): pid = listeners[0] name = psutil.Process(pid).name() display = zope.component.getUtility(interfaces.IDisplay) + extra = "" + if renewer: + extra = (" For automated renewal, you may want to use a script that stops" + " and starts your webserver. You can find an example at" + " https://letsencrypt.org/howitworks/#writing-your-own-renewal-script" + ". Alternatively you can use the webroot plugin to renew without" + " needing to stop and start your webserver.") display.notification( "The program {0} (process ID {1}) is already listening " "on TCP port {2}. This will prevent us from binding to " "that port. Please stop the {0} program temporarily " - "and then try again.".format(name, pid, port)) + "and then try again.{3}".format(name, pid, port, extra)) return True except (psutil.NoSuchProcess, psutil.AccessDenied): # Perhaps the result of a race where the process could have From 9fde7fe476052aef75bd1ce104bca241f1b2cc68 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 2 Feb 2016 16:52:12 -0800 Subject: [PATCH 018/208] don't ask for donations if renewing --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 12430ae57..6c209f05f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -383,7 +383,7 @@ def _report_new_cert(cert_path, fullchain_path): def _suggest_donation_if_appropriate(config): """Potentially suggest a donation to support Let's Encrypt.""" - if not config.staging: # --dry-run implies --staging + if not config.staging or config.verb == "renew": # --dry-run implies --staging reporter_util = zope.component.getUtility(interfaces.IReporter) msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" From c084814c6fc28600f269340fd0332be8dbe78e1c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 2 Feb 2016 17:01:00 -0800 Subject: [PATCH 019/208] Attempt to display better... --- letsencrypt/plugins/util.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/letsencrypt/plugins/util.py b/letsencrypt/plugins/util.py index 53d2f439a..ca311cd26 100644 --- a/letsencrypt/plugins/util.py +++ b/letsencrypt/plugins/util.py @@ -51,16 +51,18 @@ def already_listening(port, renewer=False): display = zope.component.getUtility(interfaces.IDisplay) extra = "" if renewer: - extra = (" For automated renewal, you may want to use a script that stops" - " and starts your webserver. You can find an example at" - " https://letsencrypt.org/howitworks/#writing-your-own-renewal-script" - ". Alternatively you can use the webroot plugin to renew without" - " needing to stop and start your webserver.") + extra = ( + " For automated renewal, you may want to use a script that stops" + " and starts your webserver. You can find an example at" + " https://letsencrypt.org/howitworks/#writing-your-own-renewal-script" + ". Alternatively you can use the webroot plugin to renew without" + " needing to stop and start your webserver.") display.notification( "The program {0} (process ID {1}) is already listening " "on TCP port {2}. This will prevent us from binding to " "that port. Please stop the {0} program temporarily " - "and then try again.{3}".format(name, pid, port, extra)) + "and then try again.{3}".format(name, pid, port, extra), + height=20) return True except (psutil.NoSuchProcess, psutil.AccessDenied): # Perhaps the result of a race where the process could have From c2fa9b95c1a823bcc389c62d85fc7bd08d87c790 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 2 Feb 2016 17:05:54 -0800 Subject: [PATCH 020/208] Pick a display height that works pretty well --- letsencrypt/plugins/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/plugins/util.py b/letsencrypt/plugins/util.py index ca311cd26..3382b73dd 100644 --- a/letsencrypt/plugins/util.py +++ b/letsencrypt/plugins/util.py @@ -62,7 +62,7 @@ def already_listening(port, renewer=False): "on TCP port {2}. This will prevent us from binding to " "that port. Please stop the {0} program temporarily " "and then try again.{3}".format(name, pid, port, extra), - height=20) + height=13) return True except (psutil.NoSuchProcess, psutil.AccessDenied): # Perhaps the result of a race where the process could have From b97ddb92f06f5a8c1dc4fa321466ff38862ca138 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 2 Feb 2016 17:27:40 -0800 Subject: [PATCH 021/208] fix bool --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6c209f05f..1d1fb03ea 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -383,7 +383,7 @@ def _report_new_cert(cert_path, fullchain_path): def _suggest_donation_if_appropriate(config): """Potentially suggest a donation to support Let's Encrypt.""" - if not config.staging or config.verb == "renew": # --dry-run implies --staging + if not config.staging and not config.verb == "renew": # --dry-run implies --staging reporter_util = zope.component.getUtility(interfaces.IReporter) msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" From 06b6dcc9c8c483071615c40ef1bed742b3f988cf Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 2 Feb 2016 17:35:46 -0800 Subject: [PATCH 022/208] set noninteractive to true if we're doing renew --- letsencrypt/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1d1fb03ea..d52393b62 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -726,6 +726,7 @@ def renew(cli_config, plugins): # XXX: does this succeed in making a fully independent config object # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) + config.noninteractive_mode = True full_path = os.path.join(configs_dir, renewal_file) try: renewal_candidate = storage.RenewableCert(full_path, config) From 6ae0852071ce71537341fb1a78298818e17800af Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 2 Feb 2016 17:41:48 -0800 Subject: [PATCH 023/208] Refactor: decide whether to renew or not in a single function --- letsencrypt/cli.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e4a72682d..001ceb842 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -237,7 +237,8 @@ def _find_duplicative_certs(config, domains): def _treat_as_renewal(config, domains): - """Determine whether there are duplicated names and how to handle them. + """Determine whether there are duplicated names and how to handle them + (renew, reinstall, newcert, or no action). :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -264,6 +265,21 @@ def _treat_as_renewal(config, domains): elif subset_names_cert is not None: return _handle_subset_cert_request(config, domains, subset_names_cert) + +def _should_renew(config, lineage): + "Return true if any of the circumstances for automatic renewal apply." + if config.renew_by_default: + logger.info("Auto-renewal forced with --renew-by-default...") + return True + if cert.should_autorenew(interactive=True): + logger.info("Cert is due for renewal, auto-renewing...") + return True + if config.dry_run: + logger.info("Cert not due for renewal, but simulating renewal for dry run") + return True + return False + + def _handle_identical_cert_request(config, cert): """Figure out what to do if a cert has the same names as a previously obtained one @@ -273,17 +289,12 @@ def _handle_identical_cert_request(config, cert): :rtype: tuple """ - if config.renew_by_default: - logger.info("Auto-renewal forced with --renew-by-default...") - return "renew", cert - if cert.should_autorenew(interactive=True): - logger.info("Cert is due for renewal, auto-renewing...") + if _should_renew(config, cert): return "renew", cert if config.reinstall: # Set with --reinstall, force an identical certificate to be # reinstalled without further prompting. return "reinstall", cert - question = ( "You have an existing certificate that contains exactly the same " "domains you requested and isn't close to expiry." @@ -414,12 +425,7 @@ def _auth_from_domains(le_client, config, domains, lineage=None): else: # Renewal, where we already know the specific lineage we're # interested in - action = "renew" if lineage.should_autorenew() else "reinstall" - - if config.dry_run and action == "reinstall": - logger.info( - "Cert not due for renewal, but simulating renewal for dry run") - action = "renew" + action = "renew" if _should_renew(config, lineage) else "reinstall" if action == "reinstall": # The lineage already exists; allow the caller to try installing From 674d71d4e9ec08db176d53259e454d289df77618 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 17:45:55 -0800 Subject: [PATCH 024/208] Separate constants for what to pull from renewal config --- letsencrypt/cli.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 588952bc8..f44892da8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -45,6 +45,15 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) +# These are the items which get pulled out of a renewal configuration +# file's renewalparams and actually used in the client configuration +# during the renewal process. We have to record their types here because +# the renewal configuration process loses this information. +STR_CONFIG_ITEMS = ["config_dir", "log_dir", "work_dir", "user_agent", + "server", "account", "authenticator", "installer", + "standalone_supported_challenges"] +INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] + # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: # "/home/user/.local/share/letsencrypt/bin/letsencrypt" @@ -751,15 +760,14 @@ def renew(cli_config, plugins): "an authenticator. Skipping.", full_path) continue # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) + # webroot_map is, uniquely, a dict if "webroot_map" in renewalparams: config.__setattr__("webroot_map", renewalparams["webroot_map"]) print ("webroot_map", renewalparams["webroot_map"]) raw_input() # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present - for config_item in ["config_dir", "log_dir", "work_dir", "user_agent", - "server", "account", "authenticator", "installer", - "standalone_supported_challenges"]: + for config_item in STR_CONFIG_ITEMS: if config_item in renewalparams: value = renewalparams[config_item] # Unfortunately, we've lost type information from ConfigObj, @@ -768,7 +776,7 @@ def renew(cli_config, plugins): value = None config.__setattr__(config_item, value) # int-valued items to add if they're present - for config_item in ["rsa_key_size", "tls_sni_01_port", "http01_port"]: + for config_item in INT_CONFIG_ITEMS: if config_item in renewalparams: try: value = int(renewalparams[config_item]) @@ -1286,6 +1294,7 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) + import code; code.interact(local=locals()) return helpful.parse_args() From 0a2b5376295d9d15d7b5e73e039e9a7a9e76dc0e Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 17:49:42 -0800 Subject: [PATCH 025/208] Fix mistaken parameter reference in _should_renew --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f44892da8..76883bc60 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -280,7 +280,7 @@ def _should_renew(config, lineage): if config.renew_by_default: logger.info("Auto-renewal forced with --renew-by-default...") return True - if cert.should_autorenew(interactive=True): + if lineage.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") return True if config.dry_run: From 3697ca7e3e744d7eda8bdbb122f78f6cec465a44 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 2 Feb 2016 17:56:12 -0800 Subject: [PATCH 026/208] throw an error if manual is run non-interactively --- letsencrypt/plugins/manual.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 7f782a41b..29f4639fe 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -91,6 +91,8 @@ s.serve_forever()" """ help="Automatically allows public IP logging.") def prepare(self): # pylint: disable=missing-docstring,no-self-use + if self.config.noninteractive_mode: + raise errors.PluginError("Running manual mode non-interactively is not supported") pass # pragma: no cover def more_info(self): # pylint: disable=missing-docstring,no-self-use From 6682e370a8a0041f9bb4274117c7ec5f4ccc1399 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 18:28:04 -0800 Subject: [PATCH 027/208] Very ugly approach to extract types from the parser! --- letsencrypt/cli.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 76883bc60..c9eaa1346 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -45,6 +45,10 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) +# This is global scope in order to be able to extract type information from +# it later +_parser = None + # These are the items which get pulled out of a renewal configuration # file's renewalparams and actually used in the client configuration # during the renewal process. We have to record their types here because @@ -54,6 +58,10 @@ STR_CONFIG_ITEMS = ["config_dir", "log_dir", "work_dir", "user_agent", "standalone_supported_challenges"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] +# These are the plugins for which we should try to automatically extract +# the types when pulling items from a renewal configuration. +EXTRACT_PLUGIN_PREFIXES = ["apache_", "nginx_", "standalone_"] + # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: # "/home/user/.local/share/letsencrypt/bin/letsencrypt" @@ -723,6 +731,8 @@ def install(config, plugins): def renew(cli_config, plugins): + print ("Beginning renew") + import code; code.interact(local=locals()) """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) if cli_config.domains != []: @@ -743,6 +753,8 @@ def renew(cli_config, plugins): config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) config.noninteractive_mode = True full_path = os.path.join(configs_dir, renewal_file) + + try: renewal_candidate = storage.RenewableCert(full_path, config) except (errors.CertStorageError, IOError): @@ -786,6 +798,19 @@ def renew(cli_config, plugins): "a non-numeric value for %s. Skipping.", full_path, config_item) continue + # Now use parser to get plugin-prefixed items with correct types + # XXX: is it true that an item will end up in _parser._actions even + # when no action was explicitly specified? + for plugin_prefix in EXTRACT_PLUGIN_PREFIXES: + for config_item in renewalparams.keys(): + if config_item.startswith(plugin_prefix): + for action in _parser.parser._actions: + if action.dest == config_item: + if action.type is not None: + config.__setattr__(config_item, action.type(renewalparams[config_item])) + break + else: + config.__setattr__(config_item, str(renewalparams[config_item])) # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(config) # try: @@ -1294,7 +1319,9 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) - import code; code.interact(local=locals()) + global _parser + _parser = helpful + print("stored _parser") return helpful.parse_args() From b69a1020872bfa8866ab52d34d025bea4577d113 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 2 Feb 2016 18:30:22 -0800 Subject: [PATCH 028/208] Removing debug prints --- letsencrypt/cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c9eaa1346..02ccfa5f3 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -731,8 +731,6 @@ def install(config, plugins): def renew(cli_config, plugins): - print ("Beginning renew") - import code; code.interact(local=locals()) """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) if cli_config.domains != []: @@ -761,7 +759,6 @@ def renew(cli_config, plugins): logger.warning("Renewal configuration file %s is broken. " "Skipping.", full_path) continue - print(renewal_candidate.names(), renewal_candidate.should_autorenew()) if "renewalparams" not in renewal_candidate.configuration: logger.warning("Renewal configuration file %s lacks " "renewalparams. Skipping.", full_path) @@ -775,8 +772,6 @@ def renew(cli_config, plugins): # webroot_map is, uniquely, a dict if "webroot_map" in renewalparams: config.__setattr__("webroot_map", renewalparams["webroot_map"]) - print ("webroot_map", renewalparams["webroot_map"]) - raw_input() # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: @@ -829,7 +824,6 @@ def renew(cli_config, plugins): # continue #authenticator = authenticator.init(config) #installer = installer.init(config) - print(config) #le_client = _init_le_client(config, config, authenticator, installer) try: domains = [le_util.enforce_domain_sanity(x) for x in From 05c07ad90cfc51c07571b42ee2bea20e7ada6377 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 11:08:49 -0800 Subject: [PATCH 029/208] Reduce spaminess --- letsencrypt/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 1ed7af2db..72aabe548 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -97,7 +97,6 @@ class RenewerConfiguration(object): return getattr(self.namespace, name) def __setattr_implementation__(self, var, value): - print("in __setattr_implementation__, setting", var, value) return self.namespace.__setattr__(var, value) @property From ec7e957fe6e94f82a703ee746bbf582747ddc574 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 11:52:57 -0800 Subject: [PATCH 030/208] Minimal test unbreakage Though really this test will need to be redesigned :( --- letsencrypt/cli.py | 1 - letsencrypt/tests/cli_test.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 02ccfa5f3..1d4764835 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1315,7 +1315,6 @@ def prepare_and_parse_args(plugins, args): global _parser _parser = helpful - print("stored _parser") return helpful.parse_args() diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f0ac954f9..7f20d65df 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -567,7 +567,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) def test_certonly_dry_run_reinstall_is_renewal(self): - _, get_utility = self._test_certonly_renewal_common('reinstall', + _, get_utility = self._test_certonly_renewal_common('renew', ['--dry-run']) self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) From bfd182ae39fac4b11072867acd94ff89f7b44128 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 12:14:49 -0800 Subject: [PATCH 031/208] Fix _test_certonly_csr_common (more) properly --- letsencrypt/tests/cli_test.py | 40 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 7f20d65df..6b2e6f9f1 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -530,35 +530,35 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - def _test_certonly_renewal_common(self, renewal_verb, extra_args=None): + @mock.patch('letsencrypt.cli._find_duplicative_certs') + def _test_certonly_renewal_common(self, extra_args, mock_fdc): cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') - with mock.patch('letsencrypt.cli._treat_as_renewal') as mock_renewal: - mock_renewal.return_value = (renewal_verb, mock_lineage) - mock_client = mock.MagicMock() - mock_client.obtain_certificate.return_value = (mock_certr, 'chain', - mock_key, 'csr') - with mock.patch('letsencrypt.cli._init_le_client') as mock_init: - mock_init.return_value = mock_client - get_utility_path = 'letsencrypt.cli.zope.component.getUtility' - with mock.patch(get_utility_path) as mock_get_utility: - with mock.patch('letsencrypt.cli.OpenSSL'): - with mock.patch('letsencrypt.cli.crypto_util'): - args = ['-d', 'foo.bar', '-a', - 'standalone', 'certonly'] - if extra_args: - args += extra_args - self._call(args) + mock_fdc.return_value = (mock_lineage, None) + mock_client = mock.MagicMock() + mock_client.obtain_certificate.return_value = (mock_certr, 'chain', + mock_key, 'csr') + with mock.patch('letsencrypt.cli._init_le_client') as mock_init: + mock_init.return_value = mock_client + get_utility_path = 'letsencrypt.cli.zope.component.getUtility' + with mock.patch(get_utility_path) as mock_get_utility: + with mock.patch('letsencrypt.cli.OpenSSL'): + with mock.patch('letsencrypt.cli.crypto_util'): + args = ['-d', 'foo.bar', '-a', + 'standalone', 'certonly'] + if extra_args: + args += extra_args + self._call(args) mock_client.obtain_certificate.assert_called_once_with(['foo.bar']) return mock_lineage, mock_get_utility def test_certonly_renewal(self): - lineage, get_utility = self._test_certonly_renewal_common('renew') + lineage, get_utility = self._test_certonly_renewal_common([]) self.assertEqual(lineage.save_successor.call_count, 1) lineage.update_all_links_to.assert_called_once_with( lineage.latest_common_version()) @@ -567,11 +567,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) def test_certonly_dry_run_reinstall_is_renewal(self): - _, get_utility = self._test_certonly_renewal_common('renew', - ['--dry-run']) + _, get_utility = self._test_certonly_renewal_common(['--dry-run']) self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From 37709f2e078dc0e7ed793993ecbb30ca84d2451c Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 12:15:51 -0800 Subject: [PATCH 032/208] Remove commented-out lines --- letsencrypt/cli.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1d4764835..0fe0bd805 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -768,7 +768,6 @@ def renew(cli_config, plugins): logger.warning("Renewal configuration file %s does not specify " "an authenticator. Skipping.", full_path) continue - # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) # webroot_map is, uniquely, a dict if "webroot_map" in renewalparams: config.__setattr__("webroot_map", renewalparams["webroot_map"]) @@ -808,23 +807,6 @@ def renew(cli_config, plugins): config.__setattr__(config_item, str(renewalparams[config_item])) # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(config) - # try: - # authenticator = plugins[renewalparams["authenticator"]] - # if "installer" in renewalparams and renewalparams["installer"] != "None": - # installer = plugins[renewalparams["installer"]] - # except KeyError: - # if "authenticator" in renewal_params: - # logger.warning("Renewal configuration file %s specifies an " - # "authenticator plugin (%s) that could not be " - # "found. Skipping.", full_path, - # renewal_params["authenticator"]) - # else: - # logger.warning("Renewal configuration file %s specifies no " - # "authenticator plugin. Skipping.", full_path) - # continue - #authenticator = authenticator.init(config) - #installer = installer.init(config) - #le_client = _init_le_client(config, config, authenticator, installer) try: domains = [le_util.enforce_domain_sanity(x) for x in renewal_candidate.names()] From 5642edfbffc23871d23f083bf4f75308846272c1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 12:54:56 -0800 Subject: [PATCH 033/208] Clobber EXTRACT_PLUGIN_PREFIXES; fix several bugs --- letsencrypt/cli.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0fe0bd805..6278859db 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -58,10 +58,6 @@ STR_CONFIG_ITEMS = ["config_dir", "log_dir", "work_dir", "user_agent", "standalone_supported_challenges"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] -# These are the plugins for which we should try to automatically extract -# the types when pulling items from a renewal configuration. -EXTRACT_PLUGIN_PREFIXES = ["apache_", "nginx_", "standalone_"] - # For help strings, figure out how the user ran us. # When invoked from letsencrypt-auto, sys.argv[0] is something like: # "/home/user/.local/share/letsencrypt/bin/letsencrypt" @@ -768,9 +764,6 @@ def renew(cli_config, plugins): logger.warning("Renewal configuration file %s does not specify " "an authenticator. Skipping.", full_path) continue - # webroot_map is, uniquely, a dict - if "webroot_map" in renewalparams: - config.__setattr__("webroot_map", renewalparams["webroot_map"]) # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: @@ -793,11 +786,23 @@ def renew(cli_config, plugins): full_path, config_item) continue # Now use parser to get plugin-prefixed items with correct types + # XXX: the current approach of extracting only prefixed items + # related to the actually-used installer and authenticator + # works as long as plugins don't need to read plugin-specific + # variables set by someone else (e.g., assuming Apache + # configurator doesn't need to read webroot_ variables). # XXX: is it true that an item will end up in _parser._actions even # when no action was explicitly specified? - for plugin_prefix in EXTRACT_PLUGIN_PREFIXES: + plugin_prefixes = [renewalparams["authenticator"]] + if "installer" in renewalparams and renewalparams["installer"] != None: + plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(renewalparams): for config_item in renewalparams.keys(): - if config_item.startswith(plugin_prefix): + if renewalparams[config_item] == "None": + # Avoid confusion when, for example, csr = None (avoid + # trying to read the file called "None") + continue + if config_item.startswith(plugin_prefix + "_"): for action in _parser.parser._actions: if action.dest == config_item: if action.type is not None: @@ -805,6 +810,10 @@ def renew(cli_config, plugins): break else: config.__setattr__(config_item, str(renewalparams[config_item])) + # webroot_map is, uniquely, a dict, and the logic above is not able + # to correctly parse it from the serialized form. + if "webroot_map" in renewalparams: + config.__setattr__("webroot_map", renewalparams["webroot_map"]) # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(config) try: @@ -819,7 +828,11 @@ def renew(cli_config, plugins): config.__setattr__("domains", domains) print("Trying...") - print(obtain_cert(config, plugins, renewal_candidate)) + # Because obtain_cert itself indirectly decides whether to renew + # or not, we couldn't currently make a UI/logging distinction at + # this stage to indicate whether renewal was actually attempted + # (or successful). + obtain_cert(config, plugins, renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From c0d55c7c33a032b105887fa46e96ce3e935191a3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 12:57:49 -0800 Subject: [PATCH 034/208] Improve certonly renewal test coverage --- letsencrypt/cli.py | 3 +++ letsencrypt/tests/cli_test.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1d4764835..fabe33c38 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -287,12 +287,15 @@ def _should_renew(config, lineage): "Return true if any of the circumstances for automatic renewal apply." if config.renew_by_default: logger.info("Auto-renewal forced with --renew-by-default...") + print("forced") return True if lineage.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") + print("due") return True if config.dry_run: logger.info("Cert not due for renewal, but simulating renewal for dry run") + print("dry") return True return False diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 6b2e6f9f1..36d39590d 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -531,10 +531,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._certonly_new_request_common, mock_client) @mock.patch('letsencrypt.cli._find_duplicative_certs') - def _test_certonly_renewal_common(self, extra_args, mock_fdc): + def _test_renewal_common(self, due_for_renewal, extra_args, outstring, mock_fdc): cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) + mock_lineage.should_autorenew.return_value = due_for_renewal mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') mock_fdc.return_value = (mock_lineage, None) @@ -553,12 +554,16 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args += extra_args self._call(args) + if outstring: + with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: + self.assertTrue(outstring in lf.read()) + mock_client.obtain_certificate.assert_called_once_with(['foo.bar']) return mock_lineage, mock_get_utility def test_certonly_renewal(self): - lineage, get_utility = self._test_certonly_renewal_common([]) + lineage, get_utility = self._test_renewal_common(True, [], None) self.assertEqual(lineage.save_successor.call_count, 1) lineage.update_all_links_to.assert_called_once_with( lineage.latest_common_version()) @@ -566,11 +571,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue('fullchain.pem' in cert_msg) self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) - def test_certonly_dry_run_reinstall_is_renewal(self): - _, get_utility = self._test_certonly_renewal_common(['--dry-run']) + def test_certonly_renewal_triggers(self): + # --dry-run should force renewal + _, get_utility = self._test_renewal_common(False, ['--dry-run'], None) self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) + _, _ = self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], "Auto-renewal forced") + self.assertEqual(get_utility().add_message.call_count, 1) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From af39b52122f69c414b92f53af06fb27011cd804b Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 13:20:55 -0800 Subject: [PATCH 035/208] Split out _reconstitute() from renew() --- letsencrypt/cli.py | 183 ++++++++++++++++++++++++++------------------- 1 file changed, 107 insertions(+), 76 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6278859db..5a466dc25 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -726,6 +726,101 @@ def install(config, plugins): le_client.enhance_config(domains, config) +def _reconstitute(full_path, config): + """Try to instantiate a RenewableCert, updating config with relevant items. + + This is specifically for use in renewal and enforces several checks + and policies to ensure that we can try to proceed with the renwal + request. The config argument is modified by including relevant options + read from the renewal configuration file. + + :returns: the RenewableCert object or None if a fatal error occurred + :rtype: `storage.RenewableCert` or NoneType + """ + + try: + renewal_candidate = storage.RenewableCert(full_path, config) + except (errors.CertStorageError, IOError): + logger.warning("Renewal configuration file %s is broken. " + "Skipping.", full_path) + return None + if "renewalparams" not in renewal_candidate.configuration: + logger.warning("Renewal configuration file %s lacks " + "renewalparams. Skipping.", full_path) + return None + renewalparams = renewal_candidate.configuration["renewalparams"] + if "authenticator" not in renewalparams: + logger.warning("Renewal configuration file %s does not specify " + "an authenticator. Skipping.", full_path) + return None + # string-valued items to add if they're present + for config_item in STR_CONFIG_ITEMS: + if config_item in renewalparams: + value = renewalparams[config_item] + # Unfortunately, we've lost type information from ConfigObj, + # so we don't know if the original was NoneType or str! + if value == "None": + value = None + config.__setattr__(config_item, value) + # int-valued items to add if they're present + for config_item in INT_CONFIG_ITEMS: + if config_item in renewalparams: + try: + value = int(renewalparams[config_item]) + config.__setattr__(config_item, value) + except ValueError: + logger.warning("Renewal configuration file %s specifies " + "a non-numeric value for %s. Skipping.", + full_path, config_item) + return None + # Now use parser to get plugin-prefixed items with correct types + # XXX: the current approach of extracting only prefixed items + # related to the actually-used installer and authenticator + # works as long as plugins don't need to read plugin-specific + # variables set by someone else (e.g., assuming Apache + # configurator doesn't need to read webroot_ variables). + # XXX: is it true that an item will end up in _parser._actions even + # when no action was explicitly specified? + plugin_prefixes = [renewalparams["authenticator"]] + if "installer" in renewalparams and renewalparams["installer"] != None: + plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(renewalparams): + for config_item in renewalparams.keys(): + if renewalparams[config_item] == "None": + # Avoid confusion when, for example, "csr = None" (avoid + # trying to read the file called "None") + # Should we omit the item entirely rather than setting + # its value to None? + config.__setattr__(config_item, None) + continue + if config_item.startswith(plugin_prefix + "_"): + for action in _parser.parser._actions: + if action.dest == config_item: + if action.type is not None: + config.__setattr__(config_item, action.type(renewalparams[config_item])) + break + else: + config.__setattr__(config_item, str(renewalparams[config_item])) + # webroot_map is, uniquely, a dict, and the logic above is not able + # to correctly parse it from the serialized form. + if "webroot_map" in renewalparams: + config.__setattr__("webroot_map", renewalparams["webroot_map"]) + + try: + domains = [le_util.enforce_domain_sanity(x) for x in + renewal_candidate.names()] + except UnicodeError, ValueError: + logger.warning("Renewal configuration file %s references a cert " + "that mentions a domain name that we regarded as " + "invalid. Skipping.", full_path) + return None + + config.__setattr__("domains", domains) + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(config) + return renewal_candidate + + def renew(cli_config, plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) @@ -738,7 +833,7 @@ def renew(cli_config, plugins): "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") configs_dir = cli_config.renewal_configs_dir - for renewal_file in reversed(os.listdir(configs_dir)): + for renewal_file in os.listdir(configs_dir): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) @@ -748,84 +843,20 @@ def renew(cli_config, plugins): config.noninteractive_mode = True full_path = os.path.join(configs_dir, renewal_file) - + # Note that this modifies config (to add back the configuration + # elements from within the renewal configuration file). try: - renewal_candidate = storage.RenewableCert(full_path, config) - except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) - continue - if "renewalparams" not in renewal_candidate.configuration: - logger.warning("Renewal configuration file %s lacks " - "renewalparams. Skipping.", full_path) - continue - renewalparams = renewal_candidate.configuration["renewalparams"] - if "authenticator" not in renewalparams: - logger.warning("Renewal configuration file %s does not specify " - "an authenticator. Skipping.", full_path) - continue - # XXX: also need: nginx_, apache_, and plesk_ items - # string-valued items to add if they're present - for config_item in STR_CONFIG_ITEMS: - if config_item in renewalparams: - value = renewalparams[config_item] - # Unfortunately, we've lost type information from ConfigObj, - # so we don't know if the original was NoneType or str! - if value == "None": - value = None - config.__setattr__(config_item, value) - # int-valued items to add if they're present - for config_item in INT_CONFIG_ITEMS: - if config_item in renewalparams: - try: - value = int(renewalparams[config_item]) - config.__setattr__(config_item, value) - except ValueError: - logger.warning("Renewal configuration file %s specifies " - "a non-numeric value for %s. Skipping.", - full_path, config_item) - continue - # Now use parser to get plugin-prefixed items with correct types - # XXX: the current approach of extracting only prefixed items - # related to the actually-used installer and authenticator - # works as long as plugins don't need to read plugin-specific - # variables set by someone else (e.g., assuming Apache - # configurator doesn't need to read webroot_ variables). - # XXX: is it true that an item will end up in _parser._actions even - # when no action was explicitly specified? - plugin_prefixes = [renewalparams["authenticator"]] - if "installer" in renewalparams and renewalparams["installer"] != None: - plugin_prefixes.append(renewalparams["installer"]) - for plugin_prefix in set(renewalparams): - for config_item in renewalparams.keys(): - if renewalparams[config_item] == "None": - # Avoid confusion when, for example, csr = None (avoid - # trying to read the file called "None") - continue - if config_item.startswith(plugin_prefix + "_"): - for action in _parser.parser._actions: - if action.dest == config_item: - if action.type is not None: - config.__setattr__(config_item, action.type(renewalparams[config_item])) - break - else: - config.__setattr__(config_item, str(renewalparams[config_item])) - # webroot_map is, uniquely, a dict, and the logic above is not able - # to correctly parse it from the serialized form. - if "webroot_map" in renewalparams: - config.__setattr__("webroot_map", renewalparams["webroot_map"]) - # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(config) - try: - domains = [le_util.enforce_domain_sanity(x) for x in - renewal_candidate.names()] - except UnicodeError, ValueError: - logger.warning("Renewal configuration file %s references a cert " - "that mentions a domain name that we regarded as " - "invalid. Skipping.", full_path) + renewal_candidate = _reconstitute(full_path, config) + except Exception as e: + # reconstitute encountered an unanticipated problem. + logger.warning("Renewal configuration file %s produced an " + "unexpected error: %s. Skipping.", full_path, e) continue - config.__setattr__("domains", domains) + if renewal_candidate is None: + # reconstitute indicated an error or problem which has + # already been logged. Go on to the next config. + continue print("Trying...") # Because obtain_cert itself indirectly decides whether to renew From f7a350b0f8cac9d030e5842d9b8ca3e4b12f39fc Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 13:21:43 -0800 Subject: [PATCH 036/208] Move zope call back inside renew() --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5a466dc25..faa27c88c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -816,8 +816,6 @@ def _reconstitute(full_path, config): return None config.__setattr__("domains", domains) - # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(config) return renewal_candidate @@ -857,6 +855,8 @@ def renew(cli_config, plugins): # reconstitute indicated an error or problem which has # already been logged. Go on to the next config. continue + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(config) print("Trying...") # Because obtain_cert itself indirectly decides whether to renew From fd0fd1444d4e8ace752e3d2c86fa48f0b49427e8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 13:29:13 -0800 Subject: [PATCH 037/208] Thorough checking, less printfs --- letsencrypt/cli.py | 4 +--- letsencrypt/tests/cli_test.py | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 11a9ea292..02195645e 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -283,16 +283,14 @@ def _should_renew(config, lineage): "Return true if any of the circumstances for automatic renewal apply." if config.renew_by_default: logger.info("Auto-renewal forced with --renew-by-default...") - print("forced") return True if lineage.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") - print("due") return True if config.dry_run: logger.info("Cert not due for renewal, but simulating renewal for dry run") - print("dry") return True + logger.info("Cert not yet due for renewal") return False diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 36d39590d..42f6c3bdd 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -573,13 +573,15 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_certonly_renewal_triggers(self): # --dry-run should force renewal - _, get_utility = self._test_renewal_common(False, ['--dry-run'], None) + _, get_utility = self._test_renewal_common(False, ['--dry-run'], "simulating renewal") self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) - _, _ = self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], "Auto-renewal forced") + _, _ = self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], + "Auto-renewal forced") self.assertEqual(get_utility().add_message.call_count, 1) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From 22adea60bfcb495c640cea5b41e5ded294ac8468 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 13:49:26 -0800 Subject: [PATCH 038/208] _restore_required_config_elements + fix except syntax --- letsencrypt/cli.py | 76 +++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 247c5fc51..4c30e2ca8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -727,33 +727,7 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _reconstitute(full_path, config): - """Try to instantiate a RenewableCert, updating config with relevant items. - - This is specifically for use in renewal and enforces several checks - and policies to ensure that we can try to proceed with the renwal - request. The config argument is modified by including relevant options - read from the renewal configuration file. - - :returns: the RenewableCert object or None if a fatal error occurred - :rtype: `storage.RenewableCert` or NoneType - """ - - try: - renewal_candidate = storage.RenewableCert(full_path, config) - except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) - return None - if "renewalparams" not in renewal_candidate.configuration: - logger.warning("Renewal configuration file %s lacks " - "renewalparams. Skipping.", full_path) - return None - renewalparams = renewal_candidate.configuration["renewalparams"] - if "authenticator" not in renewalparams: - logger.warning("Renewal configuration file %s does not specify " - "an authenticator. Skipping.", full_path) - return None +def _restore_required_config_elements(full_path, config, renewalparams): # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: if config_item in renewalparams: @@ -773,7 +747,7 @@ def _reconstitute(full_path, config): logger.warning("Renewal configuration file %s specifies " "a non-numeric value for %s. Skipping.", full_path, config_item) - return None + raise # Now use parser to get plugin-prefixed items with correct types # XXX: the current approach of extracting only prefixed items # related to the actually-used installer and authenticator @@ -802,15 +776,55 @@ def _reconstitute(full_path, config): break else: config.__setattr__(config_item, str(renewalparams[config_item])) - # webroot_map is, uniquely, a dict, and the logic above is not able - # to correctly parse it from the serialized form. + return True + + +def _reconstitute(full_path, config): + """Try to instantiate a RenewableCert, updating config with relevant items. + + This is specifically for use in renewal and enforces several checks + and policies to ensure that we can try to proceed with the renwal + request. The config argument is modified by including relevant options + read from the renewal configuration file. + + :returns: the RenewableCert object or None if a fatal error occurred + :rtype: `storage.RenewableCert` or NoneType + """ + + try: + renewal_candidate = storage.RenewableCert(full_path, config) + except (errors.CertStorageError, IOError): + logger.warning("Renewal configuration file %s is broken. " + "Skipping.", full_path) + return None + if "renewalparams" not in renewal_candidate.configuration: + logger.warning("Renewal configuration file %s lacks " + "renewalparams. Skipping.", full_path) + return None + renewalparams = renewal_candidate.configuration["renewalparams"] + if "authenticator" not in renewalparams: + logger.warning("Renewal configuration file %s does not specify " + "an authenticator. Skipping.", full_path) + return None + # Now restore specific values along with their data types, if + # those elements are present. + try: + _restore_required_config_elements(full_path, config, renewalparams) + except ValueError: + # There was a data type error which has already been + # logged. + return None + + # webroot_map is, uniquely, a dict, and the general-purpose + # configuration restoring logic is not able to correctly parse it + # from the serialized form. if "webroot_map" in renewalparams: config.__setattr__("webroot_map", renewalparams["webroot_map"]) try: domains = [le_util.enforce_domain_sanity(x) for x in renewal_candidate.names()] - except UnicodeError, ValueError: + except (UnicodeError, ValueError): logger.warning("Renewal configuration file %s references a cert " "that mentions a domain name that we regarded as " "invalid. Skipping.", full_path) From 78fd28a4865709fa5409d99e794fb69e595297ae Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 13:51:01 -0800 Subject: [PATCH 039/208] coverage++ --- letsencrypt/tests/cli_test.py | 39 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 42f6c3bdd..f083018b3 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -530,35 +530,38 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - @mock.patch('letsencrypt.cli._find_duplicative_certs') - def _test_renewal_common(self, due_for_renewal, extra_args, outstring, mock_fdc): + def _test_renewal_common(self, due_for_renewal, extra_args, outstring, renew=True): cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) mock_lineage.should_autorenew.return_value = due_for_renewal mock_certr = mock.MagicMock() mock_key = mock.MagicMock(pem='pem_key') - mock_fdc.return_value = (mock_lineage, None) mock_client = mock.MagicMock() mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') - with mock.patch('letsencrypt.cli._init_le_client') as mock_init: - mock_init.return_value = mock_client - get_utility_path = 'letsencrypt.cli.zope.component.getUtility' - with mock.patch(get_utility_path) as mock_get_utility: - with mock.patch('letsencrypt.cli.OpenSSL'): - with mock.patch('letsencrypt.cli.crypto_util'): - args = ['-d', 'foo.bar', '-a', - 'standalone', 'certonly'] - if extra_args: - args += extra_args - self._call(args) + with mock.patch('letsencrypt.cli._find_duplicative_certs') as mock_fdc: + mock_fdc.return_value = (mock_lineage, None) + with mock.patch('letsencrypt.cli._init_le_client') as mock_init: + mock_init.return_value = mock_client + get_utility_path = 'letsencrypt.cli.zope.component.getUtility' + with mock.patch(get_utility_path) as mock_get_utility: + with mock.patch('letsencrypt.cli.OpenSSL'): + with mock.patch('letsencrypt.cli.crypto_util'): + args = ['-d', 'foo.bar', '-a', + 'standalone', 'certonly'] + if extra_args: + args += extra_args + self._call(args) if outstring: with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: self.assertTrue(outstring in lf.read()) - mock_client.obtain_certificate.assert_called_once_with(['foo.bar']) + if renew: + mock_client.obtain_certificate.assert_called_once_with(['foo.bar']) + else: + self.assertEqual(mock_client.obtain_certificate.call_count, 0) return mock_lineage, mock_get_utility @@ -573,7 +576,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_certonly_renewal_triggers(self): # --dry-run should force renewal - _, get_utility = self._test_renewal_common(False, ['--dry-run'], "simulating renewal") + _, get_utility = self._test_renewal_common(False, ['--dry-run', '--keep'], + "simulating renewal") self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) @@ -581,6 +585,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods "Auto-renewal forced") self.assertEqual(get_utility().add_message.call_count, 1) + _, _ = self._test_renewal_common(False, ['-tvv', '--debug', '--keep'], + "not yet due", renew=False) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From e9f59f6fe2b4352b3ff72b8b552962ab4277ef74 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 13:51:41 -0800 Subject: [PATCH 040/208] add diff checks before each setattr --- letsencrypt/cli.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1d4764835..fa9e77d94 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -53,7 +53,7 @@ _parser = None # file's renewalparams and actually used in the client configuration # during the renewal process. We have to record their types here because # the renewal configuration process loses this information. -STR_CONFIG_ITEMS = ["config_dir", "log_dir", "work_dir", "user_agent", +STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "server", "account", "authenticator", "installer", "standalone_supported_challenges"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] @@ -730,6 +730,17 @@ def install(config, plugins): le_client.enhance_config(domains, config) +def _diff_from_default(default_conf, cli_conf, value): + try: + default = default_conf.__getattr__(value) + cli = cli_conf.__getattr__(value) + except AttributeError: + return False + if cli != default: + return True + else: + return False + def renew(cli_config, plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) @@ -750,6 +761,8 @@ def renew(cli_config, plugins): # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) config.noninteractive_mode = True + default_args = prepare_and_parse_args(plugins, []) + default_conf = configuration.NamespaceConfig(default_args) full_path = os.path.join(configs_dir, renewal_file) @@ -770,12 +783,13 @@ def renew(cli_config, plugins): continue # ?? config = configuration.NamespaceConfig(_AttrDict(renewalparams)) # webroot_map is, uniquely, a dict - if "webroot_map" in renewalparams: + if "webroot_map" in renewalparams and not _diff_from_default(default_conf, cli_config, "webroot_map"): config.__setattr__("webroot_map", renewalparams["webroot_map"]) # XXX: also need: nginx_, apache_, and plesk_ items # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: - if config_item in renewalparams: + #TODO make sure that we don't lose passed command line args if they aren't in renewal params? + if config_item in renewalparams and not _diff_from_default(default_conf, cli_config, config_item): value = renewalparams[config_item] # Unfortunately, we've lost type information from ConfigObj, # so we don't know if the original was NoneType or str! @@ -784,7 +798,7 @@ def renew(cli_config, plugins): config.__setattr__(config_item, value) # int-valued items to add if they're present for config_item in INT_CONFIG_ITEMS: - if config_item in renewalparams: + if config_item in renewalparams and not _diff_from_default(default_conf, cli_config, config_item): try: value = int(renewalparams[config_item]) config.__setattr__(config_item, value) @@ -797,7 +811,7 @@ def renew(cli_config, plugins): # XXX: is it true that an item will end up in _parser._actions even # when no action was explicitly specified? for plugin_prefix in EXTRACT_PLUGIN_PREFIXES: - for config_item in renewalparams.keys(): + for config_item in renewalparams.keys() and not _diff_from_default(default_conf, cli_config, config_item): if config_item.startswith(plugin_prefix): for action in _parser.parser._actions: if action.dest == config_item: From a706f5c8c086944185111a99999e03d1083f9a9a Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 3 Feb 2016 13:53:42 -0800 Subject: [PATCH 041/208] Try to add the new renew verb to integration testing --- tests/boulder-integration.sh | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 53996cd20..5d9ed4859 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -44,20 +44,21 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -# the following assumes that Boulder issues certificates for less than -# 10 years, otherwise renewal will not take place -cat < "$root/conf/renewer.conf" -renew_before_expiry = 10 years -deploy_before_expiry = 10 years -EOF -letsencrypt-renewer $store_flags -dir="$root/conf/archive/le1.wtf" -for x in cert chain fullchain privkey; -do - latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" - live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" - [ "${dir}/${latest}" = "$live" ] # renewer fails this test -done +# This won't renew (because it's not time yet) +common renew + +# This will renew +sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" +common renew + +# letsencrypt-renewer $store_flags +# dir="$root/conf/archive/le1.wtf" +# for x in cert chain fullchain privkey; +# do +# latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" +# live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" +# [ "${dir}/${latest}" = "$live" ] # renewer fails this test +# done # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" From 3c871cadbf897bd1687b281d9d61913b0f514f40 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 13:59:06 -0800 Subject: [PATCH 042/208] Be less spammy about donation messages --- letsencrypt/cli.py | 25 +++++++++++++++---------- letsencrypt/tests/cli_test.py | 7 ++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4c30e2ca8..597fb3731 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -406,14 +406,18 @@ def _report_new_cert(cert_path, fullchain_path): reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY) -def _suggest_donation_if_appropriate(config): +def _suggest_donation_if_appropriate(config, action): """Potentially suggest a donation to support Let's Encrypt.""" - if not config.staging and not config.verb == "renew": # --dry-run implies --staging - reporter_util = zope.component.getUtility(interfaces.IReporter) - msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" - "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" - "Donating to EFF: https://eff.org/donate-le\n\n") - reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) + if config.staging or config.verb == "renew": + # --dry-run implies --staging + return + if action not in ["renew", "newcert"]: + return + reporter_util = zope.component.getUtility(interfaces.IReporter) + msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" + "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" + "Donating to EFF: https://eff.org/donate-le\n\n") + reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) def _report_successful_dry_run(): @@ -666,7 +670,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals else: display_ops.success_renewal(domains, action) - _suggest_donation_if_appropriate(config) + _suggest_donation_if_appropriate(config, action) def obtain_cert(config, plugins, lineage=None): @@ -686,6 +690,7 @@ def obtain_cert(config, plugins, lineage=None): # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) + action = "newcert" # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" @@ -700,11 +705,11 @@ def obtain_cert(config, plugins, lineage=None): _report_new_cert(cert_path, cert_fullchain) else: domains = _find_domains(config, installer) - _auth_from_domains(le_client, config, domains, lineage) + _, action = _auth_from_domains(le_client, config, domains, lineage) if config.dry_run: _report_successful_dry_run() - _suggest_donation_if_appropriate(config) + _suggest_donation_if_appropriate(config, action) def install(config, plugins): diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f083018b3..c414d9054 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -233,7 +233,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") with mock.patch("letsencrypt.cli._init_le_client") as mock_init: - with mock.patch("letsencrypt.cli._auth_from_domains"): + with mock.patch("letsencrypt.cli._auth_from_domains") as mock_afd: + mock_afd.return_value = (mock.MagicMock(), mock.MagicMock()) self._call(["certonly", "--manual", "-d", "foo.bar"]) unused_config, auth, unused_installer = mock_init.call_args[0] self.assertTrue(isinstance(auth, manual.Authenticator)) @@ -598,8 +599,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._call(['-d', 'foo.bar', '-a', 'standalone', 'certonly']) self.assertFalse(mock_client.obtain_certificate.called) self.assertFalse(mock_client.obtain_and_enroll_certificate.called) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) + self.assertEqual(mock_get_utility().add_message.call_count, 0) + #self.assertTrue('donate' not in mock_get_utility().add_message.call_args[0][0]) def _test_certonly_csr_common(self, extra_args=None): certr = 'certr' From 72aa72ec748f637ba61aeecc3e241bb8335896b9 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 14:07:25 -0800 Subject: [PATCH 043/208] move inside for loop and check domains as well --- letsencrypt/cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index fa9e77d94..2812b1592 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -811,8 +811,8 @@ def renew(cli_config, plugins): # XXX: is it true that an item will end up in _parser._actions even # when no action was explicitly specified? for plugin_prefix in EXTRACT_PLUGIN_PREFIXES: - for config_item in renewalparams.keys() and not _diff_from_default(default_conf, cli_config, config_item): - if config_item.startswith(plugin_prefix): + for config_item in renewalparams.keys(): + if config_item.startswith(plugin_prefix) and not _diff_from_default(default_conf, cli_config, config_item): for action in _parser.parser._actions: if action.dest == config_item: if action.type is not None: @@ -848,7 +848,8 @@ def renew(cli_config, plugins): "invalid. Skipping.", full_path) continue - config.__setattr__("domains", domains) + if not _diff_from_default(default_conf, cli_config, "domains"): + config.__setattr__("domains", domains) print("Trying...") print(obtain_cert(config, plugins, renewal_candidate)) From 31ddf0a3d87954573df15e78725e2c2ae61e32d1 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 14:48:54 -0800 Subject: [PATCH 044/208] include alt confs --- letsencrypt/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5ec9b08f6..6833349fb 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -738,7 +738,7 @@ def _diff_from_default(default_conf, cli_conf, value): else: return False -def _restore_required_config_elements(full_path, config, renewalparams): +def _restore_required_config_elements(full_path, config, renewalparams, cli_config, default_conf): # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: if config_item in renewalparams and not _diff_from_default(default_conf, cli_config, config_item): @@ -790,7 +790,7 @@ def _restore_required_config_elements(full_path, config, renewalparams): return True -def _reconstitute(full_path, config): +def _reconstitute(full_path, config, cli_config): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -802,6 +802,8 @@ def _reconstitute(full_path, config): :rtype: `storage.RenewableCert` or NoneType """ + default_args = prepare_and_parse_args(plugins, []) + default_conf = configuration.NamespaceConfig(default_args) try: renewal_candidate = storage.RenewableCert(full_path, config) except (errors.CertStorageError, IOError): @@ -820,7 +822,7 @@ def _reconstitute(full_path, config): # Now restore specific values along with their data types, if # those elements are present. try: - _restore_required_config_elements(full_path, config, renewalparams) + _restore_required_config_elements(full_path, config, renewalparams, cli_config, default_conf) except ValueError: # There was a data type error which has already been # logged. @@ -866,14 +868,12 @@ def renew(cli_config, plugins): # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) config.noninteractive_mode = True - default_args = prepare_and_parse_args(plugins, []) - default_conf = configuration.NamespaceConfig(default_args) full_path = os.path.join(configs_dir, renewal_file) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(full_path, config) + renewal_candidate = _reconstitute(full_path, config, cli_config) except Exception as e: # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " From 50470c91ffeb0ce26965aaf08218fd8e42e8cefd Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 15:04:31 -0800 Subject: [PATCH 045/208] make default_conf in renew --- letsencrypt/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6833349fb..ffea8faa5 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -790,7 +790,7 @@ def _restore_required_config_elements(full_path, config, renewalparams, cli_conf return True -def _reconstitute(full_path, config, cli_config): +def _reconstitute(full_path, config, cli_config, default_conf): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -802,8 +802,6 @@ def _reconstitute(full_path, config, cli_config): :rtype: `storage.RenewableCert` or NoneType """ - default_args = prepare_and_parse_args(plugins, []) - default_conf = configuration.NamespaceConfig(default_args) try: renewal_candidate = storage.RenewableCert(full_path, config) except (errors.CertStorageError, IOError): @@ -868,6 +866,8 @@ def renew(cli_config, plugins): # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) config.noninteractive_mode = True + default_args = prepare_and_parse_args(plugins, []) + default_conf = configuration.NamespaceConfig(default_args) full_path = os.path.join(configs_dir, renewal_file) # Note that this modifies config (to add back the configuration From cd1c04a4efb4b0cd0ea934da05645a79b96843e7 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 15:06:04 -0800 Subject: [PATCH 046/208] pass default_conf --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ffea8faa5..25b687886 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -873,7 +873,7 @@ def renew(cli_config, plugins): # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(full_path, config, cli_config) + renewal_candidate = _reconstitute(full_path, config, cli_config, default_conf) except Exception as e: # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " From 1c52e1982ccb580e5bc4a3e0d28902c4865fe7e0 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 3 Feb 2016 17:49:08 -0800 Subject: [PATCH 047/208] made test for renew verb --- .../letstest/scripts/test_renew_standalone.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 tests/letstest/scripts/test_renew_standalone.sh diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh new file mode 100755 index 000000000..955cb104a --- /dev/null +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x + +# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution + +# with curl, instance metadata available from EC2 metadata service: +#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) +#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) +#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) + +cd letsencrypt +./letsencrypt-auto certonly -v --standalone --debug \ + --text --agree-dev-preview --agree-tos \ + --renew-by-default --redirect \ + --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL + +./letsencrypt-auto renew --renew-by-default + +ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem +if [ $? -ne 0 ] ; then + FAIL=1 +fi From c152b452b253d7f17cdf3c9fa6a8c72e14f40852 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 18:40:50 -0800 Subject: [PATCH 048/208] Start testing the renew verb, plus other goodies: * --dry-run works with renew * test harness for renewal is now fairly usable * coverage on cli.py 80% -> 88% --- letsencrypt/cli.py | 34 +++++++------ letsencrypt/storage.py | 3 ++ letsencrypt/tests/cli_test.py | 49 ++++++++++++++----- .../testdata/archive/sample-renewal/cert1.pem | 28 +++++++++++ .../archive/sample-renewal/chain1.pem | 19 +++++++ .../archive/sample-renewal/fullchain1.pem | 47 ++++++++++++++++++ .../archive/sample-renewal/privkey1.pem | 28 +++++++++++ .../testdata/live/sample-renewal/cert.pem | 1 + .../testdata/live/sample-renewal/chain.pem | 1 + .../live/sample-renewal/fullchain.pem | 1 + .../testdata/live/sample-renewal/privkey.pem | 1 + 11 files changed, 184 insertions(+), 28 deletions(-) create mode 100644 letsencrypt/tests/testdata/archive/sample-renewal/cert1.pem create mode 100644 letsencrypt/tests/testdata/archive/sample-renewal/chain1.pem create mode 100644 letsencrypt/tests/testdata/archive/sample-renewal/fullchain1.pem create mode 100644 letsencrypt/tests/testdata/archive/sample-renewal/privkey1.pem create mode 120000 letsencrypt/tests/testdata/live/sample-renewal/cert.pem create mode 120000 letsencrypt/tests/testdata/live/sample-renewal/chain.pem create mode 120000 letsencrypt/tests/testdata/live/sample-renewal/fullchain.pem create mode 120000 letsencrypt/tests/testdata/live/sample-renewal/privkey.pem diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4c30e2ca8..f97e9b550 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -8,6 +8,7 @@ import argparse import atexit import copy import functools +import glob import json import logging import logging.handlers @@ -223,15 +224,12 @@ def _find_duplicative_certs(config, domains): # Verify the directory is there le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - for renewal_file in os.listdir(configs_dir): - if not renewal_file.endswith(".conf"): - continue + for renewal_file in _renewal_conf_files(config): try: - full_path = os.path.join(configs_dir, renewal_file) - candidate_lineage = storage.RenewableCert(full_path, cli_config) + candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) + logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file) + logger.info("Traceback was:\n%s", traceback.format_exc()) continue # TODO: Handle these differently depending on whether they are # expired or still valid? @@ -794,8 +792,8 @@ def _reconstitute(full_path, config): try: renewal_candidate = storage.RenewableCert(full_path, config) except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) + logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) + logger.info("Traceback was:\n%s", traceback.format_exc()) return None if "renewalparams" not in renewal_candidate.configuration: logger.warning("Renewal configuration file %s lacks " @@ -834,6 +832,11 @@ def _reconstitute(full_path, config): return renewal_candidate +def _renewal_conf_files(config): + """Return /path/to/*.conf in the renewal conf directory""" + return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + + def renew(cli_config, plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) @@ -845,8 +848,7 @@ def renew(cli_config, plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") - configs_dir = cli_config.renewal_configs_dir - for renewal_file in os.listdir(configs_dir): + for renewal_file in _renewal_conf_files(cli_config): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) @@ -854,16 +856,16 @@ def renew(cli_config, plugins): # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) config.noninteractive_mode = True - full_path = os.path.join(configs_dir, renewal_file) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(full_path, config) + renewal_candidate = _reconstitute(renewal_file, config) except Exception as e: # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " - "unexpected error: %s. Skipping.", full_path, e) + "unexpected error: %s. Skipping.", renewal_file, e) + logger.info("Traceback was:\n%s", traceback.format_exc()) continue if renewal_candidate is None: @@ -1063,9 +1065,9 @@ class HelpfulArgumentParser(object): parsed_args.server = constants.STAGING_URI if parsed_args.dry_run: - if self.verb != "certonly": + if self.verb not in ["certonly", "renew"]: raise errors.Error("--dry-run currently only works with the " - "'certonly' subcommand") + "'certonly' or 'renew' subcommands") parsed_args.break_my_certs = parsed_args.staging = True return parsed_args diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index e41805459..ae43c3e41 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -728,6 +728,9 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes target_version = self.next_free_version() archive = self.cli_config.archive_dir + # XXX if anyone ever moves a renewal configuration file, this will + # break... perhaps prefix should be the dirname of the previous + # cert.pem? prefix = os.path.join(archive, self.lineagename) target = dict( [(kind, diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f083018b3..721b38e9c 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -530,7 +530,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - def _test_renewal_common(self, due_for_renewal, extra_args, outstring, renew=True): + def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, + args=None, renew=True, out=False): cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) @@ -546,27 +547,33 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods mock_init.return_value = mock_client get_utility_path = 'letsencrypt.cli.zope.component.getUtility' with mock.patch(get_utility_path) as mock_get_utility: - with mock.patch('letsencrypt.cli.OpenSSL'): + with mock.patch('letsencrypt.cli.OpenSSL') as mock_ssl: + mock_latest = mock.MagicMock() + mock_latest.get_issuer.return_value = "Fake fake" + mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('letsencrypt.cli.crypto_util'): - args = ['-d', 'foo.bar', '-a', - 'standalone', 'certonly'] + if not args: + args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: args += extra_args - self._call(args) + if out: + self._call_stdout(args) + else: + self._call(args) - if outstring: + if log_out: with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(outstring in lf.read()) + self.assertTrue(log_out in lf.read()) if renew: - mock_client.obtain_certificate.assert_called_once_with(['foo.bar']) + mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) else: self.assertEqual(mock_client.obtain_certificate.call_count, 0) return mock_lineage, mock_get_utility def test_certonly_renewal(self): - lineage, get_utility = self._test_renewal_common(True, [], None) + lineage, get_utility = self._test_renewal_common(True, []) self.assertEqual(lineage.save_successor.call_count, 1) lineage.update_all_links_to.assert_called_once_with( lineage.latest_common_version()) @@ -577,17 +584,35 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_certonly_renewal_triggers(self): # --dry-run should force renewal _, get_utility = self._test_renewal_common(False, ['--dry-run', '--keep'], - "simulating renewal") + log_out="simulating renewal") self.assertEqual(get_utility().add_message.call_count, 1) self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) _, _ = self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], - "Auto-renewal forced") + log_out="Auto-renewal forced") self.assertEqual(get_utility().add_message.call_count, 1) _, _ = self._test_renewal_common(False, ['-tvv', '--debug', '--keep'], - "not yet due", renew=False) + log_out="not yet due", renew=False) + def _dump_log(self): + with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: + print "Logs:" + print lf.read() + + def test_renewal_verb(self): + + with open(test_util.vector_path('sample-renewal.conf')) as src: + # put the correct path for cert.pem, chain.pem etc in the renewal conf + renewal_conf = src.read().replace("MAGICDIR", test_util.vector_path()) + rd = os.path.join(self.config_dir, "renewal") + os.makedirs(rd) + rc = os.path.join(rd, "sample-renewal.conf") + with open(rc, "w") as dest: + dest.write(renewal_conf) + + self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], + renew=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') diff --git a/letsencrypt/tests/testdata/archive/sample-renewal/cert1.pem b/letsencrypt/tests/testdata/archive/sample-renewal/cert1.pem new file mode 100644 index 000000000..4010000ef --- /dev/null +++ b/letsencrypt/tests/testdata/archive/sample-renewal/cert1.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIE1DCCA7ygAwIBAgITAPoz/CBluNQV/Eh9F+CS6dSxEDANBgkqhkiG9w0BAQsF +ADAfMR0wGwYDVQQDDBRoYXBweSBoYWNrZXIgZmFrZSBDQTAeFw0xNjAyMDIyMzQ5 +MDBaFw0xNjA1MDIyMzQ5MDBaMBQxEjAQBgNVBAMTCWlzbm90Lm9yZzCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALyudqLKcIdWZ5VaK1fuhlEDbZtvs2E+ +slm4dmSS1nFve7MdlZ69K0gdtnhkiPQ0wGQTligeDZ8fY8iL87GZO0tp5f7S+QJN +NYCiYw6j4qp5JBy/zG22kJz1Quu7/vXMYLzLvK6x6YixiWAWyqqvlUVBLS1r4W3h +A5Z+F1EIsXeyz7TJe3lAzIWAAxpfH9OviIz2rEDotuCdU771USLLNSw4qJojNlTx +UpZG6lGFs8KGb8tqROXknaMKE4PvN3SITixSUTFbktt1Wz60moWbNdLMKvgkzuUP +r4viO2P4SO5slNAY0ZeEssPpVAelN3EvrAcEZtoKmG5fnQDVo8uVag0CAwEAAaOC +AhIwggIOMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUqhI4u6aaPrcYQnmypxV8Tap8 +L54wHwYDVR0jBBgwFoAU+3hPEvlgFYMsnxd/NBmzLjbqQYkweAYIKwYBBQUHAQEE +bDBqMDMGCCsGAQUFBzABhidodHRwOi8vb2NzcC5zdGFnaW5nLXgxLmxldHNlbmNy +eXB0Lm9yZy8wMwYIKwYBBQUHMAKGJ2h0dHA6Ly9jZXJ0LnN0YWdpbmcteDEubGV0 +c2VuY3J5cHQub3JnLzAUBgNVHREEDTALgglpc25vdC5vcmcwgf4GA1UdIASB9jCB +8zAIBgZngQwBAgEwgeYGCysGAQQBgt8TAQEBMIHWMCYGCCsGAQUFBwIBFhpodHRw +Oi8vY3BzLmxldHNlbmNyeXB0Lm9yZzCBqwYIKwYBBQUHAgIwgZ4MgZtUaGlzIENl +cnRpZmljYXRlIG1heSBvbmx5IGJlIHJlbGllZCB1cG9uIGJ5IFJlbHlpbmcgUGFy +dGllcyBhbmQgb25seSBpbiBhY2NvcmRhbmNlIHdpdGggdGhlIENlcnRpZmljYXRl +IFBvbGljeSBmb3VuZCBhdCBodHRwczovL2xldHNlbmNyeXB0Lm9yZy9yZXBvc2l0 +b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAbAhX6FfQwELayneY4l5RvYSdw/Jj5CRy +KzrM7ISld7x9YPpxX6Pmht/YyMhLWrtxvFUR2+RNhSIYB8IjQEjmKjvR7UNeiUve +jzPEAuTg/9m3i0FJpPHc2aKGzlLFQCMm5/RrvnXI6ljIcyhocLvMiN46iexcExI2 +Ese3w8GoH6wARYKxU/QBexfoXQLgtAbYzNRE6EgKWtB+txV+7+d2MgbhCEit5VwU ++ydT8inp9URsA7iKM03hDdGOBysddkrm1/yEhVy/Oo6bT9WMAUHVvz61hHekWcSf +rAQ6BayubvWOUx06eTowXr1gln/rl+WXOxcsJeag127NuhmHOCXZxQ== +-----END CERTIFICATE----- diff --git a/letsencrypt/tests/testdata/archive/sample-renewal/chain1.pem b/letsencrypt/tests/testdata/archive/sample-renewal/chain1.pem new file mode 100644 index 000000000..760417fe9 --- /dev/null +++ b/letsencrypt/tests/testdata/archive/sample-renewal/chain1.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfmgAwIBAgIJAJzxkS6o1QkIMA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV +BAMMFGhhcHB5IGhhY2tlciBmYWtlIENBMB4XDTE1MDQwNzIzNTAzOFoXDTI1MDQw +NDIzNTAzOFowHzEdMBsGA1UEAwwUaGFwcHkgaGFja2VyIGZha2UgQ0EwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCCkd5mgXFErJ3F2M0E9dw+Ta/md5i +8TDId01HberAApqmydG7UZYF3zLTSzNjlNSOmtybvrSGUnZ9r9tSQcL8VM6WUOM8 +tnIpiIjEA2QkBycMwvRmZ/B2ltPdYs/R9BqNwO1g18GDZrHSzUYtNKNeFI6Glamj +7GK2Vr0SmiEamlNIR5ktAFsEErzf/d4jCF7sosMsJpMCm1p58QkP4LHLShVLXDa8 +BMfVoI+ipYcA08iNUFkgW8VWDclIDxcysa0psDDtMjX3+4aPkE/cefmP+1xOfUuD +HOGV8XFynsP4EpTfVOZr0/g9gYQ7ZArqXX7GTQkFqduwPm/w5qxSPTarAgMBAAGj +UDBOMB0GA1UdDgQWBBT7eE8S+WAVgyyfF380GbMuNupBiTAfBgNVHSMEGDAWgBT7 +eE8S+WAVgyyfF380GbMuNupBiTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQAd9Da+Zv+TjMv7NTAmliqnWHY6d3UxEZN3hFEJ58IQVHbBZVZdW7zhRktB +vR05Kweac0HJeK91TKmzvXl21IXLvh0gcNLU/uweD3no/snfdB4OoFompljThmgl +zBqiqWoKBJQrLCA8w5UB+ReomRYd/EYXF/6TAfzm6hr//Xt5mPiUHPdvYt75lMAo +vRxLSbF8TSQ6b7BYxISWjPgFASNNqJNHEItWsmQMtAjjwzb9cs01XH9pChVAWn9L +oeMKa+SlHSYrWG93+EcrIH/dGU76uNOiaDzBSKvaehG53h25MHuO1anNICJvZovW +rFo4Uv1EnkKJm3vJFe50eJGhEKlx +-----END CERTIFICATE----- diff --git a/letsencrypt/tests/testdata/archive/sample-renewal/fullchain1.pem b/letsencrypt/tests/testdata/archive/sample-renewal/fullchain1.pem new file mode 100644 index 000000000..6e24d6038 --- /dev/null +++ b/letsencrypt/tests/testdata/archive/sample-renewal/fullchain1.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIE1DCCA7ygAwIBAgITAPoz/CBluNQV/Eh9F+CS6dSxEDANBgkqhkiG9w0BAQsF +ADAfMR0wGwYDVQQDDBRoYXBweSBoYWNrZXIgZmFrZSBDQTAeFw0xNjAyMDIyMzQ5 +MDBaFw0xNjA1MDIyMzQ5MDBaMBQxEjAQBgNVBAMTCWlzbm90Lm9yZzCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBALyudqLKcIdWZ5VaK1fuhlEDbZtvs2E+ +slm4dmSS1nFve7MdlZ69K0gdtnhkiPQ0wGQTligeDZ8fY8iL87GZO0tp5f7S+QJN +NYCiYw6j4qp5JBy/zG22kJz1Quu7/vXMYLzLvK6x6YixiWAWyqqvlUVBLS1r4W3h +A5Z+F1EIsXeyz7TJe3lAzIWAAxpfH9OviIz2rEDotuCdU771USLLNSw4qJojNlTx +UpZG6lGFs8KGb8tqROXknaMKE4PvN3SITixSUTFbktt1Wz60moWbNdLMKvgkzuUP +r4viO2P4SO5slNAY0ZeEssPpVAelN3EvrAcEZtoKmG5fnQDVo8uVag0CAwEAAaOC +AhIwggIOMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB +BQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUqhI4u6aaPrcYQnmypxV8Tap8 +L54wHwYDVR0jBBgwFoAU+3hPEvlgFYMsnxd/NBmzLjbqQYkweAYIKwYBBQUHAQEE +bDBqMDMGCCsGAQUFBzABhidodHRwOi8vb2NzcC5zdGFnaW5nLXgxLmxldHNlbmNy +eXB0Lm9yZy8wMwYIKwYBBQUHMAKGJ2h0dHA6Ly9jZXJ0LnN0YWdpbmcteDEubGV0 +c2VuY3J5cHQub3JnLzAUBgNVHREEDTALgglpc25vdC5vcmcwgf4GA1UdIASB9jCB +8zAIBgZngQwBAgEwgeYGCysGAQQBgt8TAQEBMIHWMCYGCCsGAQUFBwIBFhpodHRw +Oi8vY3BzLmxldHNlbmNyeXB0Lm9yZzCBqwYIKwYBBQUHAgIwgZ4MgZtUaGlzIENl +cnRpZmljYXRlIG1heSBvbmx5IGJlIHJlbGllZCB1cG9uIGJ5IFJlbHlpbmcgUGFy +dGllcyBhbmQgb25seSBpbiBhY2NvcmRhbmNlIHdpdGggdGhlIENlcnRpZmljYXRl +IFBvbGljeSBmb3VuZCBhdCBodHRwczovL2xldHNlbmNyeXB0Lm9yZy9yZXBvc2l0 +b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAbAhX6FfQwELayneY4l5RvYSdw/Jj5CRy +KzrM7ISld7x9YPpxX6Pmht/YyMhLWrtxvFUR2+RNhSIYB8IjQEjmKjvR7UNeiUve +jzPEAuTg/9m3i0FJpPHc2aKGzlLFQCMm5/RrvnXI6ljIcyhocLvMiN46iexcExI2 +Ese3w8GoH6wARYKxU/QBexfoXQLgtAbYzNRE6EgKWtB+txV+7+d2MgbhCEit5VwU ++ydT8inp9URsA7iKM03hDdGOBysddkrm1/yEhVy/Oo6bT9WMAUHVvz61hHekWcSf +rAQ6BayubvWOUx06eTowXr1gln/rl+WXOxcsJeag127NuhmHOCXZxQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDETCCAfmgAwIBAgIJAJzxkS6o1QkIMA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV +BAMMFGhhcHB5IGhhY2tlciBmYWtlIENBMB4XDTE1MDQwNzIzNTAzOFoXDTI1MDQw +NDIzNTAzOFowHzEdMBsGA1UEAwwUaGFwcHkgaGFja2VyIGZha2UgQ0EwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCCkd5mgXFErJ3F2M0E9dw+Ta/md5i +8TDId01HberAApqmydG7UZYF3zLTSzNjlNSOmtybvrSGUnZ9r9tSQcL8VM6WUOM8 +tnIpiIjEA2QkBycMwvRmZ/B2ltPdYs/R9BqNwO1g18GDZrHSzUYtNKNeFI6Glamj +7GK2Vr0SmiEamlNIR5ktAFsEErzf/d4jCF7sosMsJpMCm1p58QkP4LHLShVLXDa8 +BMfVoI+ipYcA08iNUFkgW8VWDclIDxcysa0psDDtMjX3+4aPkE/cefmP+1xOfUuD +HOGV8XFynsP4EpTfVOZr0/g9gYQ7ZArqXX7GTQkFqduwPm/w5qxSPTarAgMBAAGj +UDBOMB0GA1UdDgQWBBT7eE8S+WAVgyyfF380GbMuNupBiTAfBgNVHSMEGDAWgBT7 +eE8S+WAVgyyfF380GbMuNupBiTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQAd9Da+Zv+TjMv7NTAmliqnWHY6d3UxEZN3hFEJ58IQVHbBZVZdW7zhRktB +vR05Kweac0HJeK91TKmzvXl21IXLvh0gcNLU/uweD3no/snfdB4OoFompljThmgl +zBqiqWoKBJQrLCA8w5UB+ReomRYd/EYXF/6TAfzm6hr//Xt5mPiUHPdvYt75lMAo +vRxLSbF8TSQ6b7BYxISWjPgFASNNqJNHEItWsmQMtAjjwzb9cs01XH9pChVAWn9L +oeMKa+SlHSYrWG93+EcrIH/dGU76uNOiaDzBSKvaehG53h25MHuO1anNICJvZovW +rFo4Uv1EnkKJm3vJFe50eJGhEKlx +-----END CERTIFICATE----- diff --git a/letsencrypt/tests/testdata/archive/sample-renewal/privkey1.pem b/letsencrypt/tests/testdata/archive/sample-renewal/privkey1.pem new file mode 100644 index 000000000..f03fdd0a3 --- /dev/null +++ b/letsencrypt/tests/testdata/archive/sample-renewal/privkey1.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC8rnaiynCHVmeV +WitX7oZRA22bb7NhPrJZuHZkktZxb3uzHZWevStIHbZ4ZIj0NMBkE5YoHg2fH2PI +i/OxmTtLaeX+0vkCTTWAomMOo+KqeSQcv8xttpCc9ULru/71zGC8y7yusemIsYlg +Fsqqr5VFQS0ta+Ft4QOWfhdRCLF3ss+0yXt5QMyFgAMaXx/Tr4iM9qxA6LbgnVO+ +9VEiyzUsOKiaIzZU8VKWRupRhbPChm/LakTl5J2jChOD7zd0iE4sUlExW5LbdVs+ +tJqFmzXSzCr4JM7lD6+L4jtj+EjubJTQGNGXhLLD6VQHpTdxL6wHBGbaCphuX50A +1aPLlWoNAgMBAAECggEAfKKWFWS6PnwSAnNErFoQeZVVItb/XB5JO8EA2+CvLNFi +mefR/MCixYlzDkYCvaXW7ISPrMJlZxYaGNBx0oAQzfkPB2wfNqj/zY/29SXGxast +8puzk0mEb1oHsaZGfeFaiXvfkFpPlI8J2uJTT7qaVNv/1sArciSv9QonpsyiRhlB +yqT49juNVoR1tJHyXzkkRfHKTG8OlJd4kuFOl3fM9dTFPQ/ft0kTNAQ/B4SFvSwF +RJsbLbsbFGsUdV9ekE6UX6oWD/Ah707rvgtCyS0Bc+0O3t2EKwmm3RXPRUMHCVxE +bKdTxRB4etbjMVXMuVhB8Y4GbfrtMCy+qxZQ6znCAQKBgQDr7bcYAZVZp/nBMVB+ +lBO9w73J6lnEWm6bZ9728KlGAKETaRhxZQSi6TN6MWwNwnk6rinyz4uVwVr9ZRCs +WkB1TbvW0JNcWdr3YClwsKXAt8X22bjGe0LagDJHG6r1TPS+MdovOS2M6IMaxlbT +rzFhSJ8ojLX3tqnOsmc7YAFLjQKBgQDMu8E9hoJt82lQzOGrjHmGzGEu2GLx9WKO +e4nkj335kX6fIhMMqSXBFbTJZwXoYvk5J8ZnaARbYG0m5nxDCwRjX5HWa8q0B2Po +ta53w01sKKznzlPjUhsdhEthun7MCFfLZpgvcZ9xVzOXo3/Zfn2+RrsPSjrVDqBy +hj+k5mW4gQKBgHFWKf3LTO7cBdvsD8ou4mjn7nVgMi1kb/wR4wdnxzmMtdR4STi4 +GYkVVBhgQ5M8mDY7UoWFdH3FfCt8cI0Lcimn5ROl8RSNSeZKeL3c7lNtNRmHr/8R +WaVTrlOAlBjxFiWEF1dWNW6ah9jF7RIV+DfOxj6ZkhTk2CAmjfb1AMpFAoGABf96 +KdNG/vGipDtcYSo8ZTaXoke0nmISARqdb5TEnAsnKoJVDInoEUARi9T411YO9x2z +MlRZzFOG3xzhhxVLi53BKAcAaUXOJ4MrGVcfbYvDhQcGbiJ5qOO3UaWlEVUtPUhE +LR+nDCsB1+9yT2zlQi3QTSJflt5W1QQZ2TrmwAECgYEAvQ7+sTcHs1K9yKj7koEu +A19FbMA0IwvrVRcV/VqmlsoW6e6wW2YND+GtaDbKdD0aBPivqLJwpNFrsRA+W0iB +vzmML6sKhhL+j7tjSgq+iQdBkKz0j9PyReuhe9CRnljMmyun+4qKEk0KUvxBrjPY +Skn+ML18qyUoEPnmbpfHxCs= +-----END PRIVATE KEY----- diff --git a/letsencrypt/tests/testdata/live/sample-renewal/cert.pem b/letsencrypt/tests/testdata/live/sample-renewal/cert.pem new file mode 120000 index 000000000..e06effe40 --- /dev/null +++ b/letsencrypt/tests/testdata/live/sample-renewal/cert.pem @@ -0,0 +1 @@ +../../archive/sample-renewal/cert1.pem \ No newline at end of file diff --git a/letsencrypt/tests/testdata/live/sample-renewal/chain.pem b/letsencrypt/tests/testdata/live/sample-renewal/chain.pem new file mode 120000 index 000000000..71f665f29 --- /dev/null +++ b/letsencrypt/tests/testdata/live/sample-renewal/chain.pem @@ -0,0 +1 @@ +../../archive/sample-renewal/chain1.pem \ No newline at end of file diff --git a/letsencrypt/tests/testdata/live/sample-renewal/fullchain.pem b/letsencrypt/tests/testdata/live/sample-renewal/fullchain.pem new file mode 120000 index 000000000..0f06f077d --- /dev/null +++ b/letsencrypt/tests/testdata/live/sample-renewal/fullchain.pem @@ -0,0 +1 @@ +../../archive/sample-renewal/fullchain1.pem \ No newline at end of file diff --git a/letsencrypt/tests/testdata/live/sample-renewal/privkey.pem b/letsencrypt/tests/testdata/live/sample-renewal/privkey.pem new file mode 120000 index 000000000..5187eda6b --- /dev/null +++ b/letsencrypt/tests/testdata/live/sample-renewal/privkey.pem @@ -0,0 +1 @@ +../../archive/sample-renewal/privkey1.pem \ No newline at end of file From a659b07b4cd0cebec53cf294e7d3b531e18caca8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 18:44:27 -0800 Subject: [PATCH 049/208] Reininitialize plugins for every lineage --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4c30e2ca8..4361ed886 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -878,8 +878,8 @@ def renew(cli_config, plugins): # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(config, plugins, renewal_candidate) - + obtain_cert(config, plugins_disco.PluginsRegistry.find_all(), + renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" From 4d8dbc9d81de9f7d8d2d1209f53df4d815bb99d1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 18:55:06 -0800 Subject: [PATCH 050/208] Lint this entire monstrosity - Doing some of @schoen's refactoring homework for him :) --- letsencrypt/cli.py | 18 ++++++++++-------- letsencrypt/plugins/manual.py | 1 - letsencrypt/tests/cli_test.py | 8 +++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f97e9b550..b6bb34a68 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -746,6 +746,8 @@ def _restore_required_config_elements(full_path, config, renewalparams): "a non-numeric value for %s. Skipping.", full_path, config_item) raise + +def _restore_plugin_configs(config, renewalparams): # Now use parser to get plugin-prefixed items with correct types # XXX: the current approach of extracting only prefixed items # related to the actually-used installer and authenticator @@ -767,13 +769,12 @@ def _restore_required_config_elements(full_path, config, renewalparams): config.__setattr__(config_item, None) continue if config_item.startswith(plugin_prefix + "_"): - for action in _parser.parser._actions: - if action.dest == config_item: - if action.type is not None: - config.__setattr__(config_item, action.type(renewalparams[config_item])) - break + for action in _parser.parser._actions: # pylint: disable=protected-access + if action.type is not None and action.dest == config_item: + config.__setattr__(config_item, action.type(renewalparams[config_item])) + break else: - config.__setattr__(config_item, str(renewalparams[config_item])) + config.__setattr__(config_item, str(renewalparams[config_item])) return True @@ -808,6 +809,7 @@ def _reconstitute(full_path, config): # those elements are present. try: _restore_required_config_elements(full_path, config, renewalparams) + _restore_plugin_configs(config, renewalparams) except ValueError: # There was a data type error which has already been # logged. @@ -861,7 +863,7 @@ def renew(cli_config, plugins): # elements from within the renewal configuration file). try: renewal_candidate = _reconstitute(renewal_file, config) - except Exception as e: + except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) @@ -1356,7 +1358,7 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) - global _parser + global _parser # pylint: disable=global-statement _parser = helpful return helpful.parse_args() diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 29f4639fe..54244db2a 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -93,7 +93,6 @@ s.serve_forever()" """ def prepare(self): # pylint: disable=missing-docstring,no-self-use if self.config.noninteractive_mode: raise errors.PluginError("Running manual mode non-interactively is not supported") - pass # pragma: no cover def more_info(self): # pylint: disable=missing-docstring,no-self-use return ("This plugin requires user's manual intervention in setting " diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 721b38e9c..46672973c 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -531,7 +531,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._certonly_new_request_common, mock_client) def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, - args=None, renew=True, out=False): + args=None, renew=True): + # pylint: disable=too-many-locals cert_path = 'letsencrypt/tests/testdata/cert.pem' chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path) @@ -556,10 +557,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: args += extra_args - if out: - self._call_stdout(args) - else: - self._call(args) + self._call(args) if log_out: with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: From 605979ce99599b93987addb24133a47b18f34d09 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 19:07:07 -0800 Subject: [PATCH 051/208] Revert "Avoid dangerous and mysterious behaviour if someone tries to modify a config" This reverts commit 83afb58a9a6099ad1e3c54097c2bb509e98f38f8. --- letsencrypt/configuration.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 72aabe548..04053c8c3 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -43,12 +43,6 @@ class NamespaceConfig(object): # Check command line parameters sanity, and error out in case of problem. check_config_sanity(self) - # We're done setting up the attic. Now pull up the ladder after ourselves... - self.__setattr__ = self.__setattr_implementation__ - - def __setattr_implementation__(self, var, value): - return self.namespace.__setattr__(var, value) - def __getattr__(self, name): return getattr(self.namespace, name) From 2762a541fffc672784b7d8685247f971080566cc Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 19:08:41 -0800 Subject: [PATCH 052/208] git add a missing file --- .../tests/testdata/sample-renewal.conf | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 letsencrypt/tests/testdata/sample-renewal.conf diff --git a/letsencrypt/tests/testdata/sample-renewal.conf b/letsencrypt/tests/testdata/sample-renewal.conf new file mode 100755 index 000000000..16778303a --- /dev/null +++ b/letsencrypt/tests/testdata/sample-renewal.conf @@ -0,0 +1,76 @@ +cert = MAGICDIR/live/sample-renewal/cert.pem +privkey = MAGICDIR/live/sample-renewal/privkey.pem +chain = MAGICDIR/live/sample-renewal/chain.pem +fullchain = MAGICDIR/live/sample-renewal/fullchain.pem +renew_before_expiry = 1 year + +# Options and defaults used in the renewal process +[renewalparams] +no_self_upgrade = False +apache_enmod = a2enmod +no_verify_ssl = False +ifaces = None +apache_dismod = a2dismod +register_unsafely_without_email = False +apache_handle_modules = True +uir = None +installer = none +nginx_ctl = nginx +config_dir = MAGICDIR +text_mode = False +func = +staging = True +prepare = False +work_dir = /var/lib/letsencrypt +tos = False +init = False +http01_port = 80 +duplicate = False +noninteractive_mode = True +key_path = None +nginx = False +nginx_server_root = /etc/nginx +fullchain_path = /home/ubuntu/letsencrypt/chain.pem +email = None +csr = None +agree_dev_preview = None +redirect = None +verb = certonly +verbose_count = -3 +config_file = None +renew_by_default = False +hsts = False +apache_handle_sites = True +authenticator = standalone +domains = isnot.org, +rsa_key_size = 2048 +apache_challenge_location = /etc/apache2 +checkpoints = 1 +manual_test_mode = False +apache = False +cert_path = /home/ubuntu/letsencrypt/cert.pem +webroot_path = None +reinstall = False +expand = False +strict_permissions = False +apache_server_root = /etc/apache2 +account = None +dry_run = False +manual_public_ip_logging_ok = False +chain_path = /home/ubuntu/letsencrypt/chain.pem +break_my_certs = False +standalone = True +manual = False +server = https://acme-staging.api.letsencrypt.org/directory +standalone_supported_challenges = "tls-sni-01,http-01" +webroot = False +os_packages_only = False +apache_init_script = None +user_agent = None +apache_le_vhost_ext = -le-ssl.conf +debug = False +tls_sni_01_port = 443 +logs_dir = /var/log/letsencrypt +apache_vhost_root = /etc/apache2/sites-available +configurator = None +[[webroot_map]] From ea76c07832b12166fb46ac7f9f13c60f2c3469df Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 19:09:40 -0800 Subject: [PATCH 053/208] s/config\./config\.namespace/ --- letsencrypt/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 385915750..caf6d677c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -734,13 +734,13 @@ def _restore_required_config_elements(full_path, config, renewalparams): # so we don't know if the original was NoneType or str! if value == "None": value = None - config.__setattr__(config_item, value) + config.namespace__setattr__(config_item, value) # int-valued items to add if they're present for config_item in INT_CONFIG_ITEMS: if config_item in renewalparams: try: value = int(renewalparams[config_item]) - config.__setattr__(config_item, value) + config.namespace__setattr__(config_item, value) except ValueError: logger.warning("Renewal configuration file %s specifies " "a non-numeric value for %s. Skipping.", @@ -766,15 +766,18 @@ def _restore_plugin_configs(config, renewalparams): # trying to read the file called "None") # Should we omit the item entirely rather than setting # its value to None? - config.__setattr__(config_item, None) + config.namespace__setattr__(config_item, None) continue if config_item.startswith(plugin_prefix + "_"): for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: - config.__setattr__(config_item, action.type(renewalparams[config_item])) + config.namespace__setattr__( + config_item, + action.type(renewalparams[config_item])) break else: - config.__setattr__(config_item, str(renewalparams[config_item])) + config.namespace__setattr__( + config_item, str(renewalparams[config_item])) return True @@ -819,7 +822,8 @@ def _reconstitute(full_path, config): # configuration restoring logic is not able to correctly parse it # from the serialized form. if "webroot_map" in renewalparams: - config.__setattr__("webroot_map", renewalparams["webroot_map"]) + config.namespace__setattr__( + "webroot_map", renewalparams["webroot_map"]) try: domains = [le_util.enforce_domain_sanity(x) for x in @@ -830,7 +834,7 @@ def _reconstitute(full_path, config): "invalid. Skipping.", full_path) return None - config.__setattr__("domains", domains) + config.namespace__setattr__("domains", domains) return renewal_candidate From 1536c8fca3252aeb4dcd28b0cd2c881b322c48a7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 19:29:27 -0800 Subject: [PATCH 054/208] Fix the things I broke --- letsencrypt/cli.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index caf6d677c..7911ba999 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -734,13 +734,13 @@ def _restore_required_config_elements(full_path, config, renewalparams): # so we don't know if the original was NoneType or str! if value == "None": value = None - config.namespace__setattr__(config_item, value) + setattr(config.namespace, config_item, value) # int-valued items to add if they're present for config_item in INT_CONFIG_ITEMS: if config_item in renewalparams: try: value = int(renewalparams[config_item]) - config.namespace__setattr__(config_item, value) + setattr(config.namespace, config_item, value) except ValueError: logger.warning("Renewal configuration file %s specifies " "a non-numeric value for %s. Skipping.", @@ -766,18 +766,17 @@ def _restore_plugin_configs(config, renewalparams): # trying to read the file called "None") # Should we omit the item entirely rather than setting # its value to None? - config.namespace__setattr__(config_item, None) + setattr(config.namespace, config_item, None) continue if config_item.startswith(plugin_prefix + "_"): for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: - config.namespace__setattr__( - config_item, - action.type(renewalparams[config_item])) + setattr(config.namespace, config_item, + action.type(renewalparams[config_item])) break else: - config.namespace__setattr__( - config_item, str(renewalparams[config_item])) + setattr(config.namespace, config_item, + str(renewalparams[config_item])) return True @@ -822,8 +821,7 @@ def _reconstitute(full_path, config): # configuration restoring logic is not able to correctly parse it # from the serialized form. if "webroot_map" in renewalparams: - config.namespace__setattr__( - "webroot_map", renewalparams["webroot_map"]) + setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) try: domains = [le_util.enforce_domain_sanity(x) for x in @@ -834,7 +832,7 @@ def _reconstitute(full_path, config): "invalid. Skipping.", full_path) return None - config.namespace__setattr__("domains", domains) + setattr(config.namespace, "domains", domains) return renewal_candidate @@ -843,7 +841,7 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def renew(cli_config, plugins): +def renew(cli_config, unused_plugins): """Renew previously-obtained certificates.""" cli_config = configuration.RenewerConfiguration(cli_config) if cli_config.domains != []: From 5e656122dee7767944b7212e750a3d16f877bd8c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 19:36:14 -0800 Subject: [PATCH 055/208] Use correct config --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 7911ba999..f4335c701 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -224,7 +224,7 @@ def _find_duplicative_certs(config, domains): # Verify the directory is there le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - for renewal_file in _renewal_conf_files(config): + for renewal_file in _renewal_conf_files(cli_config): try: candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): From 4d6a3dfdff02d8ea5078be7857b4395becdc9b9a Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 20:04:07 -0800 Subject: [PATCH 056/208] Apparently py26 can't deepcopy a MagicMock? --- letsencrypt/tests/cli_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f397c1081..3e6c050cf 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -609,8 +609,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with open(rc, "w") as dest: dest.write(renewal_conf) - self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], - renew=True) + with mock.patch('letsencrypt.cli.copy.deepcopy'): + self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], + renew=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From 139326db7a4e401c2e91342f04dc9d837dca5115 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 20:20:27 -0800 Subject: [PATCH 057/208] That didn't work :( --- letsencrypt/tests/cli_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 3e6c050cf..f397c1081 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -609,9 +609,8 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with open(rc, "w") as dest: dest.write(renewal_conf) - with mock.patch('letsencrypt.cli.copy.deepcopy'): - self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], - renew=True) + self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], + renew=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From 77e9f9f9b47efdd6101b55cbedaccfcab54873b5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 22:17:38 -0800 Subject: [PATCH 058/208] hack around horrible ancient py26 + deepcopy + mock issue --- letsencrypt/tests/cli_test.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index f397c1081..13470dbb2 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -559,14 +559,17 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args += extra_args self._call(args) - if log_out: - with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(log_out in lf.read()) + try: + if log_out: + with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: + self.assertTrue(log_out in lf.read()) - if renew: - mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) - else: - self.assertEqual(mock_client.obtain_certificate.call_count, 0) + if renew: + mock_client.obtain_certificate.assert_called_once_with(['isnot.org']) + else: + self.assertEqual(mock_client.obtain_certificate.call_count, 0) + except: + self._dump_log() return mock_lineage, mock_get_utility @@ -609,8 +612,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with open(rc, "w") as dest: dest.write(renewal_conf) - self._test_renewal_common(True, [], args=["renew", "--dry-run", "-tvv"], - renew=True) + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 + with mock.patch('letsencrypt.cli.copy.deepcopy', side_effect=lambda x: x) as hack: + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, renew=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From 94816f32a5972af13e4f83d6e650df441b669689 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 3 Feb 2016 22:36:30 -0800 Subject: [PATCH 059/208] Try this a different way --- letsencrypt/tests/cli_test.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 13470dbb2..fd00c4465 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -32,6 +32,9 @@ CERT = test_util.vector_path('cert.pem') CSR = test_util.vector_path('csr.der') KEY = test_util.vector_path('rsa256_key.pem') +def hack(x): + return x + class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods """Tests for different commands.""" @@ -601,7 +604,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods print "Logs:" print lf.read() - def test_renewal_verb(self): + + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 + @mock.patch('letsencrypt.cli.copy.deepcopy') + def test_renewal_verb(self, hack_copy): with open(test_util.vector_path('sample-renewal.conf')) as src: # put the correct path for cert.pem, chain.pem etc in the renewal conf @@ -611,12 +618,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods rc = os.path.join(rd, "sample-renewal.conf") with open(rc, "w") as dest: dest.write(renewal_conf) - - # Work around https://bugs.python.org/issue1515 for py26 tests :( :( - # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 - with mock.patch('letsencrypt.cli.copy.deepcopy', side_effect=lambda x: x) as hack: - args = ["renew", "--dry-run", "-tvv"] - self._test_renewal_common(True, [], args=args, renew=True) + hack_copy.side_effect = hack + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, renew=True) @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') From 8fdcb772d9047318c8b678a41af4eb8f8f095452 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 09:29:11 -0800 Subject: [PATCH 060/208] return failure --- tests/letstest/scripts/test_renew_standalone.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh index 955cb104a..b9f00efe0 100755 --- a/tests/letstest/scripts/test_renew_standalone.sh +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -20,3 +20,7 @@ ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem if [ $? -ne 0 ] ; then FAIL=1 fi + +if [ "$FAIL" = 1 ] ; then + exit 1 +fi From dc9a51b2e68c65ecda197842835d13e8cd94fbc3 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 09:38:45 -0800 Subject: [PATCH 061/208] make a robust test script --- .../letstest/scripts/test_renew_standalone.sh | 55 ++++++++++++++----- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh index b9f00efe0..d90ae9ab6 100755 --- a/tests/letstest/scripts/test_renew_standalone.sh +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -1,26 +1,55 @@ #!/bin/bash -x -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution - -# with curl, instance metadata available from EC2 metadata service: -#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution +# run letsencrypt-apache2 via letsencrypt-auto cd letsencrypt -./letsencrypt-auto certonly -v --standalone --debug \ - --text --agree-dev-preview --agree-tos \ - --renew-by-default --redirect \ - --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -./letsencrypt-auto renew --renew-by-default +export SUDO=sudo +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO bootstrap/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO bootstrap/_rpm_common.sh +else + echo "Dont have bootstrapping for this OS!" + exit 1 +fi -ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem +bootstrap/dev/venv.sh +sudo venv/bin/letsencrypt certonly --debug --standalone -t --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -v if [ $? -ne 0 ] ; then FAIL=1 fi +if [ "$OS_TYPE" = "ubuntu" ] ; then + venv/bin/tox -e apacheconftest +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +sudo venv/bin/letsencrypt renew --renew-by-default + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + +ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed if [ "$FAIL" = 1 ] ; then exit 1 fi From f623df772a9fd7d217d5de164183b153ee5e8548 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 10:04:12 -0800 Subject: [PATCH 062/208] Experimental solution to deepcopy py26 problems --- letsencrypt/cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f4335c701..ba282ce49 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -840,11 +840,15 @@ def _renewal_conf_files(config): """Return /path/to/*.conf in the renewal conf directory""" return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) +def _rc_from_config(config): + ns = copy.deepcopy(config.namespace) + new_config = configuration.NamespaceConfig(ns) + return configuration.RenewerConfiguration(new_config) def renew(cli_config, unused_plugins): """Renew previously-obtained certificates.""" - cli_config = configuration.RenewerConfiguration(cli_config) - if cli_config.domains != []: + config = _rc_from_config(cli_config) + if config.domains != []: raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " "to be renewed; individual domains cannot be " @@ -852,13 +856,13 @@ def renew(cli_config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") - for renewal_file in _renewal_conf_files(cli_config): + for renewal_file in _renewal_conf_files(config): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) + config = _rc_from_config(cli_config) config.noninteractive_mode = True # Note that this modifies config (to add back the configuration From ab2fed0e1d3c89273150a61adeddcfddf219fd05 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 10:20:05 -0800 Subject: [PATCH 063/208] Lint --- letsencrypt/tests/cli_test.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index fd00c4465..393531c6e 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -32,9 +32,6 @@ CERT = test_util.vector_path('cert.pem') CSR = test_util.vector_path('csr.der') KEY = test_util.vector_path('rsa256_key.pem') -def hack(x): - return x - class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods """Tests for different commands.""" @@ -573,6 +570,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(mock_client.obtain_certificate.call_count, 0) except: self._dump_log() + raise return mock_lineage, mock_get_utility @@ -618,7 +616,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods rc = os.path.join(rd, "sample-renewal.conf") with open(rc, "w") as dest: dest.write(renewal_conf) - hack_copy.side_effect = hack + hack_copy.side_effect = lambda x: x args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, renew=True) From 375543eb3208bd8171d1709000c1965d8a71b4d1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 14:43:05 -0800 Subject: [PATCH 064/208] Hoping to see if integration test is really renewing --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 5d9ed4859..8c5a93e39 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -51,7 +51,7 @@ common renew sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" common renew -# letsencrypt-renewer $store_flags +ls "$root/conf/archive/le1.wtf" # dir="$root/conf/archive/le1.wtf" # for x in cert chain fullchain privkey; # do From e14feb2919ecdda071e743137f2d6c0fb3a6cd6b Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 15:44:54 -0800 Subject: [PATCH 065/208] renew should imply noninteractive --- letsencrypt/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 3a68eab99..88c23c24b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -859,7 +859,6 @@ def renew(cli_config, unused_plugins): # XXX: does this succeed in making a fully independent config object # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) - config.noninteractive_mode = True # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). @@ -1700,6 +1699,8 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) + elif config.renew: + displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() zope.component.provideUtility(displayer) From e2e0dddaa4d31ed7676d17ff709c2f77b4e4f7b1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 16:04:53 -0800 Subject: [PATCH 066/208] Responding to comments (logger.debug, reject --csr in renew) --- letsencrypt/cli.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 3a68eab99..c43eeaadb 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -229,7 +229,7 @@ def _find_duplicative_certs(config, domains): candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file) - logger.info("Traceback was:\n%s", traceback.format_exc()) + logger.debug("Traceback was:\n%s", traceback.format_exc()) continue # TODO: Handle these differently depending on whether they are # expired or still valid? @@ -248,8 +248,10 @@ def _find_duplicative_certs(config, domains): def _treat_as_renewal(config, domains): - """Determine whether there are duplicated names and how to handle them - (renew, reinstall, newcert, or no action). + """Determine whether there are duplicated names and how to handle + them (renew, reinstall, newcert, or raising an error to stop + the client run if the user chooses to cancel the operation when + prompted). :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -852,6 +854,10 @@ def renew(cli_config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") + if cli_config.csr is not None: + raise errors.Error("Currently, the renew verb cannot be used when " + "specifying a CSR file. Please try the certonly " + "command instead.") for renewal_file in _renewal_conf_files(cli_config): if not renewal_file.endswith(".conf"): continue From 893918de00453a4552c64cc43eba1bdba14d8b66 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 16:07:56 -0800 Subject: [PATCH 067/208] Check for config.verb == "renew" rather than config.renew --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5443e0ac7..9036804cb 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1705,7 +1705,7 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) - elif config.renew: + elif config.verb == "renew": displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From 8dfb2a1d4c9ace1d8153a3d7a3b6c84916cba6d7 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 16:09:42 -0800 Subject: [PATCH 068/208] check verb --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 88c23c24b..570484f46 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1699,7 +1699,7 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) - elif config.renew: + elif config.verb == renew: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From 7dd1ea4dcfc2160e616af4fe27e628b61c46758e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 17:30:52 -0800 Subject: [PATCH 069/208] Kill this now plz --- letsencrypt/configuration.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 04053c8c3..37eaba3bd 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -84,15 +84,10 @@ class RenewerConfiguration(object): def __init__(self, namespace): self.namespace = namespace - # We're done setting up the attic. Now pull up the ladder after ourselves... - self.__setattr__ = self.__setattr_implementation__ def __getattr__(self, name): return getattr(self.namespace, name) - def __setattr_implementation__(self, var, value): - return self.namespace.__setattr__(var, value) - @property def archive_dir(self): # pylint: disable=missing-docstring return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR) From b2dae6cae27335276653ec6c466e1fe6a093cc00 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:10:39 -0800 Subject: [PATCH 070/208] Fixed it? --- letsencrypt/cli.py | 64 +++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 204327e13..e4f841d50 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -727,7 +727,15 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _restore_required_config_elements(full_path, config, renewalparams): +def _restore_required_config_elements(config, renewalparams): + """Sets non-plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: if config_item in renewalparams: @@ -744,12 +752,18 @@ def _restore_required_config_elements(full_path, config, renewalparams): value = int(renewalparams[config_item]) setattr(config.namespace, config_item, value) except ValueError: - logger.warning("Renewal configuration file %s specifies " - "a non-numeric value for %s. Skipping.", - full_path, config_item) - raise + raise errors.Error( + "Expected a numeric value for {0}".format(config_item)) def _restore_plugin_configs(config, renewalparams): + """Sets plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ # Now use parser to get plugin-prefixed items with correct types # XXX: the current approach of extracting only prefixed items # related to the actually-used installer and authenticator @@ -782,7 +796,7 @@ def _restore_plugin_configs(config, renewalparams): return True -def _reconstitute(full_path, config): +def _reconstitute(config, full_path): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -790,12 +804,18 @@ def _reconstitute(full_path, config): request. The config argument is modified by including relevant options read from the renewal configuration file. + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param str full_path: Absolute path to the configuration file that + defines this lineage + :returns: the RenewableCert object or None if a fatal error occurred :rtype: `storage.RenewableCert` or NoneType - """ + """ try: - renewal_candidate = storage.RenewableCert(full_path, config) + renewal_candidate = storage.RenewableCert( + full_path, configuration.RenewerConfiguration(config)) except (errors.CertStorageError, IOError): logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.info("Traceback was:\n%s", traceback.format_exc()) @@ -812,11 +832,12 @@ def _reconstitute(full_path, config): # Now restore specific values along with their data types, if # those elements are present. try: - _restore_required_config_elements(full_path, config, renewalparams) + _restore_required_config_elements(config, renewalparams) _restore_plugin_configs(config, renewalparams) - except ValueError: - # There was a data type error which has already been - # logged. + except (ValueError, errors.Error) as error: + logger.warning( + "An error occured while parsing %s. The error was %s. " + "Skipping the file.", full_path, error.message) return None # webroot_map is, uniquely, a dict, and the general-purpose @@ -843,10 +864,9 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def renew(cli_config, unused_plugins): +def renew(config, unused_plugins): """Renew previously-obtained certificates.""" - cli_config = configuration.RenewerConfiguration(cli_config) - if cli_config.domains != []: + if config.domains != []: raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " "to be renewed; individual domains cannot be " @@ -854,22 +874,24 @@ def renew(cli_config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") - if cli_config.csr is not None: + if config.csr is not None: raise errors.Error("Currently, the renew verb cannot be used when " "specifying a CSR file. Please try the certonly " "command instead.") - for renewal_file in _renewal_conf_files(cli_config): + renewer_config = configuration.RenewerConfiguration(config) + for renewal_file in _renewal_conf_files(renewer_config): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) + lineage_config = configuration.RenewerConfiguration( + copy.deepcopy(config)) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(renewal_file, config) + renewal_candidate = _reconstitute(renewal_file, lineage_config) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -882,14 +904,14 @@ def renew(cli_config, unused_plugins): # already been logged. Go on to the next config. continue # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(config) + zope.component.provideUtility(lineage_config) print("Trying...") # Because obtain_cert itself indirectly decides whether to renew # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(config, plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From 36a42d18304a502393a769e5fb37025abd6a5232 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:15:39 -0800 Subject: [PATCH 071/208] Tracebacks are useful --- letsencrypt/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e4f841d50..2838e8395 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -838,6 +838,7 @@ def _reconstitute(config, full_path): logger.warning( "An error occured while parsing %s. The error was %s. " "Skipping the file.", full_path, error.message) + logger.debug("Traceback was:\n%s", traceback.format_exc()) return None # webroot_map is, uniquely, a dict, and the general-purpose From b4f1d94d096e735cbaeca35a381977a7460fabbe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:21:36 -0800 Subject: [PATCH 072/208] less nesting + fixed argument order --- letsencrypt/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 2838e8395..370024a25 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -892,7 +892,7 @@ def renew(config, unused_plugins): # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(renewal_file, lineage_config) + renewal_candidate = _reconstitute(lineage_config, renewal_file) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -912,7 +912,8 @@ def renew(config, unused_plugins): # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config.namespace, + plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From 8c4721531886499f5d9f7c86590b706b56b74e66 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:28:10 -0800 Subject: [PATCH 073/208] More unnesting --- letsencrypt/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 370024a25..14298a645 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -886,8 +886,7 @@ def renew(config, unused_plugins): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - lineage_config = configuration.RenewerConfiguration( - copy.deepcopy(config)) + lineage_config = copy.deepcopy(config) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). @@ -912,8 +911,7 @@ def renew(config, unused_plugins): # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(lineage_config.namespace, - plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From 8933c51e22921f0c7597b455d82865fbe2171a8f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:32:39 -0800 Subject: [PATCH 074/208] Satisfied OCD by keeping comment capitalization consistent --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 14298a645..82bce0f21 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -732,7 +732,7 @@ def _restore_required_config_elements(config, renewalparams): :param configuration.NamespaceConfig config: configuration for the current lineage - :param configobj.Section renewalparams: Parameters from the renewal + :param configobj.Section renewalparams: parameters from the renewal configuration file that defines this lineage """ From 9cd0d5497f13d283cfb6eca0ab15a9e26deb09a8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 18:58:57 -0800 Subject: [PATCH 075/208] Remove older workarounds, comment newer ones --- letsencrypt/cli.py | 2 ++ letsencrypt/tests/cli_test.py | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 77ac9be37..471f4a2fd 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -865,6 +865,8 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def _copy_nsconfig(config): + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 ns = copy.deepcopy(config.namespace) new_config = configuration.NamespaceConfig(ns) return new_config diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 393531c6e..a5757399e 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -603,11 +603,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods print lf.read() - # Work around https://bugs.python.org/issue1515 for py26 tests :( :( - # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 - @mock.patch('letsencrypt.cli.copy.deepcopy') - def test_renewal_verb(self, hack_copy): - + def test_renewal_verb(self): with open(test_util.vector_path('sample-renewal.conf')) as src: # put the correct path for cert.pem, chain.pem etc in the renewal conf renewal_conf = src.read().replace("MAGICDIR", test_util.vector_path()) @@ -616,7 +612,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods rc = os.path.join(rd, "sample-renewal.conf") with open(rc, "w") as dest: dest.write(renewal_conf) - hack_copy.side_effect = lambda x: x args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, renew=True) From 9f57236bb124314a1c9f773e0fc0bc7bd3ed4830 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 19:14:36 -0800 Subject: [PATCH 076/208] Try things the EvenMoreProper way --- letsencrypt/cli.py | 8 +------- letsencrypt/configuration.py | 7 +++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 471f4a2fd..fd568afa2 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -864,12 +864,6 @@ def _renewal_conf_files(config): """Return /path/to/*.conf in the renewal conf directory""" return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def _copy_nsconfig(config): - # Work around https://bugs.python.org/issue1515 for py26 tests :( :( - # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 - ns = copy.deepcopy(config.namespace) - new_config = configuration.NamespaceConfig(ns) - return new_config def renew(config, unused_plugins): """Renew previously-obtained certificates.""" @@ -893,7 +887,7 @@ def renew(config, unused_plugins): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - lineage_config = _copy_nsconfig(config) + lineage_config = copy.deepcopy(config) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 37eaba3bd..2bbf1b019 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -1,4 +1,5 @@ """Let's Encrypt user-supplied configuration.""" +import copy import os import urlparse @@ -78,6 +79,12 @@ class NamespaceConfig(object): return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + def __deepcopy__(self, _memo): + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 + new_ns = copy.deepcopy(self.namespace) + return type(self)(new_ns) + class RenewerConfiguration(object): """Configuration wrapper for renewer.""" From 0b62495581327fced5a30b60306d8257b448ee15 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 19:25:50 -0800 Subject: [PATCH 077/208] Move noninteractivity to the place Brad suggests --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 7cfdbb9bb..141c19fe1 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -879,7 +879,6 @@ def renew(config, unused_plugins): raise errors.Error("Currently, the renew verb cannot be used when " "specifying a CSR file. Please try the certonly " "command instead.") - config.noninteractive_mode = True renewer_config = configuration.RenewerConfiguration(config) for renewal_file in _renewal_conf_files(renewer_config): if not renewal_file.endswith(".conf"): @@ -1729,6 +1728,7 @@ def main(cli_args=sys.argv[1:]): elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) elif config.verb == "renew": + config.noninteractive_mode = True displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From 3260efd519d77e4a056e1cfd6c21174c9edb2838 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 4 Feb 2016 19:31:05 -0800 Subject: [PATCH 078/208] Be consistent about using logger.debug for tracebacks --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 141c19fe1..1ab23ea76 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -818,7 +818,7 @@ def _reconstitute(config, full_path): full_path, configuration.RenewerConfiguration(config)) except (errors.CertStorageError, IOError): logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) - logger.info("Traceback was:\n%s", traceback.format_exc()) + logger.debug("Traceback was:\n%s", traceback.format_exc()) return None if "renewalparams" not in renewal_candidate.configuration: logger.warning("Renewal configuration file %s lacks " @@ -896,7 +896,7 @@ def renew(config, unused_plugins): # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) - logger.info("Traceback was:\n%s", traceback.format_exc()) + logger.debug("Traceback was:\n%s", traceback.format_exc()) continue if renewal_candidate is None: From acb4cbd43216425dbfbbb0796374b16c63b312c1 Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Fri, 5 Feb 2016 18:57:52 +0200 Subject: [PATCH 079/208] Fixing parameter type for get_authorizations domains parameter --- letsencrypt/auth_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index c63d8c8d4..45c51a020 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -57,7 +57,7 @@ class AuthHandler(object): def get_authorizations(self, domains, best_effort=False): """Retrieve all authorizations for challenges. - :param set domains: Domains for authorization + :param list domains: Domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) From 192c3faf7eb53b08ae9b6b97a9e821dfd1b5fdd4 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 5 Feb 2016 15:26:08 -0500 Subject: [PATCH 080/208] Make the new letsencrypt-auto script the main one. Remove the old bootstrap scripts, which have been subsumed into letsencrypt-auto-source/pieces/bootstrappers. They no longer need to be dispatched among manually: everyone can just run letsencrypt-auto --os-packages-only, regardless of OS. Make the root-level le-auto a symlink to the canonical version. It should thus still work for people running le-auto from a git checkout. --- .travis.yml | 2 +- Dockerfile | 4 +- Dockerfile-dev | 4 +- bootstrap/README | 6 - bootstrap/_arch_common.sh | 26 ---- bootstrap/_deb_common.sh | 94 ------------ bootstrap/_gentoo_common.sh | 23 --- bootstrap/_rpm_common.sh | 60 -------- bootstrap/_suse_common.sh | 14 -- bootstrap/archlinux.sh | 1 - bootstrap/centos.sh | 1 - bootstrap/debian.sh | 1 - bootstrap/fedora.sh | 1 - bootstrap/freebsd.sh | 7 - bootstrap/gentoo.sh | 1 - bootstrap/install-deps.sh | 46 ------ bootstrap/mac.sh | 18 --- bootstrap/manjaro.sh | 1 - bootstrap/suse.sh | 1 - bootstrap/ubuntu.sh | 1 - bootstrap/venv.sh | 33 ---- docs/using.rst | 19 +-- letsencrypt-auto | 203 +------------------------ tests/letstest/scripts/test_apache2.sh | 16 +- tests/letstest/scripts/test_tox.sh | 58 +------ 25 files changed, 14 insertions(+), 627 deletions(-) delete mode 100644 bootstrap/README delete mode 100755 bootstrap/_arch_common.sh delete mode 100755 bootstrap/_deb_common.sh delete mode 100755 bootstrap/_gentoo_common.sh delete mode 100755 bootstrap/_rpm_common.sh delete mode 100755 bootstrap/_suse_common.sh delete mode 120000 bootstrap/archlinux.sh delete mode 120000 bootstrap/centos.sh delete mode 120000 bootstrap/debian.sh delete mode 120000 bootstrap/fedora.sh delete mode 100755 bootstrap/freebsd.sh delete mode 120000 bootstrap/gentoo.sh delete mode 100755 bootstrap/install-deps.sh delete mode 100755 bootstrap/mac.sh delete mode 120000 bootstrap/manjaro.sh delete mode 120000 bootstrap/suse.sh delete mode 120000 bootstrap/ubuntu.sh delete mode 100755 bootstrap/venv.sh mode change 100755 => 120000 letsencrypt-auto diff --git a/.travis.yml b/.travis.yml index 719e95012..1d2f7b1db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ addons: apt: sources: - augeas - packages: # keep in sync with bootstrap/ubuntu.sh and Boulder + packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder. - python-dev - python-virtualenv - gcc diff --git a/Dockerfile b/Dockerfile index da0110604..71e217659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 61908d470..e4a22bea7 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto -RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/bootstrap/README b/bootstrap/README deleted file mode 100644 index d91780903..000000000 --- a/bootstrap/README +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains scripts that install necessary OS-specific -prerequisite dependencies (see docs/using.rst). - -General dependencies: -- ca-certificates: communication with demo ACMO server at - https://www.letsencrypt-demo.org diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh deleted file mode 100755 index 2b512792f..000000000 --- a/bootstrap/_arch_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Tested with: -# - ArchLinux (x86_64) -# -# "python-virtualenv" is Python3, but "python2-virtualenv" provides -# only "virtualenv2" binary, not "virtualenv" necessary in -# ./bootstrap/dev/_common_venv.sh - -deps=" - python2 - python-virtualenv - gcc - dialog - augeas - openssl - libffi - ca-certificates - pkg-config -" - -missing=$(pacman -T $deps) - -if [ "$missing" ]; then - pacman -S --needed $missing -fi diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh deleted file mode 100755 index c2f58db75..000000000 --- a/bootstrap/_deb_common.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -# Current version tested with: -# -# - Ubuntu -# - 14.04 (x64) -# - 15.04 (x64) -# - Debian -# - 7.9 "wheezy" (x64) -# - sid (2015-10-21) (x64) - -# Past versions tested with: -# -# - Debian 8.0 "jessie" (x64) -# - Raspbian 7.8 (armhf) - -# Believed not to work: -# -# - Debian 6.0.10 "squeeze" (x64) - -apt-get update - -# virtualenv binary can be found in different packages depending on -# distro version (#346) - -virtualenv= -if apt-cache show virtualenv > /dev/null 2>&1; then - virtualenv="virtualenv" -fi - -if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" -fi - -augeas_pkg="libaugeas0 augeas-lenses" -AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` - -AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - fi - - echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list - apt-get update - fi - fi - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - -} - - -if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Let's Encrypt apache plugin..." - fi - # XXX add a case for ubuntu PPAs -fi - -apt-get install -y --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - dialog \ - $augeas_pkg \ - libssl-dev \ - libffi-dev \ - ca-certificates \ - - - -if ! command -v virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting - exit 1 -fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh deleted file mode 100755 index f49dc00f0..000000000 --- a/bootstrap/_gentoo_common.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - dev-util/dialog - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - -case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace $PACKAGES - ;; - (portage|*) - emerge --noreplace $PACKAGES - ;; -esac diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh deleted file mode 100755 index 73890155e..000000000 --- a/bootstrap/_rpm_common.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# Tested with: -# - Fedora 22, 23 (x64) -# - Centos 7 (x64: on DigitalOcean droplet) -# - CentOS 7 Minimal install in a Hyper-V VM - -if type dnf 2>/dev/null -then - tool=dnf -elif type yum 2>/dev/null -then - tool=yum - -else - echo "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 -fi - -# Some distros and older versions of current distros use a "python27" -# instead of "python" naming convention. Try both conventions. -if ! $tool install -y \ - python \ - python-devel \ - python-virtualenv \ - python-tools \ - python-pip -then - if ! $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv \ - python27-tools \ - python27-pip - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi -fi - -if ! $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates -then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 -fi - - -if $tool list installed "httpd" >/dev/null 2>&1; then - if ! $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi -fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh deleted file mode 100755 index efeebe4f8..000000000 --- a/bootstrap/_suse_common.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# SLE12 don't have python-virtualenv - -zypper -nq in -l \ - python \ - python-devel \ - python-virtualenv \ - gcc \ - dialog \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates \ diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/archlinux.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/centos.sh b/bootstrap/centos.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/centos.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/debian.sh b/bootstrap/debian.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/debian.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/fedora.sh b/bootstrap/fedora.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/fedora.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh deleted file mode 100755 index 4482c35cd..000000000 --- a/bootstrap/freebsd.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -xe - -pkg install -Ay \ - python \ - py27-virtualenv \ - augeas \ - libffi \ diff --git a/bootstrap/gentoo.sh b/bootstrap/gentoo.sh deleted file mode 120000 index 125d6a592..000000000 --- a/bootstrap/gentoo.sh +++ /dev/null @@ -1 +0,0 @@ -_gentoo_common.sh \ No newline at end of file diff --git a/bootstrap/install-deps.sh b/bootstrap/install-deps.sh deleted file mode 100755 index e907e7035..000000000 --- a/bootstrap/install-deps.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -e -# -# Install OS dependencies. In the glorious future, letsencrypt-auto will -# source this... - -if test "`id -u`" -ne "0" ; then - SUDO=sudo -else - SUDO= -fi - -BOOTSTRAP=`dirname $0` -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/arch-release ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif [ -f /etc/gentoo-release ] ; then - echo "Bootstrapping dependencies for Gentoo-based OSes..." - $SUDO $BOOTSTRAP/_gentoo_common.sh -elif uname | grep -iq FreeBSD ; then - echo "Bootstrapping dependencies for FreeBSD..." - $SUDO $BOOTSTRAP/freebsd.sh -elif `grep -qs openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE.." - $SUDO $BOOTSTRAP/suse.sh -elif uname | grep -iq Darwin ; then - echo "Bootstrapping dependencies for Mac OS X..." - echo "WARNING: Mac support is very experimental at present..." - $BOOTSTRAP/mac.sh -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - exit 1 -fi diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh deleted file mode 100755 index 4d1fb8208..000000000 --- a/bootstrap/mac.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e -if ! hash brew 2>/dev/null; then - echo "Homebrew Not Installed\nDownloading..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -fi - -brew install augeas -brew install dialog - -if ! hash pip 2>/dev/null; then - echo "pip Not Installed\nInstalling python from Homebrew..." - brew install python -fi - -if ! hash virtualenv 2>/dev/null; then - echo "virtualenv Not Installed\nInstalling with pip" - pip install virtualenv -fi diff --git a/bootstrap/manjaro.sh b/bootstrap/manjaro.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/manjaro.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/suse.sh b/bootstrap/suse.sh deleted file mode 120000 index fc4c1dee4..000000000 --- a/bootstrap/suse.sh +++ /dev/null @@ -1 +0,0 @@ -_suse_common.sh \ No newline at end of file diff --git a/bootstrap/ubuntu.sh b/bootstrap/ubuntu.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/ubuntu.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh deleted file mode 100755 index 5042178d9..000000000 --- a/bootstrap/venv.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -e -# -# Installs and updates letencrypt virtualenv -# -# USAGE: source ./dev/venv.sh - - -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - virtualenv --no-site-packages --python ${LE_PYTHON:-python2} $VENV_PATH -fi - -. $VENV_PATH/bin/activate -pip install -U setuptools -pip install -U pip - -pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx - -echo -echo "Congratulations, Let's Encrypt has been successfully installed/updated!" -echo -printf "%s" "Your prompt should now be prepended with ($VENV_NAME). Next " -printf "time, if the prompt is different, 'source' this script again " -printf "before running 'letsencrypt'." -echo -echo -echo "You can now run 'letsencrypt --help'." diff --git a/docs/using.rst b/docs/using.rst index eb7c3962e..ebc3ef6ac 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -16,27 +16,12 @@ letsencrypt-auto ---------------- ``letsencrypt-auto`` is a wrapper which installs some dependencies -from your OS standard package repositories (e.g using `apt-get` or +from your OS standard package repositories (e.g. using `apt-get` or `yum`), and for other dependencies it sets up a virtualized Python environment with packages downloaded from PyPI [#venv]_. It also provides automated updates. -Firstly, please `install Git`_ and run the following commands: - -.. code-block:: shell - - git clone https://github.com/letsencrypt/letsencrypt - cd letsencrypt - - -.. _`install Git`: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git - -.. note:: On RedHat/CentOS 6 you will need to enable the EPEL_ - repository before install. - -.. _EPEL: http://fedoraproject.org/wiki/EPEL - -To install and run the client you just need to type: +To install and run the client, just type... .. code-block:: shell diff --git a/letsencrypt-auto b/letsencrypt-auto deleted file mode 100755 index 2b956aaf5..000000000 --- a/letsencrypt-auto +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/sh -e -# -# A script to run the latest release version of the Let's Encrypt in a -# virtual environment -# -# Installs and updates the letencrypt virtualenv, and runs letsencrypt -# using that virtual environment. This allows the client to function decently -# without requiring specific versions of its dependencies from the operating -# system. - -# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, -# if you want to change where the virtual environment will be installed -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} -VENV_BIN=${VENV_PATH}/bin -# The path to the letsencrypt-auto script. Everything that uses these might -# at some point be inlined... -LEA_PATH=`dirname "$0"` -BOOTSTRAP=${LEA_PATH}/bootstrap - -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) -for arg in "$@" ; do - # This first clause is redundant with the third, but hedging on portability - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then - VERBOSE=1 - elif [ "$arg" = "--debug" ] ; then - DEBUG=1 - fi -done - -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$DEBUG" = 1 ] ; then - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi - else - echo "WARNING: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - exit 1 - fi -} - -DeterminePythonVersion() { - if command -v python2.7 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2.7} - elif command -v python27 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python27} - elif command -v python2 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2} - elif command -v python > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python} - else - echo "Cannot find any Pythons... please install one!" - exit 1 - fi - - PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -lt 26 ] ; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." - exit 1 - fi -} - - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 - fi - - if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh - elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh - elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh - elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi - elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" - elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" - elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" - elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root - elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" - else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - fi - - DeterminePythonVersion - echo "Creating virtual environment..." - if [ "$VERBOSE" = 1 ] ; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH - else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null - fi -else - DeterminePythonVersion -fi - - -printf "Updating letsencrypt and virtual environment dependencies..." -if [ "$VERBOSE" = 1 ] ; then - echo - $VENV_BIN/pip install -U setuptools - $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache - # nginx is buggy / disabled for now, but upgrade it if the user has - # installed it manually - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - $VENV_BIN/pip install -U letsencrypt letsencrypt-nginx - fi -else - $VENV_BIN/pip install -U setuptools > /dev/null - printf . - $VENV_BIN/pip install -U pip > /dev/null - printf . - # nginx is buggy / disabled for now... - $VENV_BIN/pip install -U letsencrypt > /dev/null - printf . - $VENV_BIN/pip install -U letsencrypt-apache > /dev/null - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - printf . - $VENV_BIN/pip install -U letsencrypt-nginx > /dev/null - fi - echo -fi - -# Explain what's about to happen, for the benefit of those getting sudo -# password prompts... -echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 120000 index 000000000..af7e83a70 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1 @@ +letsencrypt-auto-source/letsencrypt-auto \ No newline at end of file diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4032e2195..178e3d4e8 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -35,19 +35,13 @@ then #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/ fi -# run letsencrypt-apache2 via letsencrypt-auto +# Run letsencrypt-apache2. cd letsencrypt -export SUDO=sudo -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO bootstrap/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO bootstrap/_rpm_common.sh -else - echo "Dont have bootstrapping for this OS!" - exit 1 +echo "Bootstrapping dependencies..." +letsencrypt-auto-source/letsencrypt-auto --os-packages-only +if [ $? -ne 0 ] ; then + exit 1 fi bootstrap/dev/venv.sh diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index f7f325d5c..98fe9b9ce 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -6,68 +6,12 @@ VENV_NAME="venv" LEA_PATH=./letsencrypt/ VENV_PATH=${LEA_PATH/$VENV_NAME} VENV_BIN=${VENV_PATH}/bin -BOOTSTRAP=${LEA_PATH}/bootstrap -SUDO=sudo - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi -} # virtualenv call is not idempotent: it overwrites pip upgraded in # later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh -elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi -elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" -elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" -elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" -elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root -elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" -fi -echo "Bootstrapped!" +"$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt ./bootstrap/dev/venv.sh From b7717bbc8e5e524f2972d616ca8df2fca75a2e7d Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 13:06:50 -0800 Subject: [PATCH 081/208] Fixes for comments from PR review --- letsencrypt/cli.py | 47 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1ab23ea76..e51490379 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -773,27 +773,25 @@ def _restore_plugin_configs(config, renewalparams): # XXX: is it true that an item will end up in _parser._actions even # when no action was explicitly specified? plugin_prefixes = [renewalparams["authenticator"]] - if "installer" in renewalparams and renewalparams["installer"] != None: + if renewalparams.get("installer", None) is not None: plugin_prefixes.append(renewalparams["installer"]) for plugin_prefix in set(renewalparams): - for config_item in renewalparams.keys(): - if renewalparams[config_item] == "None": + for config_item, config_value in renewalparams.iteritems(): + if config_item.startswith(plugin_prefix + "_"): # Avoid confusion when, for example, "csr = None" (avoid # trying to read the file called "None") # Should we omit the item entirely rather than setting # its value to None? - setattr(config.namespace, config_item, None) - continue - if config_item.startswith(plugin_prefix + "_"): + if config_value == "None": + setattr(config.namespace, config_item, None) + continue for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: setattr(config.namespace, config_item, - action.type(renewalparams[config_item])) + action.type(config_value)) break else: - setattr(config.namespace, config_item, - str(renewalparams[config_item])) - return True + setattr(config.namespace, config_item, str(config_value)) def _reconstitute(config, full_path): @@ -881,8 +879,6 @@ def renew(config, unused_plugins): "command instead.") renewer_config = configuration.RenewerConfiguration(config) for renewal_file in _renewal_conf_files(renewer_config): - if not renewal_file.endswith(".conf"): - continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? @@ -899,20 +895,21 @@ def renew(config, unused_plugins): logger.debug("Traceback was:\n%s", traceback.format_exc()) continue - if renewal_candidate is None: - # reconstitute indicated an error or problem which has - # already been logged. Go on to the next config. - continue - # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(lineage_config) + if renewal_candidate is not None: + # _reconstitute succeeded in producing a RenewableCert, so we + # have something to work with from this particular config file. + + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(lineage_config) + print("Trying...") + # Because obtain_cert itself indirectly decides whether to renew + # or not, we couldn't currently make a UI/logging distinction at + # this stage to indicate whether renewal was actually attempted + # (or successful). + obtain_cert(lineage_config, + plugins_disco.PluginsRegistry.find_all(), + renewal_candidate) - print("Trying...") - # Because obtain_cert itself indirectly decides whether to renew - # or not, we couldn't currently make a UI/logging distinction at - # this stage to indicate whether renewal was actually attempted - # (or successful). - obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), - renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" From 5c14c09027aba71ef02d7b93f7f9974498eadb3e Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 13:10:00 -0800 Subject: [PATCH 082/208] @bmw noticed we were iterating over the wrong thing! --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e51490379..88d16be73 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -775,7 +775,7 @@ def _restore_plugin_configs(config, renewalparams): plugin_prefixes = [renewalparams["authenticator"]] if renewalparams.get("installer", None) is not None: plugin_prefixes.append(renewalparams["installer"]) - for plugin_prefix in set(renewalparams): + for plugin_prefix in set(plugin_prefixes): for config_item, config_value in renewalparams.iteritems(): if config_item.startswith(plugin_prefix + "_"): # Avoid confusion when, for example, "csr = None" (avoid From 5c31b000b40566e2a5a552624ab82db96b02278f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 14:51:14 -0800 Subject: [PATCH 083/208] Error handling around obtain_cert() --- letsencrypt/cli.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 88d16be73..abe4ccc0c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -770,8 +770,10 @@ def _restore_plugin_configs(config, renewalparams): # works as long as plugins don't need to read plugin-specific # variables set by someone else (e.g., assuming Apache # configurator doesn't need to read webroot_ variables). - # XXX: is it true that an item will end up in _parser._actions even - # when no action was explicitly specified? + # Note: if a parameter that used to be defined in the parser is no + # longer defined, stored copies of that parameter will be + # deserialized as strings by this logic even if they were + # originally meant to be some other type. plugin_prefixes = [renewalparams["authenticator"]] if renewalparams.get("installer", None) is not None: plugin_prefixes.append(renewalparams["installer"]) @@ -895,20 +897,26 @@ def renew(config, unused_plugins): logger.debug("Traceback was:\n%s", traceback.format_exc()) continue - if renewal_candidate is not None: - # _reconstitute succeeded in producing a RenewableCert, so we - # have something to work with from this particular config file. + try: + if renewal_candidate is not None: + # _reconstitute succeeded in producing a RenewableCert, so we + # have something to work with from this particular config file. - # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(lineage_config) - print("Trying...") - # Because obtain_cert itself indirectly decides whether to renew - # or not, we couldn't currently make a UI/logging distinction at - # this stage to indicate whether renewal was actually attempted - # (or successful). - obtain_cert(lineage_config, - plugins_disco.PluginsRegistry.find_all(), - renewal_candidate) + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(lineage_config) + print("Trying...") + # Because obtain_cert itself indirectly decides whether to renew + # or not, we couldn't currently make a UI/logging distinction at + # this stage to indicate whether renewal was actually attempted + # (or successful). + obtain_cert(lineage_config, + plugins_disco.PluginsRegistry.find_all(), + renewal_candidate) + except Exception as e: # pylint: disable=broad-except + # obtain_cert (presumably) encountered an unanticipated problem. + logger.warning("Attempting to renew cert from %s produced an " + "unexpected error: %s. Skipping.", renewal_file, e) + logger.debug("Traceback was:\n%s", traceback.format_exc()) def revoke(config, unused_plugins): # TODO: coop with renewal config From 505e66b57c88f0bfa3ae0809a672e7f047427301 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Fri, 5 Feb 2016 18:31:41 -0500 Subject: [PATCH 084/208] Move the venv setup scripts to the tools folder. They were the last things left in the bootstrap folder, and they were lonely. --- Vagrantfile | 2 +- bootstrap/dev/README | 1 - docs/contributing.rst | 4 ++-- letsencrypt-auto-source/letsencrypt-auto | 2 +- letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh | 2 +- tests/letstest/scripts/test_apache2.sh | 2 +- tests/letstest/scripts/test_tox.sh | 2 +- {bootstrap/dev => tools}/_venv_common.sh | 0 {bootstrap/dev => tools}/venv.sh | 0 {bootstrap/dev => tools}/venv3.sh | 0 10 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 bootstrap/dev/README rename {bootstrap/dev => tools}/_venv_common.sh (100%) rename {bootstrap/dev => tools}/venv.sh (100%) rename {bootstrap/dev => tools}/venv3.sh (100%) diff --git a/Vagrantfile b/Vagrantfile index 3b9d4dc3a..678abdf72 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ VAGRANTFILE_API_VERSION = "2" $ubuntu_setup_script = <&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ $PYVER -eq 26 ] ; then diff --git a/bootstrap/dev/_venv_common.sh b/tools/_venv_common.sh similarity index 100% rename from bootstrap/dev/_venv_common.sh rename to tools/_venv_common.sh diff --git a/bootstrap/dev/venv.sh b/tools/venv.sh similarity index 100% rename from bootstrap/dev/venv.sh rename to tools/venv.sh diff --git a/bootstrap/dev/venv3.sh b/tools/venv3.sh similarity index 100% rename from bootstrap/dev/venv3.sh rename to tools/venv3.sh From 21fe41c53b6cb8847680715497fef54614076ff2 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 15:37:50 -0800 Subject: [PATCH 085/208] call .restart() on installer after renew if possible --- letsencrypt/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index abe4ccc0c..bc9777ad6 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -704,6 +704,12 @@ def obtain_cert(config, plugins, lineage=None): if config.dry_run: _report_successful_dry_run() + elif config.verb == "renew" and installer is not None: + # In case of a renewal, reload server to pick up new certificate. + # In principle we could have a configuration option to inhibit this + # from happening. + installer.restart() + print("reloaded") _suggest_donation_if_appropriate(config) From 772d424fc791937647912b4988a68810804ed4c1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 15:38:10 -0800 Subject: [PATCH 086/208] =?UTF-8?q?log=5Fdir=20=E2=86=92=20logs=5Fdir?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index bc9777ad6..d6c4ce930 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -54,7 +54,7 @@ _parser = None # file's renewalparams and actually used in the client configuration # during the renewal process. We have to record their types here because # the renewal configuration process loses this information. -STR_CONFIG_ITEMS = ["config_dir", "log_dir", "work_dir", "user_agent", +STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", "server", "account", "authenticator", "installer", "standalone_supported_challenges"] INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] From 6ef0f71e0eb07e063ffdd7ee96dc2d991c6be1a5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 15:53:54 -0800 Subject: [PATCH 087/208] -n implies -t for logging --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index a4fa409c1..6e35f1a74 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1389,7 +1389,7 @@ def setup_log_file_handler(config, logfile, fmt): def _cli_log_handler(config, level, fmt): - if config.text_mode: + if config.text_mode or config.noninteractive_mode: handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) else: From 7eb2bb4d037bd64ce4e31e3de303a968cc45db11 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 16:30:19 -0800 Subject: [PATCH 088/208] Fix renew + noninteractive --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 63c6d96f8..838da4015 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1621,7 +1621,7 @@ def setup_log_file_handler(config, logfile, fmt): def _cli_log_handler(config, level, fmt): - if config.text_mode or config.noninteractive_mode: + if config.text_mode or config.noninteractive_mode or config.verb == "renew": handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) else: From 32e0faa7b6585303d8ce48476826f75dc0d6eee1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 5 Feb 2016 16:35:55 -0800 Subject: [PATCH 089/208] Detect any setting of arguments as non-default Even if the user set the argument to the default value. This involves a hack (empty_defaults=True) where we shim all the arguments so that their default values evaluate to false. This hack may be buggy... --- letsencrypt/cli.py | 88 ++++++++++++++++++++++++------------ letsencrypt/configuration.py | 5 +- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ae09b45e0..cc3d110b3 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -727,18 +727,17 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _diff_from_default(default_conf, cli_conf, value): +def _diff_from_default(default_detector_conf, value): try: - default = default_conf.__getattr__(value) - cli = cli_conf.__getattr__(value) + if default_detector_conf.__getattr__(value): + return True + else: + return False except AttributeError: - return False - if cli != default: - return True - else: + print("Missing default", value) return False -def _restore_required_config_elements(config, renewalparams, cli_config, default_conf): +def _restore_required_config_elements(config, renewalparams, default_detector_conf): """Sets non-plugin specific values in config from renewalparams :param configuration.NamespaceConfig config: configuration for the @@ -749,7 +748,8 @@ def _restore_required_config_elements(config, renewalparams, cli_config, default """ # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: - if config_item in renewalparams and not _diff_from_default(default_conf, cli_config, config_item): + if config_item in renewalparams and not _diff_from_default(default_detector_conf, + config_item): value = renewalparams[config_item] # Unfortunately, we've lost type information from ConfigObj, # so we don't know if the original was NoneType or str! @@ -758,7 +758,8 @@ def _restore_required_config_elements(config, renewalparams, cli_config, default setattr(config.namespace, config_item, value) # int-valued items to add if they're present for config_item in INT_CONFIG_ITEMS: - if config_item in renewalparams and not _diff_from_default(default_conf, cli_config, config_item): + if config_item in renewalparams and not _diff_from_default(default_detector_conf, + config_item): try: value = int(renewalparams[config_item]) setattr(config.namespace, config_item, value) @@ -766,7 +767,7 @@ def _restore_required_config_elements(config, renewalparams, cli_config, default raise errors.Error( "Expected a numeric value for {0}".format(config_item)) -def _restore_plugin_configs(config, renewalparams): +def _restore_plugin_configs(config, renewalparams, default_detector_conf): """Sets plugin specific values in config from renewalparams :param configuration.NamespaceConfig config: configuration for the @@ -795,7 +796,8 @@ def _restore_plugin_configs(config, renewalparams): # its value to None? setattr(config.namespace, config_item, None) continue - if config_item.startswith(plugin_prefix + "_") and not _diff_from_default(default_conf, cli_config, config_item): + if config_item.startswith(plugin_prefix + "_") and not _diff_from_default( + default_detector_conf, config_item): for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: setattr(config.namespace, config_item, @@ -807,7 +809,7 @@ def _restore_plugin_configs(config, renewalparams): return True -def _reconstitute(config, full_path, cli_config, default_conf): +def _reconstitute(config, full_path, default_detector_conf): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -843,8 +845,8 @@ def _reconstitute(config, full_path, cli_config, default_conf): # Now restore specific values along with their data types, if # those elements are present. try: - _restore_required_config_elements(config, renewalparams, cli_config, default_conf) - _restore_plugin_configs(config, renewalparams) + _restore_required_config_elements(config, renewalparams, default_detector_conf) + _restore_plugin_configs(config, renewalparams, default_detector_conf) except (ValueError, errors.Error) as error: logger.warning( "An error occured while parsing %s. The error was %s. " @@ -855,7 +857,8 @@ def _reconstitute(config, full_path, cli_config, default_conf): # webroot_map is, uniquely, a dict, and the general-purpose # configuration restoring logic is not able to correctly parse it # from the serialized form. - if "webroot_map" in renewalparams and not _diff_from_default(default_conf, cli_config, "webroot_map"): + if "webroot_map" in renewalparams and not _diff_from_default(default_detector_conf, + "webroot_map"): setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) try: @@ -867,7 +870,7 @@ def _reconstitute(config, full_path, cli_config, default_conf): "invalid. Skipping.", full_path) return None - if not _diff_from_default(default_conf, cli_config, "domains"): + if not _diff_from_default(default_detector_conf, "domains"): setattr(config.namespace, "domains", domains) return renewal_candidate @@ -877,8 +880,12 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def renew(config, unused_plugins): +def renew(config, plugins): """Renew previously-obtained certificates.""" + + default_args = prepare_and_parse_args(plugins, sys.argv[1:], empty_defaults=True) + default_detector_conf = configuration.NamespaceConfig(default_args, fake=True) + if config.domains != []: raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " @@ -892,20 +899,20 @@ def renew(config, unused_plugins): "specifying a CSR file. Please try the certonly " "command instead.") renewer_config = configuration.RenewerConfiguration(config) + for renewal_file in _renewal_conf_files(renewer_config): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - default_args = prepare_and_parse_args(plugins, []) - default_conf = configuration.NamespaceConfig(default_args) lineage_config = copy.deepcopy(config) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(lineage_config, renewal_file, config, default_conf) + renewal_candidate = _reconstitute(lineage_config, renewal_file, + default_detector_conf) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -1053,7 +1060,7 @@ class HelpfulArgumentParser(object): HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + VERBS.keys() - def __init__(self, args, plugins): + def __init__(self, args, plugins, empty_defaults=False): plugin_names = [name for name, _p in plugins.iteritems()] self.help_topics = self.HELP_TOPICS + plugin_names + [None] usage, short_usage = usage_strings(plugins) @@ -1067,6 +1074,11 @@ class HelpfulArgumentParser(object): self.parser._add_config_file_help = False # pylint: disable=protected-access self.silent_parser = SilentParser(self.parser) + # This setting attempts to force all default values to None; it + # is used to detect when values have been explicitly set by the user, + # including when they are set to their normal default value + self.empty_defaults = empty_defaults + self.args = args self.determine_verb() help1 = self.prescan_for_flag("-h", self.help_topics) @@ -1100,19 +1112,19 @@ class HelpfulArgumentParser(object): parsed_args.domains.append(domain) if parsed_args.staging or parsed_args.dry_run: - if (parsed_args.server not in - (flag_default("server"), constants.STAGING_URI)): + if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): conflicts = ["--staging"] if parsed_args.staging else [] conflicts += ["--dry-run"] if parsed_args.dry_run else [] - raise errors.Error("--server value conflicts with {0}".format( - " and ".join(conflicts))) + if not self.empty_defaults: + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) parsed_args.server = constants.STAGING_URI if parsed_args.dry_run: if self.verb not in ["certonly", "renew"]: raise errors.Error("--dry-run currently only works with the " - "'certonly' or 'renew' subcommands") + "'certonly' or 'renew' subcommands (%r)" % self.verb) parsed_args.break_my_certs = parsed_args.staging = True return parsed_args @@ -1169,6 +1181,24 @@ class HelpfulArgumentParser(object): it, but can be None for `always documented'. """ + + if self.empty_defaults: + # These are config elements which cannot tolerate being set to "" + # during parsing; that's fine as long as their defaults evalute to + # boolean false. + if not any(exception in args for exception in ["--webroot-map", "-d", "-w", "-v"]): + if kwargs.get("type", None) == int: + kwargs["default"] = 0 + elif "--csr" in args: + kwargs["default"] = "" + kwargs["type"] = str + else: + kwargs["default"] = "" + #logger.info("Munging %r %r", args, "-v" in args) + else: + #logger.info("Not munging default for %r", args) + pass + if self.visible_topics[topic]: if topic in self.groups: group = self.groups[topic] @@ -1246,7 +1276,7 @@ class HelpfulArgumentParser(object): return dict([(t, t == chosen_topic) for t in self.help_topics]) -def prepare_and_parse_args(plugins, args): +def prepare_and_parse_args(plugins, args, empty_defaults=False): """Returns parsed command line arguments. :param .PluginsRegistry plugins: available plugins @@ -1256,7 +1286,7 @@ def prepare_and_parse_args(plugins, args): :rtype: argparse.Namespace """ - helpful = HelpfulArgumentParser(args, plugins) + helpful = HelpfulArgumentParser(args, plugins, empty_defaults) # --help is automatically provided by argparse helpful.add( diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 2bbf1b019..979d5e985 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -34,7 +34,7 @@ class NamespaceConfig(object): """ zope.interface.implements(interfaces.IConfig) - def __init__(self, namespace): + def __init__(self, namespace, fake=False): self.namespace = namespace self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) @@ -42,7 +42,8 @@ class NamespaceConfig(object): self.namespace.logs_dir = os.path.abspath(self.namespace.logs_dir) # Check command line parameters sanity, and error out in case of problem. - check_config_sanity(self) + if not fake: + check_config_sanity(self) def __getattr__(self, name): return getattr(self.namespace, name) From fd76b32aed364b365f851877a0071b8bda0262c3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 5 Feb 2016 16:42:41 -0800 Subject: [PATCH 090/208] Slightly better renewal debuggery --- letsencrypt/tests/cli_test.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a5757399e..b893bc85a 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -542,24 +542,24 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods mock_client = mock.MagicMock() mock_client.obtain_certificate.return_value = (mock_certr, 'chain', mock_key, 'csr') - with mock.patch('letsencrypt.cli._find_duplicative_certs') as mock_fdc: - mock_fdc.return_value = (mock_lineage, None) - with mock.patch('letsencrypt.cli._init_le_client') as mock_init: - mock_init.return_value = mock_client - get_utility_path = 'letsencrypt.cli.zope.component.getUtility' - with mock.patch(get_utility_path) as mock_get_utility: - with mock.patch('letsencrypt.cli.OpenSSL') as mock_ssl: - mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" - mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('letsencrypt.cli.crypto_util'): - if not args: - args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] - if extra_args: - args += extra_args - self._call(args) - try: + with mock.patch('letsencrypt.cli._find_duplicative_certs') as mock_fdc: + mock_fdc.return_value = (mock_lineage, None) + with mock.patch('letsencrypt.cli._init_le_client') as mock_init: + mock_init.return_value = mock_client + get_utility_path = 'letsencrypt.cli.zope.component.getUtility' + with mock.patch(get_utility_path) as mock_get_utility: + with mock.patch('letsencrypt.cli.OpenSSL') as mock_ssl: + mock_latest = mock.MagicMock() + mock_latest.get_issuer.return_value = "Fake fake" + mock_ssl.crypto.load_certificate.return_value = mock_latest + with mock.patch('letsencrypt.cli.crypto_util'): + if not args: + args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] + if extra_args: + args += extra_args + self._call(args) + if log_out: with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf: self.assertTrue(log_out in lf.read()) From 09337517d3cb24ad8606ec3bc37f289e5f7a65e7 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 16:57:41 -0800 Subject: [PATCH 091/208] Try to distinguish renew and non-renew in integration test --- tests/boulder-integration.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 8c5a93e39..294522e05 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -44,12 +44,13 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -# This won't renew (because it's not time yet) -common renew +# This won't renew (because it's not time yet) - not using common because +# common forces renewal +letsencrypt_test --authenticator standalone --installer null renew # This will renew sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" -common renew +letsencrypt_test --authenticator standalone --installer null renew ls "$root/conf/archive/le1.wtf" # dir="$root/conf/archive/le1.wtf" From 8b02f485b02e2170a140a5fc07bacf5e1916e658 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 17:13:30 -0800 Subject: [PATCH 092/208] Have a way not to force renewal in integration test --- tests/boulder-integration.sh | 16 +++++++++++----- tests/integration/_common.sh | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 294522e05..b6c76ee22 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -27,6 +27,13 @@ common() { "$@" } +common_no_force_renew() { + letsencrypt_test_no_force_renew \ + --authenticator standalone \ + --installer null \ + "$@" +} + common --domains le1.wtf --standalone-supported-challenges tls-sni-01 auth common --domains le2.wtf --standalone-supported-challenges http-01 run common -a manual -d le.wtf auth @@ -44,13 +51,12 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -# This won't renew (because it's not time yet) - not using common because -# common forces renewal -letsencrypt_test --authenticator standalone --installer null renew +# This won't renew (because it's not time yet) +letsencrypt_test_no_force_renew --authenticator standalone --installer null renew -# This will renew +# This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" -letsencrypt_test --authenticator standalone --installer null renew +letsencrypt_test_no_force_renew --authenticator standalone --installer null renew ls "$root/conf/archive/le1.wtf" # dir="$root/conf/archive/le1.wtf" diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 4572b0fb3..f133600a0 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -28,3 +28,21 @@ letsencrypt_test () { -vvvvvvv \ "$@" } + +letsencrypt_test_no_force_renew () { + letsencrypt \ + --server "${SERVER:-http://localhost:4000/directory}" \ + --no-verify-ssl \ + --tls-sni-01-port 5001 \ + --http-01-port 5002 \ + --manual-test-mode \ + $store_flags \ + --text \ + --no-redirect \ + --agree-tos \ + --register-unsafely-without-email \ + --renew-by-default \ + --debug \ + -vvvvvvv \ + "$@" +} From fd3d2fa8224b85179f7a278e78fdf1bd90b951df Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 5 Feb 2016 17:19:39 -0800 Subject: [PATCH 093/208] Make _no_force_renew not force renewal --- tests/integration/_common.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index f133600a0..9230cc682 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -41,7 +41,6 @@ letsencrypt_test_no_force_renew () { --no-redirect \ --agree-tos \ --register-unsafely-without-email \ - --renew-by-default \ --debug \ -vvvvvvv \ "$@" From 7906b31f55d91bde8f8f2a665f8491652883c5ba Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 5 Feb 2016 18:25:38 -0800 Subject: [PATCH 094/208] Cleanup and refactor a little --- letsencrypt/cli.py | 57 +++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6bec3808d..ee2001707 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -46,8 +46,7 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) -# This is global scope in order to be able to extract type information from -# it later +# Global, to save us from a lot of argument passing within the scope of this module _parser = None # These are the items which get pulled out of a renewal configuration @@ -733,17 +732,31 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _diff_from_default(default_detector_conf, value): +def _set_by_cli(variable): + """ + Return True if a particular config variable has been set by the user + (CLI or config file) including if the user explicitly set it to the + default. Returns False if the variable was assigned a default variable. + """ + if _set_by_cli.detector is None: + # Setup on first run: `detector` is a weird version of config in which + # the default value of every attribute is wrangled to be boolean-false + plugins = plugins_disco.PluginsRegistry.find_all() + default_args = prepare_and_parse_args(plugins, sys.argv[1:], empty_defaults=True) + _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) try: - if default_detector_conf.__getattr__(value): + # Is detector.variable something that isn't false? + if _set_by_cli.detector.__getattr__(variable): return True else: return False except AttributeError: - print("Missing default", value) + logger.warning("Missing default analysis for %r", variable) return False +# static housekeeping variable +_set_by_cli.detector = None -def _restore_required_config_elements(config, renewalparams, default_detector_conf): +def _restore_required_config_elements(config, renewalparams): """Sets non-plugin specific values in config from renewalparams :param configuration.NamespaceConfig config: configuration for the @@ -754,8 +767,7 @@ def _restore_required_config_elements(config, renewalparams, default_detector_co """ # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: - if config_item in renewalparams and not _diff_from_default(default_detector_conf, - config_item): + if config_item in renewalparams and not _set_by_cli(config_item): value = renewalparams[config_item] # Unfortunately, we've lost type information from ConfigObj, # so we don't know if the original was NoneType or str! @@ -764,8 +776,7 @@ def _restore_required_config_elements(config, renewalparams, default_detector_co setattr(config.namespace, config_item, value) # int-valued items to add if they're present for config_item in INT_CONFIG_ITEMS: - if config_item in renewalparams and not _diff_from_default(default_detector_conf, - config_item): + if config_item in renewalparams and not _set_by_cli(config_item): try: value = int(renewalparams[config_item]) setattr(config.namespace, config_item, value) @@ -773,7 +784,7 @@ def _restore_required_config_elements(config, renewalparams, default_detector_co raise errors.Error( "Expected a numeric value for {0}".format(config_item)) -def _restore_plugin_configs(config, renewalparams, default_detector_conf): +def _restore_plugin_configs(config, renewalparams): """Sets plugin specific values in config from renewalparams :param configuration.NamespaceConfig config: configuration for the @@ -797,8 +808,7 @@ def _restore_plugin_configs(config, renewalparams, default_detector_conf): plugin_prefixes.append(renewalparams["installer"]) for plugin_prefix in set(plugin_prefixes): for config_item, config_value in renewalparams.iteritems(): - if config_item.startswith(plugin_prefix + "_") and not _diff_from_default( - default_detector_conf, config_item): + if config_item.startswith(plugin_prefix + "_") and not _set_by_cli(config_item): # Avoid confusion when, for example, "csr = None" (avoid # trying to read the file called "None") # Should we omit the item entirely rather than setting @@ -816,7 +826,7 @@ def _restore_plugin_configs(config, renewalparams, default_detector_conf): setattr(config.namespace, config_item, str(config_value)) -def _reconstitute(config, full_path, default_detector_conf): +def _reconstitute(config, full_path): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -852,8 +862,8 @@ def _reconstitute(config, full_path, default_detector_conf): # Now restore specific values along with their data types, if # those elements are present. try: - _restore_required_config_elements(config, renewalparams, default_detector_conf) - _restore_plugin_configs(config, renewalparams, default_detector_conf) + _restore_required_config_elements(config, renewalparams) + _restore_plugin_configs(config, renewalparams) except (ValueError, errors.Error) as error: logger.warning( "An error occured while parsing %s. The error was %s. " @@ -864,8 +874,7 @@ def _reconstitute(config, full_path, default_detector_conf): # webroot_map is, uniquely, a dict, and the general-purpose # configuration restoring logic is not able to correctly parse it # from the serialized form. - if "webroot_map" in renewalparams and not _diff_from_default(default_detector_conf, - "webroot_map"): + if "webroot_map" in renewalparams and not _set_by_cli("webroot_map"): setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) try: @@ -877,7 +886,7 @@ def _reconstitute(config, full_path, default_detector_conf): "invalid. Skipping.", full_path) return None - if not _diff_from_default(default_detector_conf, "domains"): + if not _set_by_cli("domains"): setattr(config.namespace, "domains", domains) return renewal_candidate @@ -887,12 +896,9 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def renew(config, plugins): +def renew(config, unused_plugins): """Renew previously-obtained certificates.""" - default_args = prepare_and_parse_args(plugins, sys.argv[1:], empty_defaults=True) - default_detector_conf = configuration.NamespaceConfig(default_args, fake=True) - if config.domains != []: raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " @@ -916,8 +922,7 @@ def renew(config, plugins): # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(lineage_config, renewal_file, - default_detector_conf) + renewal_candidate = _reconstitute(lineage_config, renewal_file) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -1752,9 +1757,9 @@ def _handle_exception(exc_type, exc_value, trace, config): def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" sys.excepthook = functools.partial(_handle_exception, config=None) + plugins = plugins_disco.PluginsRegistry.find_all() # note: arg parser internally handles --help (and exits afterwards) - plugins = plugins_disco.PluginsRegistry.find_all() args = prepare_and_parse_args(plugins, cli_args) config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) From f675c57242be352682b49aaf740e3d7d0c5870fd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 18:48:33 -0800 Subject: [PATCH 095/208] Test no exception on empty config file --- letsencrypt/tests/cli_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a5757399e..a411b7b8c 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -602,19 +602,26 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods print "Logs:" print lf.read() - - def test_renewal_verb(self): + def test_renew_verb(self): with open(test_util.vector_path('sample-renewal.conf')) as src: # put the correct path for cert.pem, chain.pem etc in the renewal conf renewal_conf = src.read().replace("MAGICDIR", test_util.vector_path()) rd = os.path.join(self.config_dir, "renewal") - os.makedirs(rd) + if not os.path.exists(rd): + os.makedirs(rd) rc = os.path.join(rd, "sample-renewal.conf") with open(rc, "w") as dest: dest.write(renewal_conf) args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, renew=True) + def test_renew_verb_empty_config(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'empty.conf'), 'w'): + pass # leave the file empty + self.test_renew_verb() + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From ad2b6b2047abf9a154ec826e24f61a964bf01f14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 18:59:16 -0800 Subject: [PATCH 096/208] Test config file without renewal params --- letsencrypt/tests/cli_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a411b7b8c..b2db89cd4 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -622,6 +622,20 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods pass # leave the file empty self.test_renew_verb() + def test_renew_sparse_config(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: + mock_lineage = mock.MagicMock() + mock_rc.return_value = mock_lineage + mock_lineage.configuration = ["not renewalparams"] + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertFalse(mock_obtain_cert.called) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From d8c0eb6d7f68113abb13643845cdc9f938d72866 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 19:02:34 -0800 Subject: [PATCH 097/208] Test no authenticator --- letsencrypt/tests/cli_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index b2db89cd4..876e262ac 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -630,7 +630,12 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: mock_lineage = mock.MagicMock() mock_rc.return_value = mock_lineage - mock_lineage.configuration = ["not renewalparams"] + mock_lineage.configuration = ['not renewalparams'] + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertFalse(mock_obtain_cert.called) + mock_lineage.configuration = {'renewalparams': ['no auth']} with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: self._test_renewal_common(True, None, args=['renew'], renew=False) From d6e207e912f20dc9e016f030f2ac3626ec839454 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Feb 2016 19:11:36 -0800 Subject: [PATCH 098/208] Test renewal with bad int value in config --- letsencrypt/tests/cli_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 876e262ac..00588618a 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -641,6 +641,22 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args=['renew'], renew=False) self.assertFalse(mock_obtain_cert.called) + def test_renew_with_bad_int(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: + mock_lineage = mock.MagicMock() + mock_rc.return_value = mock_lineage + mock_lineage.configuration = { + 'renewalparams': {'authenticator': None, + 'rsa_key_size': 'over 9000'}} + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertFalse(mock_obtain_cert.called) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From 8d8a95800c9fe05369cfb635dd3dfd14f545b259 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Sat, 6 Feb 2016 12:14:42 -0800 Subject: [PATCH 099/208] Preliminary fix for #2386 --- letsencrypt/cli.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 838da4015..fc5a5439b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -704,12 +704,18 @@ def obtain_cert(config, plugins, lineage=None): if config.dry_run: _report_successful_dry_run() - elif config.verb == "renew" and installer is not None: - # In case of a renewal, reload server to pick up new certificate. - # In principle we could have a configuration option to inhibit this - # from happening. - installer.restart() - print("reloaded") + elif config.verb == "renew": + if installer is None: + # Tell the user that the server was not restarted. + print("new certificate deployed without restart, fullchain", + lineage.fullchain) + else: + # In case of a renewal, reload server to pick up new certificate. + # In principle we could have a configuration option to inhibit this + # from happening. + installer.restart() + print("new certificate deployed with restart of plugin", + config.installer, "fullchain is", lineage.fullchain) _suggest_donation_if_appropriate(config) From a3fd5c73a6f713d9b3fa2125717e3af780b19da0 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Sat, 6 Feb 2016 12:16:10 -0800 Subject: [PATCH 100/208] =?UTF-8?q?restart=20=E2=86=92=20reload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index fc5a5439b..7d90361ef 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -707,14 +707,14 @@ def obtain_cert(config, plugins, lineage=None): elif config.verb == "renew": if installer is None: # Tell the user that the server was not restarted. - print("new certificate deployed without restart, fullchain", + print("new certificate deployed without reload, fullchain", lineage.fullchain) else: # In case of a renewal, reload server to pick up new certificate. # In principle we could have a configuration option to inhibit this # from happening. installer.restart() - print("new certificate deployed with restart of plugin", + print("new certificate deployed with reload of plugin", config.installer, "fullchain is", lineage.fullchain) _suggest_donation_if_appropriate(config) From 46984689ae95f637951a58a03f2a5aea265c18d4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 6 Feb 2016 13:19:55 -0800 Subject: [PATCH 101/208] Attempt to get --csr and -w to play together --- letsencrypt/cli.py | 3 +-- letsencrypt/client.py | 23 ++++++++++++++++------- letsencrypt/tests/client_test.py | 23 ++++++++++++++++------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 838da4015..3b614d4b4 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -689,8 +689,7 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( - file=config.csr[0], data=config.csr[1], form="der")) + certr, chain = le_client.obtain_certificate_from_csr(_process_domain) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 57b21a55f..b4d6c5b56 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -228,21 +228,30 @@ class Client(object): authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, domain_callback): """Obtain certficiate from CSR. - :param .le_util.CSR csr: DER-encoded Certificate Signing - Request. + :param function(config, domains) domain_callback: callback for each + domain extracted from the CSR, to ensure that webroot-map and similar + housekeeping in cli.py is performed correctly :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). :rtype: tuple """ - return self._obtain_certificate( - # TODO: add CN to domains? - crypto_util.get_sans_from_csr( - csr.data, OpenSSL.crypto.FILETYPE_ASN1), csr) + + #raise TypeError("About to call %r" % le_util.CSR) + csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") + # TODO: add CN to domains? + try: + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + except: + raise TypeError("Failed %r %r %r" % (self.config.csr, csr, csr.data)) + for d in domains: + domain_callback(self.config, d) + + return self._obtain_certificate(domains, csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 2f117f80c..f051b6618 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -82,6 +82,7 @@ class ClientTest(unittest.TestCase): no_verify_ssl=False, config_dir="/etc/letsencrypt") # pylint: disable=star-args self.account = mock.MagicMock(**{"key.pem": KEY}) + self.eg_domains = ["example.com", "www.example.com"] from letsencrypt.client import Client with mock.patch("letsencrypt.client.acme_client.Client") as acme: @@ -101,8 +102,7 @@ class ClientTest(unittest.TestCase): self.acme.fetch_chain.return_value = mock.sentinel.chain def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - ["example.com", "www.example.com"]) + self.client.auth_handler.get_authorizations.assert_called_once_with(self.eg_domains) self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, CSR_SAN)), @@ -111,11 +111,20 @@ class ClientTest(unittest.TestCase): def test_obtain_certificate_from_csr(self): self._mock_obtain_certificate() - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr(le_util.CSR( - form="der", file=None, data=CSR_SAN))) - self._check_obtain_certificate() + mock_process_domain = mock.MagicMock() + test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) + with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: + mock_CSR.return_value = test_csr + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr(mock_process_domain)) + + # make sure cli processing occurred + cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) + self.assertEqual(set(cli_processed), set(self.eg_domains)) + + # and that the cert was obtained correctly + self._check_obtain_certificate() @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): From 89df062a1c6be00153807b4fb015b04e63f1d318 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 6 Feb 2016 13:38:35 -0800 Subject: [PATCH 102/208] Allow config.domains to exist in CSR mode --- letsencrypt/cli.py | 5 ----- letsencrypt/client.py | 12 ++++++++---- letsencrypt/tests/client_test.py | 3 ++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 3b614d4b4..99ee7884a 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -672,11 +672,6 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals def obtain_cert(config, plugins, lineage=None): """Implements "certonly": authenticate & obtain cert, but do not install it.""" - if config.domains and config.csr is not None: - # TODO: --csr could have a priority, when --domains is - # supplied, check if CSR matches given domains? - return "--domains and --csr are mutually exclusive" - try: # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") diff --git a/letsencrypt/client.py b/letsencrypt/client.py index b4d6c5b56..046c58cc7 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -244,13 +244,17 @@ class Client(object): #raise TypeError("About to call %r" % le_util.CSR) csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") # TODO: add CN to domains? - try: - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - except: - raise TypeError("Failed %r %r %r" % (self.config.csr, csr, csr.data)) + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) for d in domains: domain_callback(self.config, d) + csr_domains, config_domains = set(domains), set(self.config.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\ncsr:{0}\n:cli config{1}" + .format(", ".join(csr_domains), ", ".join(config_domains)) + ) + return self._obtain_certificate(domains, csr) def obtain_certificate(self, domains): diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index f051b6618..5e8fd57a7 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -115,13 +115,14 @@ class ClientTest(unittest.TestCase): test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr + self.client.config.domains=self.eg_domains self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr(mock_process_domain)) # make sure cli processing occurred cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) - self.assertEqual(set(cli_processed), set(self.eg_domains)) + self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) # and that the cert was obtained correctly self._check_obtain_certificate() From dd20788e1cc37e5a1ec80ac8de65c8b790fbe8d1 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 6 Feb 2016 13:39:32 -0800 Subject: [PATCH 103/208] lint --- letsencrypt/tests/client_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 5e8fd57a7..6a8899c3b 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -115,7 +115,7 @@ class ClientTest(unittest.TestCase): test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr - self.client.config.domains=self.eg_domains + self.client.config.domains = self.eg_domains self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), self.client.obtain_certificate_from_csr(mock_process_domain)) From 6df94bf68dff22f2dc91dac7f2d8f772de2e5793 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sat, 6 Feb 2016 13:47:52 -0800 Subject: [PATCH 104/208] Better webroot configuration error Fixes: #2377 --- letsencrypt/plugins/webroot.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index f8176417c..3f5bc6d28 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -49,8 +49,10 @@ to serve all files under specified web root ({0}).""" path_map = self.conf("map") if not path_map: - raise errors.PluginError("--{0} must be set".format( - self.option_name("path"))) + raise errors.PluginError( + "Missing parts of webroot configuration; please set either " + "--webroot-path and --domains, or --webroot-map. Run with " + " --help webroot for examples.") for name, path in path_map.items(): if not os.path.isdir(path): raise errors.PluginError(path + " does not exist or is not a directory") From 7281df234fd93c864ed854fc65400c0395292657 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 18:20:05 -0800 Subject: [PATCH 105/208] Fix lint / merge error. --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 78e908f98..2e20bb288 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -715,7 +715,7 @@ def obtain_cert(config, plugins, lineage=None): # from happening. installer.restart() print("reloaded") - _suggest_donation_if_appropriate(config) + _suggest_donation_if_appropriate(config, action) def install(config, plugins): From 2ba2dde9ed4740db196fffc095acd8928475bcf8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 18:47:50 -0800 Subject: [PATCH 106/208] Fix some broken tests --- letsencrypt/tests/cli_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a5757399e..1a86fb99b 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -228,7 +228,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["certonly", "--webroot"] ret, _, _, _ = self._call(args) - self.assertTrue("--webroot-path must be set" in ret) + self.assertTrue("please set either --webroot-path" in ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -323,9 +323,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(config.fullchain_path, os.path.abspath(fullchain)) def test_certonly_bad_args(self): - ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) - self.assertEqual(ret, '--domains and --csr are mutually exclusive') - ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') From c9df10a87e0c85a3c2cc4f8e63266afa66679351 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 19:04:43 -0800 Subject: [PATCH 107/208] Debugging / work in progress --- letsencrypt/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ee2001707..c8a458c7f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1204,9 +1204,12 @@ class HelpfulArgumentParser(object): # during parsing; that's fine as long as their defaults evalute to # boolean false. if not any(exception in args for exception in ["--webroot-map", "-d", "-w", "-v"]): - if kwargs.get("type", None) == int: + arg_type = kwargs.get("type", None) + if arg_type == int: kwargs["default"] = 0 - elif "--csr" in args: + elif arg_type == read_file or "-c" in args: + #if "-c" in args: + # raise TypeError("Skipping %r " % args) kwargs["default"] = "" kwargs["type"] = str else: From 7b0e70173126f46d5b8be57a48ec45672681db4b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 19:08:26 -0800 Subject: [PATCH 108/208] Fix error formatting --- letsencrypt/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 046c58cc7..413409ded 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -251,7 +251,7 @@ class Client(object): csr_domains, config_domains = set(domains), set(self.config.domains) if csr_domains != config_domains: raise errors.ConfigurationError( - "Inconsistent domain requests:\ncsr:{0}\n:cli config{1}" + "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" .format(", ".join(csr_domains), ", ".join(config_domains)) ) From f3655f9ab3ffde9f7a5be6580b7cb4b17beb62c5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 19:14:24 -0800 Subject: [PATCH 109/208] Throw in an extra test for good measure --- letsencrypt/tests/client_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 6a8899c3b..222e9c707 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -127,6 +127,12 @@ class ClientTest(unittest.TestCase): # and that the cert was obtained correctly self._check_obtain_certificate() + # Now provoke an inconsistent domains error... + + self.client.config.domains.append("hippopotamus.io") + self.assertRaises(errors.ConfigurationError, + self.client.obtain_certificate_from_csr, mock_process_domain) + @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): self._mock_obtain_certificate() From 317557086c19b376289b97723e52ba743dc802c2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 20:08:32 -0800 Subject: [PATCH 110/208] sys.argv[1:] does not work in tests --- letsencrypt/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c8a458c7f..32dd3ceca 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -742,7 +742,9 @@ def _set_by_cli(variable): # Setup on first run: `detector` is a weird version of config in which # the default value of every attribute is wrangled to be boolean-false plugins = plugins_disco.PluginsRegistry.find_all() - default_args = prepare_and_parse_args(plugins, sys.argv[1:], empty_defaults=True) + # reconstructed_args == sys.argv[1:], or whatever was passed to main() + reconstructed_args = _parser.args + [_parser.verb] + default_args = prepare_and_parse_args(plugins, reconstructed_args, empty_defaults=True) _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) try: # Is detector.variable something that isn't false? From d2ea078422fe25dece972415df9382311469ba20 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Sun, 7 Feb 2016 20:12:01 -0800 Subject: [PATCH 111/208] Remove some commented debugging statements --- letsencrypt/cli.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 32dd3ceca..47a8f7dc0 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1210,15 +1210,11 @@ class HelpfulArgumentParser(object): if arg_type == int: kwargs["default"] = 0 elif arg_type == read_file or "-c" in args: - #if "-c" in args: - # raise TypeError("Skipping %r " % args) kwargs["default"] = "" kwargs["type"] = str else: kwargs["default"] = "" - #logger.info("Munging %r %r", args, "-v" in args) else: - #logger.info("Not munging default for %r", args) pass if self.visible_topics[topic]: From edb07aeecb51b23ba45a6cf3f92c38a4f468667c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 10:24:09 -0800 Subject: [PATCH 112/208] Tweak changelog (but really, provoke a re-run of travis) --- CHANGES.rst | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3ed13041b..55e4bd796 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,23 +5,7 @@ Please note: the change log will only get updated after first release - for now please use the `commit log `_. +To see the changes in a given release, inspect the github milestone for the +release. For instance: -Release 0.1.0 (not released yet) --------------------------------- - -New Features: - -* ... - -Fixes: - -* ... - -Other changes: - -* ... - -Release 0.0.0 (not released yet) --------------------------------- - -Initial release. +https://github.com/letsencrypt/letsencrypt/issues?utf8=%E2%9C%93&q=milestone%3A0.3.0 From 0ba4b0c0b5dc67d07042738f6d6543f2c7ffc7c2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 11:42:55 -0800 Subject: [PATCH 113/208] Add bad domain renew test --- letsencrypt/tests/cli_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 00588618a..aebfd42a6 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -657,6 +657,22 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args=['renew'], renew=False) self.assertFalse(mock_obtain_cert.called) + def test_renew_with_bad_domain(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: + mock_lineage = mock.MagicMock() + mock_rc.return_value = mock_lineage + mock_rc.names.return_value = ['*.example.com'] + mock_lineage.configuration = { + 'renewalparams': {'authenticator': None}} + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertFalse(mock_obtain_cert.called) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From 12f1ec685054fc09ee1ae59122420ab194c985c8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 12:11:45 -0800 Subject: [PATCH 114/208] Fix test and bad domain error handling --- letsencrypt/cli.py | 6 +++--- letsencrypt/tests/cli_test.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 838da4015..c335d8d5b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -856,10 +856,10 @@ def _reconstitute(config, full_path): try: domains = [le_util.enforce_domain_sanity(x) for x in renewal_candidate.names()] - except (UnicodeError, ValueError): + except errors.ConfigurationError as error: logger.warning("Renewal configuration file %s references a cert " - "that mentions a domain name that we regarded as " - "invalid. Skipping.", full_path) + "that contains an invalid domain name. The problem " + "was: %s. Skipping.", full_path, error) return None setattr(config.namespace, "domains", domains) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index aebfd42a6..d0c2f3b91 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -650,7 +650,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods mock_lineage = mock.MagicMock() mock_rc.return_value = mock_lineage mock_lineage.configuration = { - 'renewalparams': {'authenticator': None, + 'renewalparams': {'authenticator': 'webroot', 'rsa_key_size': 'over 9000'}} with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: self._test_renewal_common(True, None, @@ -665,9 +665,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: mock_lineage = mock.MagicMock() mock_rc.return_value = mock_lineage - mock_rc.names.return_value = ['*.example.com'] mock_lineage.configuration = { - 'renewalparams': {'authenticator': None}} + 'renewalparams': {'authenticator': 'webroot'}} + mock_lineage.names.return_value = ['*.example.com'] with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: self._test_renewal_common(True, None, args=['renew'], renew=False) From c0715d168bf32b6e4b3cec2e31ce06bf63be15cf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 12:29:04 -0800 Subject: [PATCH 115/208] test _restore_plugin_configs --- letsencrypt/tests/cli_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index d0c2f3b91..2276bad97 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -673,6 +673,24 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args=['renew'], renew=False) self.assertFalse(mock_obtain_cert.called) + def test_renew_plugin_config_restoration(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: + mock_lineage = mock.MagicMock() + mock_rc.return_value = mock_lineage + mock_lineage.configuration = { + 'renewalparams': + {'authenticator': 'webroot', + 'webroot_path': 'None', + 'webroot_imaginary_flag': '42'}} + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertEqual(mock_obtain_cert.call_count, 1) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From 93ca160a1b84afed2bee67810eeb846bfe8d2af4 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 12:34:58 -0800 Subject: [PATCH 116/208] test renew with -d/--csr --- letsencrypt/tests/cli_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 2276bad97..171cc0aaa 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -691,6 +691,13 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args=['renew'], renew=False) self.assertEqual(mock_obtain_cert.call_count, 1) + def test_renew_with_bad_cli_args(self): + self.assertRaises(errors.Error, self._test_renewal_common, True, None, + args='renew -d example.com'.split(), renew=False) + self.assertRaises(errors.Error, self._test_renewal_common, True, None, + args='renew --csr {0}'.format(CSR).split(), + renew=False) + @mock.patch('letsencrypt.cli.zope.component.getUtility') @mock.patch('letsencrypt.cli._treat_as_renewal') @mock.patch('letsencrypt.cli._init_le_client') From d7be27fd847dcc6fb8e6ab829c8de9418bb45f3a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 12:38:18 -0800 Subject: [PATCH 117/208] Test renew catches all exceptions from reconstitute --- letsencrypt/tests/cli_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 171cc0aaa..a9c885f27 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -691,6 +691,19 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args=['renew'], renew=False) self.assertEqual(mock_obtain_cert.call_count, 1) + def test_renew_reconstitute_error(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + # pylint: disable=protected-access + with mock.patch('letsencrypt.cli._reconstitute') as mock_reconstitute: + mock_reconstitute.side_effect = [Exception] + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + self._test_renewal_common(True, None, + args=['renew'], renew=False) + self.assertFalse(mock_obtain_cert.called) + def test_renew_with_bad_cli_args(self): self.assertRaises(errors.Error, self._test_renewal_common, True, None, args='renew -d example.com'.split(), renew=False) From fdb9857dd8b16743286e5f28ad624a56ad242c00 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 12:54:46 -0800 Subject: [PATCH 118/208] test obtain_cert error is caught by renew --- letsencrypt/tests/cli_test.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a9c885f27..3647060d3 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -698,12 +698,27 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods f.write("My contents don't matter") # pylint: disable=protected-access with mock.patch('letsencrypt.cli._reconstitute') as mock_reconstitute: - mock_reconstitute.side_effect = [Exception] + mock_reconstitute.side_effect = Exception with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: self._test_renewal_common(True, None, args=['renew'], renew=False) self.assertFalse(mock_obtain_cert.called) + def test_renew_obtain_cert_error(self): + renewer_configs_dir = os.path.join(self.config_dir, 'renewal') + os.makedirs(renewer_configs_dir) + with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: + f.write("My contents don't matter") + with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: + mock_lineage = mock.MagicMock() + mock_rc.return_value = mock_lineage + mock_lineage.configuration = { + 'renewalparams': {'authenticator': 'webroot'}} + with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: + mock_obtain_cert.side_effect = Exception + self._test_renewal_common(True, None, + args=['renew'], renew=False) + def test_renew_with_bad_cli_args(self): self.assertRaises(errors.Error, self._test_renewal_common, True, None, args='renew -d example.com'.split(), renew=False) From 71faa50820db415186fdd0f59a08de4bef00e350 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 13:12:18 -0800 Subject: [PATCH 119/208] duplication-- --- letsencrypt/tests/cli_test.py | 99 +++++++++++++---------------------- 1 file changed, 35 insertions(+), 64 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 3647060d3..c41f45116 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -622,93 +622,64 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods pass # leave the file empty self.test_renew_verb() - def test_renew_sparse_config(self): + def _make_dummy_renewal_config(self): renewer_configs_dir = os.path.join(self.config_dir, 'renewal') os.makedirs(renewer_configs_dir) with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: f.write("My contents don't matter") + + def _test_renew_common(self, renewalparams=None, + names=None, assert_oc_called=None): + self._make_dummy_renewal_config() with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: mock_lineage = mock.MagicMock() + if renewalparams is not None: + mock_lineage.configuration = {'renewalparams': renewalparams} + if names is not None: + mock_lineage.names.return_value = names mock_rc.return_value = mock_lineage - mock_lineage.configuration = ['not renewalparams'] with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: self._test_renewal_common(True, None, args=['renew'], renew=False) - self.assertFalse(mock_obtain_cert.called) - mock_lineage.configuration = {'renewalparams': ['no auth']} - with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, - args=['renew'], renew=False) - self.assertFalse(mock_obtain_cert.called) + if assert_oc_called is not None: + if assert_oc_called: + self.assertTrue(mock_obtain_cert.called) + else: + self.assertFalse(mock_obtain_cert.called) + + def test_renew_no_renewalparams(self): + self._test_renew_common(assert_oc_called=False) + + def test_renew_no_authenticator(self): + self._test_renew_common(renewalparams={}, assert_oc_called=False) def test_renew_with_bad_int(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: - f.write("My contents don't matter") - with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: - mock_lineage = mock.MagicMock() - mock_rc.return_value = mock_lineage - mock_lineage.configuration = { - 'renewalparams': {'authenticator': 'webroot', - 'rsa_key_size': 'over 9000'}} - with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, - args=['renew'], renew=False) - self.assertFalse(mock_obtain_cert.called) + renewalparams = {'authenticator': 'webroot', + 'rsa_key_size': 'over 9000'} + self._test_renew_common(renewalparams=renewalparams, + assert_oc_called=False) def test_renew_with_bad_domain(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: - f.write("My contents don't matter") - with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: - mock_lineage = mock.MagicMock() - mock_rc.return_value = mock_lineage - mock_lineage.configuration = { - 'renewalparams': {'authenticator': 'webroot'}} - mock_lineage.names.return_value = ['*.example.com'] - with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, - args=['renew'], renew=False) - self.assertFalse(mock_obtain_cert.called) + renewalparams = {'authenticator': 'webroot'} + names = ['*.example.com'] + self._test_renew_common(renewalparams=renewalparams, + names=names, assert_oc_called=False) def test_renew_plugin_config_restoration(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: - f.write("My contents don't matter") - with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: - mock_lineage = mock.MagicMock() - mock_rc.return_value = mock_lineage - mock_lineage.configuration = { - 'renewalparams': - {'authenticator': 'webroot', - 'webroot_path': 'None', - 'webroot_imaginary_flag': '42'}} - with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, - args=['renew'], renew=False) - self.assertEqual(mock_obtain_cert.call_count, 1) + renewalparams = {'authenticator': 'webroot', + 'webroot_path': 'None', + 'webroot_imaginary_flag': '42'} + self._test_renew_common(renewalparams=renewalparams, + assert_oc_called=True) def test_renew_reconstitute_error(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: - f.write("My contents don't matter") # pylint: disable=protected-access with mock.patch('letsencrypt.cli._reconstitute') as mock_reconstitute: mock_reconstitute.side_effect = Exception - with mock.patch('letsencrypt.cli.obtain_cert') as mock_obtain_cert: - self._test_renewal_common(True, None, - args=['renew'], renew=False) - self.assertFalse(mock_obtain_cert.called) + self._test_renew_common(assert_oc_called=False) def test_renew_obtain_cert_error(self): - renewer_configs_dir = os.path.join(self.config_dir, 'renewal') - os.makedirs(renewer_configs_dir) - with open(os.path.join(renewer_configs_dir, 'test.conf'), 'w') as f: - f.write("My contents don't matter") + self._make_dummy_renewal_config() with mock.patch('letsencrypt.storage.RenewableCert') as mock_rc: mock_lineage = mock.MagicMock() mock_rc.return_value = mock_lineage From 9c7af6a93f72d73f50d87ba1715995c429d00061 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 13:22:49 -0800 Subject: [PATCH 120/208] Better reporting of renewal results --- letsencrypt/cli.py | 67 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 7d90361ef..8566765f8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -471,7 +471,7 @@ def _auth_from_domains(le_client, config, domains, lineage=None): if lineage is False: raise errors.Error("Certificate could not be obtained") - if not config.dry_run: + if not config.dry_run and not config.verb == "renew": _report_new_cert(lineage.cert, lineage.fullchain) return lineage, action @@ -681,7 +681,9 @@ def obtain_cert(config, plugins, lineage=None): # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") except errors.PluginSelectionError as e: - return e.message + logger.info( + "Could not choose appropriate plugin: %s", e) + raise # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) @@ -707,7 +709,7 @@ def obtain_cert(config, plugins, lineage=None): elif config.verb == "renew": if installer is None: # Tell the user that the server was not restarted. - print("new certificate deployed without reload, fullchain", + print("new certificate deployed without reload, fullchain is", lineage.fullchain) else: # In case of a renewal, reload server to pick up new certificate. @@ -877,6 +879,30 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) +def _renew_describe_results(renew_successes, renew_failures, parse_failures): + print() + if not renew_successes and not renew_failures: + print("No renewals were attempted.") + elif renew_successes and not renew_failures: + print("Congratulations, all renewals succeeded. The following certs " + "have been renewed:") + print("\t" + "\n\t".join(x + " (success)" for x in renew_successes)) + elif renew_failures and not renew_successes: + print("All renewal attempts failed. The following certs could not be " + "renewed:") + print("\t" + "\n\t".join(x + " (failure)" for x in renew_failures)) + elif renew_failures and renew_successes: + print("The following certs were successfully renewed:") + print("\t" + "\n\t".join(x + " (success)" for x in renew_successes)) + print("\nThe following certs could not be renewed:") + print("\t" + "\n\t".join(x + " (failure)" for x in renew_failures)) + + if parse_failures: + print("\nAdditionally, the following renewal configuration files " + "were invalid: ") + print("\t" + "\n\t".join(x + " (parsefail)" for x in parse_failures)) + + def renew(config, unused_plugins): """Renew previously-obtained certificates.""" if config.domains != []: @@ -892,6 +918,9 @@ def renew(config, unused_plugins): "specifying a CSR file. Please try the certonly " "command instead.") renewer_config = configuration.RenewerConfiguration(config) + renew_successes = [] + renew_failures = [] + parse_failures = [] for renewal_file in _renewal_conf_files(renewer_config): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object @@ -907,28 +936,42 @@ def renew(config, unused_plugins): logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) logger.debug("Traceback was:\n%s", traceback.format_exc()) + parse_failures.append(renewal_file) continue try: - if renewal_candidate is not None: + if renewal_candidate is None: + parse_failures.append(renewal_file) + else: # _reconstitute succeeded in producing a RenewableCert, so we # have something to work with from this particular config file. # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(lineage_config) - print("Trying...") - # Because obtain_cert itself indirectly decides whether to renew - # or not, we couldn't currently make a UI/logging distinction at - # this stage to indicate whether renewal was actually attempted - # (or successful). - obtain_cert(lineage_config, - plugins_disco.PluginsRegistry.find_all(), - renewal_candidate) + # Although obtain_cert itself also indirectly decides + # whether to renew or not, we need to check at this + # stage in order to avoid claiming that renewal + # succeeded when it wasn't even attempted (since + # obtain_cert wouldn't raise an error in that case). + if _should_renew(lineage_config, renewal_candidate): + err = obtain_cert(lineage_config, + plugins_disco.PluginsRegistry.find_all(), + renewal_candidate) + if err is None: + renew_successes.append(renewal_candidate.fullchain) + else: + renew_failures.append(renewal_candidate.fullchain) + else: + print("We skipped this one at the outset!") except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert from %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) logger.debug("Traceback was:\n%s", traceback.format_exc()) + renew_failures.append(renewal_candidate.fullchain) + + # Describe all the results + _renew_describe_results(renew_successes, renew_failures, parse_failures) def revoke(config, unused_plugins): # TODO: coop with renewal config From 72dfaea434723245ff58e802ebc7d07324609fee Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 13:35:54 -0800 Subject: [PATCH 121/208] Fix ACME error reporting regression --- letsencrypt/auth_handler.py | 7 ++++++- letsencrypt/tests/auth_handler_test.py | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 45c51a020..3b8c5e393 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -480,6 +480,9 @@ def is_preferred(offered_challb, satisfied, return True +_ACME_PREFIX = "urn:acme:error:" + + _ERROR_HELP_COMMON = ( "To fix these errors, please make sure that your domain name was entered " "correctly and the DNS A record(s) for that domain contain(s) the " @@ -540,11 +543,13 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ + if typ.startswith(_ACME_PREFIX): + typ = typ[len(_ACME_PREFIX):] msg = ["The following errors were reported by the server:"] for achall in failed_achalls: msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( - achall.domain, achall.error.typ, achall.error.detail)) + achall.domain, typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/tests/auth_handler_test.py b/letsencrypt/tests/auth_handler_test.py index 5b4c2bfc7..5a6199ca3 100644 --- a/letsencrypt/tests/auth_handler_test.py +++ b/letsencrypt/tests/auth_handler_test.py @@ -437,9 +437,12 @@ class ReportFailedChallsTest(unittest.TestCase): "chall": acme_util.HTTP01, "uri": "uri", "status": messages.STATUS_INVALID, - "error": messages.Error(typ="tls", detail="detail"), + "error": messages.Error(typ="urn:acme:error:tls", detail="detail"), } + # Prevent future regressions if the error type changes + self.assertTrue(kwargs["error"].description is not None) + self.http01 = achallenges.KeyAuthorizationAnnotatedChallenge( # pylint: disable=star-args challb=messages.ChallengeBody(**kwargs), From 5ef3e5399d578e8cb587392be821a6a9df74d565 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 14:06:34 -0800 Subject: [PATCH 122/208] Add webroot help to connection message --- letsencrypt/auth_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index 3b8c5e393..ffbd70ced 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -493,7 +493,9 @@ _ERROR_HELP = { "connection": _ERROR_HELP_COMMON + " Additionally, please check that your computer " "has a publicly routable IP address and that no firewalls are preventing " - "the server from communicating with the client.", + "the server from communicating with the client. If you're using the " + "webroot plugin, you should also verify that you are serving files " + "from the webroot path you provided.", "dnssec": _ERROR_HELP_COMMON + " Additionally, if you have DNSSEC enabled for " "your domain, please ensure that the signature is valid.", From 1fd3f8a8dcb9a3a8a3060bf168f7852c4cdde7cd Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 14:21:31 -0800 Subject: [PATCH 123/208] Making tests pass after CLI change --- letsencrypt/tests/cli_test.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index a5757399e..8731b8112 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -222,13 +222,16 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods if "nginx" in real_plugins: # Sending nginx a non-existent conf dir will simulate misconfiguration # (we can only do that if letsencrypt-nginx is actually present) - ret, _, _, _ = self._call(args) - self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("MisconfigurationError" in ret) + self._call(args) + # XXX: This probably now raises an exception (when nginx is + # present, but I don't know which one!) + # self.assertTrue("The nginx plugin is not working" in ret) + # self.assertTrue("MisconfigurationError" in ret) args = ["certonly", "--webroot"] - ret, _, _, _ = self._call(args) - self.assertTrue("--webroot-path must be set" in ret) + # ret, _, _, _ = self._call(args) + self.assertRaises(errors.PluginSelectionError, self._call, args) + # self.assertTrue("--webroot-path must be set" in ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -324,10 +327,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_certonly_bad_args(self): ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) - self.assertEqual(ret, '--domains and --csr are mutually exclusive') + # self.assertEqual(ret, '--domains and --csr are mutually exclusive') + # self.assertRaises(errors.Error, self._call, + # ['-d', 'foo.bar', 'certonly', '--csr', CSR]) - ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) - self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') + # ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) + self.assertRaises(errors.PluginSelectionError, self._call, + ['-a', 'bad_auth', 'certonly']) + # self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') def test_check_config_sanity_domain(self): # Punycode From c5c72de95932aeae2b6517314875570b7ab881b3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 15:09:47 -0800 Subject: [PATCH 124/208] Cleanup default detection a little, and handle an extra weird case - Noah spotted a theoretical issue with store_false args, so handle that prospectively - overall probably not really making anything neater --- letsencrypt/cli.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 47a8f7dc0..6390a3b1c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -732,11 +732,11 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _set_by_cli(variable): +def _set_by_cli(var): """ Return True if a particular config variable has been set by the user (CLI or config file) including if the user explicitly set it to the - default. Returns False if the variable was assigned a default variable. + default. Returns False if the variable was assigned a default value. """ if _set_by_cli.detector is None: # Setup on first run: `detector` is a weird version of config in which @@ -747,15 +747,20 @@ def _set_by_cli(variable): default_args = prepare_and_parse_args(plugins, reconstructed_args, empty_defaults=True) _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) try: - # Is detector.variable something that isn't false? - if _set_by_cli.detector.__getattr__(variable): + # Is detector.var something that isn't false? + change_detected = _set_by_cli.detector.__getattr__(var) + if change_detected: + return True + # Special case: like --no-redirect vars that get set True -> False are + # will be None here + elif var in _set_by_cli.detector.store_false_vars and change_detected == False: return True else: return False except AttributeError: - logger.warning("Missing default analysis for %r", variable) + logger.warning("Missing default analysis for %r", var) return False -# static housekeeping variable +# static housekeeping var _set_by_cli.detector = None def _restore_required_config_elements(config, renewalparams): @@ -1097,6 +1102,8 @@ class HelpfulArgumentParser(object): # is used to detect when values have been explicitly set by the user, # including when they are set to their normal default value self.empty_defaults = empty_defaults + if empty_defaults: + self.store_false_vars = {} # vars that use "store_false" self.args = args self.determine_verb() @@ -1109,7 +1116,7 @@ class HelpfulArgumentParser(object): print(usage) sys.exit(0) self.visible_topics = self.determine_help_topics(self.help_arg) - self.groups = {} # elements are added by .add_group() + self.groups = {} # elements are added by .add_group() def parse_args(self): """Parses command line arguments and returns the result. @@ -1205,17 +1212,25 @@ class HelpfulArgumentParser(object): # These are config elements which cannot tolerate being set to "" # during parsing; that's fine as long as their defaults evalute to # boolean false. - if not any(exception in args for exception in ["--webroot-map", "-d", "-w", "-v"]): + + # argument either doesn't have a default, or the default doesn't + # isn't Pythonically false + if kwargs.get("default", True): arg_type = kwargs.get("type", None) - if arg_type == int: + if arg_type == int or kwargs.get("action", "") == "count": kwargs["default"] = 0 elif arg_type == read_file or "-c" in args: kwargs["default"] = "" kwargs["type"] = str else: kwargs["default"] = "" - else: - pass + # This doesn't matter at present (none of the store_false args + # are renewal-relevant), but implement it for future sanity: + # detect the setting of args whose presence causes True -> False + elif kwargs.get("action", "") == "store_false": + kwargs["default"] = None + for var in args: + self.store_false_vars[var] = True if self.visible_topics[topic]: if topic in self.groups: From de43a4b0520c61b2c8cf167a91e91da846517590 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 15:20:04 -0800 Subject: [PATCH 125/208] Split default detection into its own method --- letsencrypt/cli.py | 80 ++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6390a3b1c..9e9b8fa52 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -744,16 +744,16 @@ def _set_by_cli(var): plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = _parser.args + [_parser.verb] - default_args = prepare_and_parse_args(plugins, reconstructed_args, empty_defaults=True) + default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) try: # Is detector.var something that isn't false? change_detected = _set_by_cli.detector.__getattr__(var) if change_detected: return True - # Special case: like --no-redirect vars that get set True -> False are - # will be None here - elif var in _set_by_cli.detector.store_false_vars and change_detected == False: + # Special case: vars like --no-redirect that get set True -> False + # default to None; False means they were set + elif var in _set_by_cli.detector.store_false_vars and change_detected is not None: return True else: return False @@ -1084,7 +1084,7 @@ class HelpfulArgumentParser(object): HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + VERBS.keys() - def __init__(self, args, plugins, empty_defaults=False): + def __init__(self, args, plugins, detect_defaults=False): plugin_names = [name for name, _p in plugins.iteritems()] self.help_topics = self.HELP_TOPICS + plugin_names + [None] usage, short_usage = usage_strings(plugins) @@ -1101,8 +1101,8 @@ class HelpfulArgumentParser(object): # This setting attempts to force all default values to None; it # is used to detect when values have been explicitly set by the user, # including when they are set to their normal default value - self.empty_defaults = empty_defaults - if empty_defaults: + self.detect_defaults = detect_defaults + if detect_defaults: self.store_false_vars = {} # vars that use "store_false" self.args = args @@ -1141,7 +1141,7 @@ class HelpfulArgumentParser(object): if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): conflicts = ["--staging"] if parsed_args.staging else [] conflicts += ["--dry-run"] if parsed_args.dry_run else [] - if not self.empty_defaults: + if not self.detect_defaults: raise errors.Error("--server value conflicts with {0}".format( " and ".join(conflicts))) @@ -1203,34 +1203,15 @@ class HelpfulArgumentParser(object): def add(self, topic, *args, **kwargs): """Add a new command line argument. - @topic is required, to indicate which part of the help will document - it, but can be None for `always documented'. + :param str: help topic this should be listed under, can be None for + "always documented" + :param list *args: the names of this argument flag + :param dict **kwargs: various argparse settings for this argument """ - if self.empty_defaults: - # These are config elements which cannot tolerate being set to "" - # during parsing; that's fine as long as their defaults evalute to - # boolean false. - - # argument either doesn't have a default, or the default doesn't - # isn't Pythonically false - if kwargs.get("default", True): - arg_type = kwargs.get("type", None) - if arg_type == int or kwargs.get("action", "") == "count": - kwargs["default"] = 0 - elif arg_type == read_file or "-c" in args: - kwargs["default"] = "" - kwargs["type"] = str - else: - kwargs["default"] = "" - # This doesn't matter at present (none of the store_false args - # are renewal-relevant), but implement it for future sanity: - # detect the setting of args whose presence causes True -> False - elif kwargs.get("action", "") == "store_false": - kwargs["default"] = None - for var in args: - self.store_false_vars[var] = True + if self.detect_defaults: + self.modify_arg_for_default_detection(self, *args, **kwargs) if self.visible_topics[topic]: if topic in self.groups: @@ -1242,6 +1223,35 @@ class HelpfulArgumentParser(object): kwargs["help"] = argparse.SUPPRESS self.parser.add_argument(*args, **kwargs) + + def modify_arg_for_default_detection(self, *args, **kwargs): + """ + Adding an arg, but ensure that it has a default that evaluates to false, + so that _set_by_cli can tell if it was set. Only called if detect_defaults==True. + + :param list *args: the names of this argument flag + :param dict **kwargs: various argparse settings for this argument + """ + # argument either doesn't have a default, or the default doesn't + # isn't Pythonically false + if kwargs.get("default", True): + arg_type = kwargs.get("type", None) + if arg_type == int or kwargs.get("action", "") == "count": + kwargs["default"] = 0 + elif arg_type == read_file or "-c" in args: + kwargs["default"] = "" + kwargs["type"] = str + else: + kwargs["default"] = "" + # This doesn't matter at present (none of the store_false args + # are renewal-relevant), but implement it for future sanity: + # detect the setting of args whose presence causes True -> False + elif kwargs.get("action", "") == "store_false": + kwargs["default"] = None + for var in args: + self.store_false_vars[var] = True + + def add_deprecated_argument(self, argument_name, num_args): """Adds a deprecated argument with the name argument_name. @@ -1309,7 +1319,7 @@ class HelpfulArgumentParser(object): return dict([(t, t == chosen_topic) for t in self.help_topics]) -def prepare_and_parse_args(plugins, args, empty_defaults=False): +def prepare_and_parse_args(plugins, args, detect_defaults=False): """Returns parsed command line arguments. :param .PluginsRegistry plugins: available plugins @@ -1319,7 +1329,7 @@ def prepare_and_parse_args(plugins, args, empty_defaults=False): :rtype: argparse.Namespace """ - helpful = HelpfulArgumentParser(args, plugins, empty_defaults) + helpful = HelpfulArgumentParser(args, plugins, detect_defaults) # --help is automatically provided by argparse helpful.add( From d8ea828de6b84cf3393037d6ea07ede2a01abc53 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 16:11:20 -0800 Subject: [PATCH 126/208] Fix lint complaints about cli.py --- letsencrypt/cli.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 8566765f8..60b5fdbbc 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -338,6 +338,7 @@ def _handle_identical_cert_request(config, cert): else: assert False, "This is impossible" + def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested @@ -488,7 +489,7 @@ def _avoid_invalidating_lineage(config, lineage, original_server): open(lineage.cert).read()) # all our test certs are from happy hacker fake CA, though maybe one day # we should test more methodically - now_valid = not "fake" in repr(latest_cert.get_issuer()).lower() + now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if _is_staging(config.server): if not _is_staging(original_server) or now_valid: @@ -547,6 +548,7 @@ def set_configurator(previously, now): raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) return now + def cli_plugin_requests(config): """ Figure out which plugins the user requested with CLI and config options @@ -575,6 +577,7 @@ def cli_plugin_requests(config): noninstaller_plugins = ["webroot", "manual", "standalone"] + def choose_configurator_plugins(config, plugins, verb): """ Figure out which configurator we're going to use, modifies @@ -597,7 +600,7 @@ def choose_configurator_plugins(config, plugins, verb): '{1} {2} certonly --{0}{1}{1}' '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins' '{1} and "--help plugins" for more information.)'.format( - req_auth, os.linesep, cli_command)) + req_auth, os.linesep, cli_command)) raise errors.MissingCommandlineFlag(msg) else: @@ -769,6 +772,7 @@ def _restore_required_config_elements(config, renewalparams): raise errors.Error( "Expected a numeric value for {0}".format(config_item)) + def _restore_plugin_configs(config, renewalparams): """Sets plugin specific values in config from renewalparams @@ -801,7 +805,7 @@ def _restore_plugin_configs(config, renewalparams): if config_value == "None": setattr(config.namespace, config_item, None) continue - for action in _parser.parser._actions: # pylint: disable=protected-access + for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: setattr(config.namespace, config_item, action.type(config_value)) @@ -931,7 +935,7 @@ def renew(config, unused_plugins): # elements from within the renewal configuration file). try: renewal_candidate = _reconstitute(lineage_config, renewal_file) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) @@ -963,7 +967,7 @@ def renew(config, unused_plugins): renew_failures.append(renewal_candidate.fullchain) else: print("We skipped this one at the outset!") - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert from %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) @@ -1447,7 +1451,7 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) - global _parser # pylint: disable=global-statement + global _parser # pylint: disable=global-statement _parser = helpful return helpful.parse_args() @@ -1584,14 +1588,17 @@ def _plugins_parsing(helpful, plugins): "www.example.com -w /var/www/thing -d thing.net -d m.thing.net`") # --webroot-map still has some awkward properties, so it is undocumented helpful.add("webroot", "--webroot-map", default={}, action=WebrootMapProcessor, - help="JSON dictionary mapping domains to webroot paths; this implies -d " - "for each entry. You may need to escape this from your shell. " - """Eg: --webroot-map '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ - "This option is merged with, but takes precedence over, -w / -d entries." - " At present, if you put webroot-map in a config file, it needs to be " - ' on a single line, like: webroot-map = {"example.com":"/var/www"}.') + help="JSON dictionary mapping domains to webroot paths; this " + "implies -d for each entry. You may need to escape this " + "from your shell. " + """E.g.: --webroot-map '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ + "This option is merged with, but takes precedence over, " + "-w / -d entries. At present, if you put webroot-map in " + "a config file, it needs to be on a single line, like: " + 'webroot-map = {"example.com":"/var/www"}.') -class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring + +class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring def __init__(self, *args, **kwargs): self.domain_before_webroot = False argparse.Action.__init__(self, *args, **kwargs) @@ -1640,14 +1647,14 @@ def _process_domain(args_or_config, domain_arg, webroot_path=None): args_or_config.webroot_map.setdefault(domain, webroot_path[-1]) -class WebrootMapProcessor(argparse.Action): # pylint: disable=missing-docstring +class WebrootMapProcessor(argparse.Action): # pylint: disable=missing-docstring def __call__(self, parser, args, webroot_map_arg, option_string=None): webroot_map = json.loads(webroot_map_arg) for domains, webroot_path in webroot_map.iteritems(): _process_domain(args, domains, [webroot_path]) -class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring +class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring def __call__(self, parser, args, domain_arg, option_string=None): """Just wrap _process_domain in argparseese.""" _process_domain(args, domain_arg) @@ -1738,8 +1745,8 @@ def _handle_exception(exc_type, exc_value, trace, config): # acme.messages.Error: urn:acme:error:malformed :: The request message was # malformed :: Error creating new registration :: Validation of contact # mailto:none@longrandomstring.biz failed: Server failure at resolver - if ("urn:acme" in err and ":: " in err - and config.verbose_count <= flag_default("verbose_count")): + if (("urn:acme" in err and ":: " in err and + config.verbose_count <= flag_default("verbose_count"))): # prune ACME error code, we have a human description _code, _sep, err = err.partition(":: ") msg = "An unexpected error occurred:\n" + err + "Please see the " From b61bfdc468d5cc31583241b8aa72bf9ee9d83c9b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 16:18:30 -0800 Subject: [PATCH 127/208] Don't set default="" for store_false args... --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 9e9b8fa52..43a6ed1ab 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1246,7 +1246,7 @@ class HelpfulArgumentParser(object): # This doesn't matter at present (none of the store_false args # are renewal-relevant), but implement it for future sanity: # detect the setting of args whose presence causes True -> False - elif kwargs.get("action", "") == "store_false": + if kwargs.get("action", "") == "store_false": kwargs["default"] = None for var in args: self.store_false_vars[var] = True From 4d7ad032ee6d4cd3224d5195c0f7c7bbccbe7618 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 16:34:46 -0800 Subject: [PATCH 128/208] Mention skipped lineages too --- letsencrypt/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index bae124d81..886616dbf 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -719,8 +719,8 @@ def obtain_cert(config, plugins, lineage=None): # In principle we could have a configuration option to inhibit this # from happening. installer.restart() - print("new certificate deployed with reload of plugin", - config.installer, "fullchain is", lineage.fullchain) + print("new certificate deployed with reload of", + config.installer, "server; fullchain is", lineage.fullchain) _suggest_donation_if_appropriate(config) @@ -883,8 +883,12 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def _renew_describe_results(renew_successes, renew_failures, parse_failures): +def _renew_describe_results(renew_successes, renew_failures, renew_skipped, + parse_failures): print() + if renew_skipped: + print("The following certs are not due for renewal yet:") + print("\t" + "\n\t".join(x + " (skipped)" for x in renew_skipped)) if not renew_successes and not renew_failures: print("No renewals were attempted.") elif renew_successes and not renew_failures: @@ -924,11 +928,10 @@ def renew(config, unused_plugins): renewer_config = configuration.RenewerConfiguration(config) renew_successes = [] renew_failures = [] + renew_skipped = [] parse_failures = [] for renewal_file in _renewal_conf_files(renewer_config): print("Processing " + renewal_file) - # XXX: does this succeed in making a fully independent config object - # each time? lineage_config = copy.deepcopy(config) # Note that this modifies config (to add back the configuration @@ -966,7 +969,7 @@ def renew(config, unused_plugins): else: renew_failures.append(renewal_candidate.fullchain) else: - print("We skipped this one at the outset!") + renew_skipped.append(renewal_candidate.fullchain) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert from %s produced an " @@ -975,7 +978,8 @@ def renew(config, unused_plugins): renew_failures.append(renewal_candidate.fullchain) # Describe all the results - _renew_describe_results(renew_successes, renew_failures, parse_failures) + _renew_describe_results(renew_successes, renew_failures, renew_skipped, + parse_failures) def revoke(config, unused_plugins): # TODO: coop with renewal config From ad4b8ec147e6560fd3368fbf2274ca6f364b6778 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 16:41:42 -0800 Subject: [PATCH 129/208] lambda to simplify printing lists of success/failure --- letsencrypt/cli.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 886616dbf..27a935ca8 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -885,30 +885,31 @@ def _renewal_conf_files(config): def _renew_describe_results(renew_successes, renew_failures, renew_skipped, parse_failures): + status = lambda x, msg: " " + "\n ".join(i + " (" + msg +")" for i in x) print() if renew_skipped: print("The following certs are not due for renewal yet:") - print("\t" + "\n\t".join(x + " (skipped)" for x in renew_skipped)) + print(status(renew_skipped, "skipped")) if not renew_successes and not renew_failures: print("No renewals were attempted.") elif renew_successes and not renew_failures: print("Congratulations, all renewals succeeded. The following certs " "have been renewed:") - print("\t" + "\n\t".join(x + " (success)" for x in renew_successes)) + print(status(renew_successes, "success")) elif renew_failures and not renew_successes: print("All renewal attempts failed. The following certs could not be " "renewed:") - print("\t" + "\n\t".join(x + " (failure)" for x in renew_failures)) + print(status(renew_failures, "failure")) elif renew_failures and renew_successes: print("The following certs were successfully renewed:") - print("\t" + "\n\t".join(x + " (success)" for x in renew_successes)) + print(status(renew_successes, "success")) print("\nThe following certs could not be renewed:") - print("\t" + "\n\t".join(x + " (failure)" for x in renew_failures)) + print(status(renew_failures, "failure")) if parse_failures: print("\nAdditionally, the following renewal configuration files " "were invalid: ") - print("\t" + "\n\t".join(x + " (parsefail)" for x in parse_failures)) + print(status(parse_failures, "parsefail")) def renew(config, unused_plugins): From 0946ea8e046133abcc602bb35f818b76636054f7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 16:42:13 -0800 Subject: [PATCH 130/208] Bleh. --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 43a6ed1ab..0b5c2bdfd 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -753,7 +753,7 @@ def _set_by_cli(var): return True # Special case: vars like --no-redirect that get set True -> False # default to None; False means they were set - elif var in _set_by_cli.detector.store_false_vars and change_detected is not None: + elif var in _set_by_cli.detector.namespace.store_false_vars and change_detected is not None: return True else: return False From 994c96180a0984133834aa083666332e0e236011 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 16:48:21 -0800 Subject: [PATCH 131/208] Don't catch the wrong exception by accident --- letsencrypt/cli.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0b5c2bdfd..12aa69b46 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -746,20 +746,22 @@ def _set_by_cli(var): reconstructed_args = _parser.args + [_parser.verb] default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) + try: # Is detector.var something that isn't false? change_detected = _set_by_cli.detector.__getattr__(var) - if change_detected: - return True - # Special case: vars like --no-redirect that get set True -> False - # default to None; False means they were set - elif var in _set_by_cli.detector.namespace.store_false_vars and change_detected is not None: - return True - else: - return False except AttributeError: logger.warning("Missing default analysis for %r", var) return False + + if change_detected: + return True + # Special case: vars like --no-redirect that get set True -> False + # default to None; False means they were set + elif var in _set_by_cli.detector.namespace.store_false_vars and change_detected is not None: + return True + else: + return False # static housekeeping var _set_by_cli.detector = None From bcf38476faa9bf3c35fa24df81d2f9f732a2ee79 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 16:59:11 -0800 Subject: [PATCH 132/208] Fix bugs, clean up plumbing. --- letsencrypt/cli.py | 14 +++++++++----- letsencrypt/configuration.py | 5 ++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 12aa69b46..79a27a200 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -744,8 +744,8 @@ def _set_by_cli(var): plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = _parser.args + [_parser.verb] - default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) - _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) + _set_by_cli.detector = prepare_and_parse_args(plugins, reconstructed_args, + detect_defaults=True) try: # Is detector.var something that isn't false? @@ -758,7 +758,7 @@ def _set_by_cli(var): return True # Special case: vars like --no-redirect that get set True -> False # default to None; False means they were set - elif var in _set_by_cli.detector.namespace.store_false_vars and change_detected is not None: + elif var in _set_by_cli.detector.store_false_vars and change_detected is not None: return True else: return False @@ -1154,6 +1154,9 @@ class HelpfulArgumentParser(object): raise errors.Error("--dry-run currently only works with the " "'certonly' or 'renew' subcommands (%r)" % self.verb) parsed_args.break_my_certs = parsed_args.staging = True + + if self.detect_defaults: # plumbing + parsed_args.store_false_vars = self.store_false_vars return parsed_args @@ -1476,8 +1479,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) - global _parser # pylint: disable=global-statement - _parser = helpful + if not detect_defaults: + global _parser # pylint: disable=global-statement + _parser = helpful return helpful.parse_args() diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 979d5e985..2bbf1b019 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -34,7 +34,7 @@ class NamespaceConfig(object): """ zope.interface.implements(interfaces.IConfig) - def __init__(self, namespace, fake=False): + def __init__(self, namespace): self.namespace = namespace self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) @@ -42,8 +42,7 @@ class NamespaceConfig(object): self.namespace.logs_dir = os.path.abspath(self.namespace.logs_dir) # Check command line parameters sanity, and error out in case of problem. - if not fake: - check_config_sanity(self) + check_config_sanity(self) def __getattr__(self, name): return getattr(self.namespace, name) From 0af5b4b0a9f64f1a8190e4a82046a6ff990bee11 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 17:07:39 -0800 Subject: [PATCH 133/208] arghlint --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 9c9bf912a..35211c0bd 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1154,7 +1154,7 @@ class HelpfulArgumentParser(object): raise errors.Error("--dry-run currently only works with the " "'certonly' or 'renew' subcommands (%r)" % self.verb) parsed_args.break_my_certs = parsed_args.staging = True - + if self.detect_defaults: # plumbing parsed_args.store_false_vars = self.store_false_vars From de455ac6e06244d747da0c8b219390377a5eae57 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 17:23:06 -0800 Subject: [PATCH 134/208] Don't check _should_renew twice --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 63c672227..19358a8bd 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -440,7 +440,7 @@ def _auth_from_domains(le_client, config, domains, lineage=None): else: # Renewal, where we already know the specific lineage we're # interested in - action = "renew" if _should_renew(config, lineage) else "reinstall" + action = "renew" if action == "reinstall": # The lineage already exists; allow the caller to try installing From d65a3c65c20ca7c12c4f85802a8bc84a14b95611 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 17:25:47 -0800 Subject: [PATCH 135/208] Revert "Allow webroot-map and --csr to exist together." --- letsencrypt/cli.py | 8 +++++++- letsencrypt/client.py | 27 +++++++-------------------- letsencrypt/plugins/webroot.py | 6 ++---- letsencrypt/tests/cli_test.py | 5 ++++- letsencrypt/tests/client_test.py | 24 +++++++----------------- 5 files changed, 27 insertions(+), 43 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index de1321ac9..c335d8d5b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -672,6 +672,11 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals def obtain_cert(config, plugins, lineage=None): """Implements "certonly": authenticate & obtain cert, but do not install it.""" + if config.domains and config.csr is not None: + # TODO: --csr could have a priority, when --domains is + # supplied, check if CSR matches given domains? + return "--domains and --csr are mutually exclusive" + try: # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") @@ -684,7 +689,8 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(_process_domain) + certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( + file=config.csr[0], data=config.csr[1], form="der")) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 413409ded..57b21a55f 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -228,34 +228,21 @@ class Client(object): authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, domain_callback): + def obtain_certificate_from_csr(self, csr): """Obtain certficiate from CSR. - :param function(config, domains) domain_callback: callback for each - domain extracted from the CSR, to ensure that webroot-map and similar - housekeeping in cli.py is performed correctly + :param .le_util.CSR csr: DER-encoded Certificate Signing + Request. :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). :rtype: tuple """ - - #raise TypeError("About to call %r" % le_util.CSR) - csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") - # TODO: add CN to domains? - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - for d in domains: - domain_callback(self.config, d) - - csr_domains, config_domains = set(domains), set(self.config.domains) - if csr_domains != config_domains: - raise errors.ConfigurationError( - "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" - .format(", ".join(csr_domains), ", ".join(config_domains)) - ) - - return self._obtain_certificate(domains, csr) + return self._obtain_certificate( + # TODO: add CN to domains? + crypto_util.get_sans_from_csr( + csr.data, OpenSSL.crypto.FILETYPE_ASN1), csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index 3f5bc6d28..f8176417c 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -49,10 +49,8 @@ to serve all files under specified web root ({0}).""" path_map = self.conf("map") if not path_map: - raise errors.PluginError( - "Missing parts of webroot configuration; please set either " - "--webroot-path and --domains, or --webroot-map. Run with " - " --help webroot for examples.") + raise errors.PluginError("--{0} must be set".format( + self.option_name("path"))) for name, path in path_map.items(): if not os.path.isdir(path): raise errors.PluginError(path + " does not exist or is not a directory") diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index 2d36a9d21..c41f45116 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -228,7 +228,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["certonly", "--webroot"] ret, _, _, _ = self._call(args) - self.assertTrue("please set either --webroot-path" in ret) + self.assertTrue("--webroot-path must be set" in ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -323,6 +323,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(config.fullchain_path, os.path.abspath(fullchain)) def test_certonly_bad_args(self): + ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) + self.assertEqual(ret, '--domains and --csr are mutually exclusive') + ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 6a8899c3b..2f117f80c 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -82,7 +82,6 @@ class ClientTest(unittest.TestCase): no_verify_ssl=False, config_dir="/etc/letsencrypt") # pylint: disable=star-args self.account = mock.MagicMock(**{"key.pem": KEY}) - self.eg_domains = ["example.com", "www.example.com"] from letsencrypt.client import Client with mock.patch("letsencrypt.client.acme_client.Client") as acme: @@ -102,7 +101,8 @@ class ClientTest(unittest.TestCase): self.acme.fetch_chain.return_value = mock.sentinel.chain def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with(self.eg_domains) + self.client.auth_handler.get_authorizations.assert_called_once_with( + ["example.com", "www.example.com"]) self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, CSR_SAN)), @@ -111,21 +111,11 @@ class ClientTest(unittest.TestCase): def test_obtain_certificate_from_csr(self): self._mock_obtain_certificate() - mock_process_domain = mock.MagicMock() - test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) - with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: - mock_CSR.return_value = test_csr - self.client.config.domains = self.eg_domains - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr(mock_process_domain)) - - # make sure cli processing occurred - cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) - self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) - - # and that the cert was obtained correctly - self._check_obtain_certificate() + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr(le_util.CSR( + form="der", file=None, data=CSR_SAN))) + self._check_obtain_certificate() @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): From 374e4ebb4dc11941f6f271f30872df6bb1bb658e Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 17:48:12 -0800 Subject: [PATCH 136/208] Trying to satisfy pylint --- letsencrypt/cli.py | 4 ++-- letsencrypt/tests/cli_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 19358a8bd..7dd0a7771 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1589,8 +1589,8 @@ def _plugins_parsing(helpful, plugins): helpful.add("webroot", "--webroot-map", default={}, action=WebrootMapProcessor, help="JSON dictionary mapping domains to webroot paths; this " "implies -d for each entry. You may need to escape this " - "from your shell. " - """E.g.: --webroot-map '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ + "from your shell. E.g.: --webroot-map " + """'{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ "This option is merged with, but takes precedence over, " "-w / -d entries. At present, if you put webroot-map in " "a config file, it needs to be on a single line, like: " diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index c263fd8ec..76b7676c3 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -326,7 +326,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(config.fullchain_path, os.path.abspath(fullchain)) def test_certonly_bad_args(self): - ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) + _, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) # self.assertEqual(ret, '--domains and --csr are mutually exclusive') # self.assertRaises(errors.Error, self._call, # ['-d', 'foo.bar', 'certonly', '--csr', CSR]) From 24a3b66b1ca7eff88fdf6ee40514898793eaab6a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 17:52:12 -0800 Subject: [PATCH 137/208] Use server_close() in standalone --- letsencrypt/plugins/standalone.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt/plugins/standalone.py b/letsencrypt/plugins/standalone.py index cde7041d8..71a17a28e 100644 --- a/letsencrypt/plugins/standalone.py +++ b/letsencrypt/plugins/standalone.py @@ -90,6 +90,9 @@ class ServerManager(object): logger.debug("Stopping server at %s:%d...", *instance.server.socket.getsockname()[:2]) instance.server.shutdown() + # Not calling server_close causes problems when renewing multiple + # certs with `letsencrypt renew` using TLSSNI01 and PyOpenSSL 0.13 + instance.server.server_close() instance.thread.join() del self._instances[port] From bb2f054f1b9529c77f8f8d536dacd8a508667ec2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 17:54:02 -0800 Subject: [PATCH 138/208] Take boulder-integration.sh from #2398 --- tests/boulder-integration.sh | 39 ++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index b6c76ee22..8b6dc5f1b 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -51,14 +51,45 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" +echo round 1 + +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/le.wtf/"* | wc -l` + if [ "$CERTCOUNT" -ne "$1" ] ; then + echo Wrong cert count, not "$1" `ls "${root}/conf/archive/le.wtf/"*` + exit 1 + fi +} + +CheckCertCount 4 # This won't renew (because it's not time yet) -letsencrypt_test_no_force_renew --authenticator standalone --installer null renew +letsencrypt_test_no_force_renew --authenticator standalone --installer null renew -tvv +CheckCertCount 4 + +echo round 2 # This will renew because the expiry is less than 10 years from now -sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" -letsencrypt_test_no_force_renew --authenticator standalone --installer null renew +sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" +letsencrypt_test_no_force_renew --authenticator standalone --installer null renew # --renew-by-default +CheckCertCount 8 + +echo round 3 + +# Check Param setting in renewal... +letsencrypt_test_no_force_renew --authenticator standalone --installer null renew --renew-by-default +CheckCertCount 12 +echo round 4 + +# The 4096 bit setting should persist to the first renewal, but be overriden in the second +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +#if ! [ "$size3" -lt "$size2" ] ; then +# echo "key size failure:" +# ls -l ${root}/conf/archive/le.wtf/ +# exit 1 +#fi + -ls "$root/conf/archive/le1.wtf" # dir="$root/conf/archive/le1.wtf" # for x in cert chain fullchain privkey; # do From 38a6d442796c6a4973365f67a96affeb11b612df Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 17:54:39 -0800 Subject: [PATCH 139/208] Remove round echos --- tests/boulder-integration.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 8b6dc5f1b..53e9b3f15 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -51,8 +51,6 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -echo round 1 - CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/le.wtf/"* | wc -l` if [ "$CERTCOUNT" -ne "$1" ] ; then @@ -66,19 +64,14 @@ CheckCertCount 4 letsencrypt_test_no_force_renew --authenticator standalone --installer null renew -tvv CheckCertCount 4 -echo round 2 - # This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" letsencrypt_test_no_force_renew --authenticator standalone --installer null renew # --renew-by-default CheckCertCount 8 -echo round 3 - # Check Param setting in renewal... letsencrypt_test_no_force_renew --authenticator standalone --installer null renew --renew-by-default CheckCertCount 12 -echo round 4 # The 4096 bit setting should persist to the first renewal, but be overriden in the second size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` From 8eb889d94251d36189ea7f5f23e7e26f91fdd901 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 17:55:28 -0800 Subject: [PATCH 140/208] Make CheckCertCount check cert counts --- tests/boulder-integration.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 53e9b3f15..dd6c1835e 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -52,26 +52,26 @@ common --domains le3.wtf install \ --key-path "${root}/csr/key.pem" CheckCertCount() { - CERTCOUNT=`ls "${root}/conf/archive/le.wtf/"* | wc -l` + CERTCOUNT=`ls "${root}/conf/archive/le.wtf/cert*" | wc -l` if [ "$CERTCOUNT" -ne "$1" ] ; then echo Wrong cert count, not "$1" `ls "${root}/conf/archive/le.wtf/"*` exit 1 fi } -CheckCertCount 4 +CheckCertCount 1 # This won't renew (because it's not time yet) letsencrypt_test_no_force_renew --authenticator standalone --installer null renew -tvv -CheckCertCount 4 +CheckCertCount 1 # This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" letsencrypt_test_no_force_renew --authenticator standalone --installer null renew # --renew-by-default -CheckCertCount 8 +CheckCertCount 2 # Check Param setting in renewal... letsencrypt_test_no_force_renew --authenticator standalone --installer null renew --renew-by-default -CheckCertCount 12 +CheckCertCount 3 # The 4096 bit setting should persist to the first renewal, but be overriden in the second size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` From 77616a975bbea2b9166efe578314ca9ec5d773f2 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 17:59:30 -0800 Subject: [PATCH 141/208] Allow non-interactive with test-mode --- letsencrypt/plugins/manual.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/plugins/manual.py b/letsencrypt/plugins/manual.py index 54244db2a..0e516b5b0 100644 --- a/letsencrypt/plugins/manual.py +++ b/letsencrypt/plugins/manual.py @@ -91,7 +91,7 @@ s.serve_forever()" """ help="Automatically allows public IP logging.") def prepare(self): # pylint: disable=missing-docstring,no-self-use - if self.config.noninteractive_mode: + if self.config.noninteractive_mode and not self.conf("test-mode"): raise errors.PluginError("Running manual mode non-interactively is not supported") def more_info(self): # pylint: disable=missing-docstring,no-self-use From c23aa37f4b910e5357c191b729d50e2d7042a715 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 18:06:46 -0800 Subject: [PATCH 142/208] Refactor --csr handling to run early enough for --webroot --- letsencrypt/cli.py | 24 +++++++++++++++++++++++- letsencrypt/client.py | 32 ++------------------------------ letsencrypt/tests/client_test.py | 17 +++++++++++------ 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index de1321ac9..c9c58ea3b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -684,7 +684,7 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(_process_domain) + certr, chain = le_client.obtain_certificate_from_csr(config.domains, config.actual_csr) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) @@ -1106,8 +1106,30 @@ class HelpfulArgumentParser(object): "'certonly' or 'renew' subcommands") parsed_args.break_my_certs = parsed_args.staging = True + if parsed_args.csr: + self.handle_csr(parsed_args) + return parsed_args + def handle_csr(self, parsed_args): + """ + Process a --csr flag. This needs to happen early enought that the + webroot plugin can know about the calls to _process_domain + """ + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") + # TODO: add CN to domains? + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + for d in domains: + _process_domain(parsed_args, d) + parsed_args.actual_csr = csr + csr_domains, config_domains = set(domains), set(parsed_args.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains)) + ) + + def determine_verb(self): """Determines the verb/subcommand provided by the user. diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 413409ded..fd851c163 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -195,7 +195,7 @@ class Client(object): else: self.auth_handler = None - def _obtain_certificate(self, domains, csr): + def obtain_certificate_from_csr(self, domains, csr): """Obtain certificate. Internal function with precondition that `domains` are @@ -228,34 +228,6 @@ class Client(object): authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, domain_callback): - """Obtain certficiate from CSR. - - :param function(config, domains) domain_callback: callback for each - domain extracted from the CSR, to ensure that webroot-map and similar - housekeeping in cli.py is performed correctly - - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). - :rtype: tuple - - """ - - #raise TypeError("About to call %r" % le_util.CSR) - csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") - # TODO: add CN to domains? - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - for d in domains: - domain_callback(self.config, d) - - csr_domains, config_domains = set(domains), set(self.config.domains) - if csr_domains != config_domains: - raise errors.ConfigurationError( - "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" - .format(", ".join(csr_domains), ", ".join(config_domains)) - ) - - return self._obtain_certificate(domains, csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -276,7 +248,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - return self._obtain_certificate(domains, csr) + (key, csr) + return self.obtain_certificate_from_csr(domains, csr) + (key, csr) def obtain_and_enroll_certificate(self, domains): """Obtain and enroll certificate. diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 6a8899c3b..d75237bab 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -109,21 +109,26 @@ class ClientTest(unittest.TestCase): self.client.auth_handler.get_authorizations()) self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) - def test_obtain_certificate_from_csr(self): + # FIXME move parts of this to test_cli.py... + @mock.patch("letsencrypt.cli._process_domain") + def test_obtain_certificate_from_csr(self, mock_process_domain): self._mock_obtain_certificate() - mock_process_domain = mock.MagicMock() + from letsencrypt import cli test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) + mock_parsed_args = mock.MagicMock() with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr - self.client.config.domains = self.eg_domains - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr(mock_process_domain)) + mock_parsed_args.domains = self.eg_domains + mock_parser = mock.MagicMock(cli.HelpfulArgumentParser) + cli.HelpfulArgumentParser.handle_csr(mock_parser, mock_parsed_args) # make sure cli processing occurred cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr(self.eg_domains, test_csr)) # and that the cert was obtained correctly self._check_obtain_certificate() From 4038be9816cbc4b942a6afad0377f84fc6944008 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:06:56 -0800 Subject: [PATCH 143/208] Test manual prepare() --- letsencrypt/plugins/manual_test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/letsencrypt/plugins/manual_test.py b/letsencrypt/plugins/manual_test.py index e16fadd13..e749eb1f9 100644 --- a/letsencrypt/plugins/manual_test.py +++ b/letsencrypt/plugins/manual_test.py @@ -23,16 +23,21 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from letsencrypt.plugins.manual import Authenticator self.config = mock.MagicMock( - http01_port=8080, manual_test_mode=False, manual_public_ip_logging_ok=False) + http01_port=8080, manual_test_mode=False, + manual_public_ip_logging_ok=False, noninteractive_mode=True) self.auth = Authenticator(config=self.config, name="manual") self.achalls = [achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.HTTP01_P, domain="foo.com", account_key=KEY)] config_test_mode = mock.MagicMock( - http01_port=8080, manual_test_mode=True) + http01_port=8080, manual_test_mode=True, noninteractive_mode=True) self.auth_test_mode = Authenticator( config=config_test_mode, name="manual") + def test_prepare(self): + self.assertRaises(errors.PluginError, self.auth.prepare) + self.auth_test_mode.prepare() # error not raised + def test_more_info(self): self.assertTrue(isinstance(self.auth.more_info(), str)) From 70402790a3cb8324eee230f304e8db53ce3442f6 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:07:56 -0800 Subject: [PATCH 144/208] Use --non-interactive instead of --text --- tests/integration/_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 9230cc682..db6e2f0f1 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -19,7 +19,7 @@ letsencrypt_test () { --http-01-port 5002 \ --manual-test-mode \ $store_flags \ - --text \ + --non-interactive \ --no-redirect \ --agree-tos \ --register-unsafely-without-email \ @@ -37,7 +37,7 @@ letsencrypt_test_no_force_renew () { --http-01-port 5002 \ --manual-test-mode \ $store_flags \ - --text \ + --non-interactive \ --no-redirect \ --agree-tos \ --register-unsafely-without-email \ From 3999d65d1c51b0de8f8b1e7a586583ffed038dc6 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 18:06:46 -0800 Subject: [PATCH 145/208] Refactor --csr handling to run early enough for --webroot --- letsencrypt/cli.py | 24 +++++++++++++++++++++++- letsencrypt/client.py | 32 ++------------------------------ letsencrypt/tests/client_test.py | 17 +++++++++++------ 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 99ee7884a..e01275153 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -684,7 +684,7 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(_process_domain) + certr, chain = le_client.obtain_certificate_from_csr(config.domains, config.actual_csr) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) @@ -1106,8 +1106,30 @@ class HelpfulArgumentParser(object): "'certonly' or 'renew' subcommands") parsed_args.break_my_certs = parsed_args.staging = True + if parsed_args.csr: + self.handle_csr(parsed_args) + return parsed_args + def handle_csr(self, parsed_args): + """ + Process a --csr flag. This needs to happen early enought that the + webroot plugin can know about the calls to _process_domain + """ + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") + # TODO: add CN to domains? + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + for d in domains: + _process_domain(parsed_args, d) + parsed_args.actual_csr = csr + csr_domains, config_domains = set(domains), set(parsed_args.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains)) + ) + + def determine_verb(self): """Determines the verb/subcommand provided by the user. diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 413409ded..fd851c163 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -195,7 +195,7 @@ class Client(object): else: self.auth_handler = None - def _obtain_certificate(self, domains, csr): + def obtain_certificate_from_csr(self, domains, csr): """Obtain certificate. Internal function with precondition that `domains` are @@ -228,34 +228,6 @@ class Client(object): authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, domain_callback): - """Obtain certficiate from CSR. - - :param function(config, domains) domain_callback: callback for each - domain extracted from the CSR, to ensure that webroot-map and similar - housekeeping in cli.py is performed correctly - - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). - :rtype: tuple - - """ - - #raise TypeError("About to call %r" % le_util.CSR) - csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") - # TODO: add CN to domains? - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - for d in domains: - domain_callback(self.config, d) - - csr_domains, config_domains = set(domains), set(self.config.domains) - if csr_domains != config_domains: - raise errors.ConfigurationError( - "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" - .format(", ".join(csr_domains), ", ".join(config_domains)) - ) - - return self._obtain_certificate(domains, csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -276,7 +248,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - return self._obtain_certificate(domains, csr) + (key, csr) + return self.obtain_certificate_from_csr(domains, csr) + (key, csr) def obtain_and_enroll_certificate(self, domains): """Obtain and enroll certificate. diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 222e9c707..429945526 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -109,21 +109,26 @@ class ClientTest(unittest.TestCase): self.client.auth_handler.get_authorizations()) self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) - def test_obtain_certificate_from_csr(self): + # FIXME move parts of this to test_cli.py... + @mock.patch("letsencrypt.cli._process_domain") + def test_obtain_certificate_from_csr(self, mock_process_domain): self._mock_obtain_certificate() - mock_process_domain = mock.MagicMock() + from letsencrypt import cli test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) + mock_parsed_args = mock.MagicMock() with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr - self.client.config.domains = self.eg_domains - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr(mock_process_domain)) + mock_parsed_args.domains = self.eg_domains + mock_parser = mock.MagicMock(cli.HelpfulArgumentParser) + cli.HelpfulArgumentParser.handle_csr(mock_parser, mock_parsed_args) # make sure cli processing occurred cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr(self.eg_domains, test_csr)) # and that the cert was obtained correctly self._check_obtain_certificate() From 9af8a875cd6a6511a95d8a3e51885464be3861ad Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 18:12:49 -0800 Subject: [PATCH 146/208] Update hippopotamus test --- letsencrypt/tests/client_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 429945526..dbc57565e 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -118,13 +118,17 @@ class ClientTest(unittest.TestCase): mock_parsed_args = mock.MagicMock() with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: mock_CSR.return_value = test_csr - mock_parsed_args.domains = self.eg_domains + mock_parsed_args.domains = self.eg_domains[:] mock_parser = mock.MagicMock(cli.HelpfulArgumentParser) cli.HelpfulArgumentParser.handle_csr(mock_parser, mock_parsed_args) # make sure cli processing occurred cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) + # Now provoke an inconsistent domains error... + mock_parsed_args.domains.append("hippopotamus.io") + self.assertRaises(errors.ConfigurationError, + cli.HelpfulArgumentParser.handle_csr, mock_parser, mock_parsed_args) self.assertEqual( (mock.sentinel.certr, mock.sentinel.chain), @@ -132,11 +136,6 @@ class ClientTest(unittest.TestCase): # and that the cert was obtained correctly self._check_obtain_certificate() - # Now provoke an inconsistent domains error... - - self.client.config.domains.append("hippopotamus.io") - self.assertRaises(errors.ConfigurationError, - self.client.obtain_certificate_from_csr, mock_process_domain) @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): From 7a902daa9f8480b527e54f6b18d38be7cd36ce73 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:14:29 -0800 Subject: [PATCH 147/208] duplication-- --- tests/boulder-integration.sh | 11 +++++------ tests/integration/_common.sh | 17 ++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index dd6c1835e..cfd0e5c16 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -20,17 +20,16 @@ else readlink="readlink" fi -common() { - letsencrypt_test \ +common_no_force_renew() { + letsencrypt_test_no_force_renew \ --authenticator standalone \ --installer null \ "$@" } -common_no_force_renew() { - letsencrypt_test_no_force_renew \ - --authenticator standalone \ - --installer null \ +common() { + common_no_force_renew \ + --renew-by-default \ "$@" } diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index db6e2f0f1..77a60112b 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -12,21 +12,8 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { - letsencrypt \ - --server "${SERVER:-http://localhost:4000/directory}" \ - --no-verify-ssl \ - --tls-sni-01-port 5001 \ - --http-01-port 5002 \ - --manual-test-mode \ - $store_flags \ - --non-interactive \ - --no-redirect \ - --agree-tos \ - --register-unsafely-without-email \ - --renew-by-default \ - --debug \ - -vvvvvvv \ - "$@" + letsencrypt_test_no_force_renew \ + --renew-by-default } letsencrypt_test_no_force_renew () { From a774922f8f5374c0919d2324661083c64739bd21 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 18:14:55 -0800 Subject: [PATCH 148/208] Revert "Revert "Allow webroot-map and --csr to exist together."" This reverts commit d65a3c65c20ca7c12c4f85802a8bc84a14b95611. --- letsencrypt/cli.py | 8 +------- letsencrypt/client.py | 27 ++++++++++++++++++++------- letsencrypt/plugins/webroot.py | 6 ++++-- letsencrypt/tests/cli_test.py | 5 +---- letsencrypt/tests/client_test.py | 24 +++++++++++++++++------- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c335d8d5b..de1321ac9 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -672,11 +672,6 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals def obtain_cert(config, plugins, lineage=None): """Implements "certonly": authenticate & obtain cert, but do not install it.""" - if config.domains and config.csr is not None: - # TODO: --csr could have a priority, when --domains is - # supplied, check if CSR matches given domains? - return "--domains and --csr are mutually exclusive" - try: # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") @@ -689,8 +684,7 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( - file=config.csr[0], data=config.csr[1], form="der")) + certr, chain = le_client.obtain_certificate_from_csr(_process_domain) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 57b21a55f..413409ded 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -228,21 +228,34 @@ class Client(object): authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, csr): + def obtain_certificate_from_csr(self, domain_callback): """Obtain certficiate from CSR. - :param .le_util.CSR csr: DER-encoded Certificate Signing - Request. + :param function(config, domains) domain_callback: callback for each + domain extracted from the CSR, to ensure that webroot-map and similar + housekeeping in cli.py is performed correctly :returns: `.CertificateResource` and certificate chain (as returned by `.fetch_chain`). :rtype: tuple """ - return self._obtain_certificate( - # TODO: add CN to domains? - crypto_util.get_sans_from_csr( - csr.data, OpenSSL.crypto.FILETYPE_ASN1), csr) + + #raise TypeError("About to call %r" % le_util.CSR) + csr = le_util.CSR(file=self.config.csr[0], data=self.config.csr[1], form="der") + # TODO: add CN to domains? + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + for d in domains: + domain_callback(self.config, d) + + csr_domains, config_domains = set(domains), set(self.config.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains)) + ) + + return self._obtain_certificate(domains, csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. diff --git a/letsencrypt/plugins/webroot.py b/letsencrypt/plugins/webroot.py index f8176417c..3f5bc6d28 100644 --- a/letsencrypt/plugins/webroot.py +++ b/letsencrypt/plugins/webroot.py @@ -49,8 +49,10 @@ to serve all files under specified web root ({0}).""" path_map = self.conf("map") if not path_map: - raise errors.PluginError("--{0} must be set".format( - self.option_name("path"))) + raise errors.PluginError( + "Missing parts of webroot configuration; please set either " + "--webroot-path and --domains, or --webroot-map. Run with " + " --help webroot for examples.") for name, path in path_map.items(): if not os.path.isdir(path): raise errors.PluginError(path + " does not exist or is not a directory") diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index c41f45116..2d36a9d21 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -228,7 +228,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args = ["certonly", "--webroot"] ret, _, _, _ = self._call(args) - self.assertTrue("--webroot-path must be set" in ret) + self.assertTrue("please set either --webroot-path" in ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -323,9 +323,6 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual(config.fullchain_path, os.path.abspath(fullchain)) def test_certonly_bad_args(self): - ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) - self.assertEqual(ret, '--domains and --csr are mutually exclusive') - ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py index 2f117f80c..6a8899c3b 100644 --- a/letsencrypt/tests/client_test.py +++ b/letsencrypt/tests/client_test.py @@ -82,6 +82,7 @@ class ClientTest(unittest.TestCase): no_verify_ssl=False, config_dir="/etc/letsencrypt") # pylint: disable=star-args self.account = mock.MagicMock(**{"key.pem": KEY}) + self.eg_domains = ["example.com", "www.example.com"] from letsencrypt.client import Client with mock.patch("letsencrypt.client.acme_client.Client") as acme: @@ -101,8 +102,7 @@ class ClientTest(unittest.TestCase): self.acme.fetch_chain.return_value = mock.sentinel.chain def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - ["example.com", "www.example.com"]) + self.client.auth_handler.get_authorizations.assert_called_once_with(self.eg_domains) self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, CSR_SAN)), @@ -111,11 +111,21 @@ class ClientTest(unittest.TestCase): def test_obtain_certificate_from_csr(self): self._mock_obtain_certificate() - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr(le_util.CSR( - form="der", file=None, data=CSR_SAN))) - self._check_obtain_certificate() + mock_process_domain = mock.MagicMock() + test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) + with mock.patch("letsencrypt.client.le_util.CSR") as mock_CSR: + mock_CSR.return_value = test_csr + self.client.config.domains = self.eg_domains + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr(mock_process_domain)) + + # make sure cli processing occurred + cli_processed = (call[0][1] for call in mock_process_domain.call_args_list) + self.assertEqual(set(cli_processed), set(("example.com", "www.example.com"))) + + # and that the cert was obtained correctly + self._check_obtain_certificate() @mock.patch("letsencrypt.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): From e798b62d2e47f8ce669da6dfaad813fadd1f442f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:18:48 -0800 Subject: [PATCH 149/208] Testing cleanup --- tests/boulder-integration.sh | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index cfd0e5c16..7e0246085 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -60,36 +60,18 @@ CheckCertCount() { CheckCertCount 1 # This won't renew (because it's not time yet) -letsencrypt_test_no_force_renew --authenticator standalone --installer null renew -tvv +letsencrypt_test_no_force_renew renew CheckCertCount 1 +# --renew-by-default is used, so renewal should occur +letsencrypt_test renew +CheckCertCount 2 + # This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" -letsencrypt_test_no_force_renew --authenticator standalone --installer null renew # --renew-by-default -CheckCertCount 2 - -# Check Param setting in renewal... -letsencrypt_test_no_force_renew --authenticator standalone --installer null renew --renew-by-default +letsencrypt_test_no_force_renew CheckCertCount 3 -# The 4096 bit setting should persist to the first renewal, but be overriden in the second -size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` -size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` -#if ! [ "$size3" -lt "$size2" ] ; then -# echo "key size failure:" -# ls -l ${root}/conf/archive/le.wtf/ -# exit 1 -#fi - - -# dir="$root/conf/archive/le1.wtf" -# for x in cert chain fullchain privkey; -# do -# latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" -# live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" -# [ "${dir}/${latest}" = "$live" ] # renewer fails this test -# done - # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" # revoke renewed From 2170c8d7d2830b0d883d605f17f3831d38ade4a7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:35:44 -0800 Subject: [PATCH 150/208] Move * outside of " --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 7e0246085..5520a75f1 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -51,7 +51,7 @@ common --domains le3.wtf install \ --key-path "${root}/csr/key.pem" CheckCertCount() { - CERTCOUNT=`ls "${root}/conf/archive/le.wtf/cert*" | wc -l` + CERTCOUNT=`ls "${root}/conf/archive/le.wtf/cert"* | wc -l` if [ "$CERTCOUNT" -ne "$1" ] ; then echo Wrong cert count, not "$1" `ls "${root}/conf/archive/le.wtf/"*` exit 1 From 8b613eed8f3a9bb0313cafe887c9cc764355dd8e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:39:59 -0800 Subject: [PATCH 151/208] Pass additional args to letsencrypt_test_no_force_renew --- tests/integration/_common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 77a60112b..e86d087cb 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -13,7 +13,8 @@ export root store_flags letsencrypt_test () { letsencrypt_test_no_force_renew \ - --renew-by-default + --renew-by-default \ + "$@" } letsencrypt_test_no_force_renew () { From 0fa61f4192131892a9e2949d6c77a6cd38e297a0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 8 Feb 2016 18:46:24 -0800 Subject: [PATCH 152/208] Use common and add verb --- tests/boulder-integration.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 5520a75f1..29618b97f 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -60,16 +60,16 @@ CheckCertCount() { CheckCertCount 1 # This won't renew (because it's not time yet) -letsencrypt_test_no_force_renew renew +common_no_force_renew renew CheckCertCount 1 # --renew-by-default is used, so renewal should occur -letsencrypt_test renew +common renew CheckCertCount 2 # This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" -letsencrypt_test_no_force_renew +common_no_force_renew renew CheckCertCount 3 # revoke by account key From a8ba6f7c2c25d279a98c08534da1fc20c051fec7 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 18:54:36 -0800 Subject: [PATCH 153/208] Dry run messages --- letsencrypt/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 96b30d25e..5ebf1ff25 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -419,7 +419,8 @@ def _suggest_donation_if_appropriate(config): def _report_successful_dry_run(): reporter_util = zope.component.getUtility(interfaces.IReporter) - reporter_util.add_message("The dry run was successful.", + reporter_util.add_message("A test certificate requested in dry run was " + "successfully issued.", reporter_util.HIGH_PRIORITY, on_crash=False) @@ -979,8 +980,14 @@ def renew(config, unused_plugins): renew_failures.append(renewal_candidate.fullchain) # Describe all the results + if config.dry_run: + print("** DRY RUN (messages below refer to test certs only!") + print("** The certificates mentioned have not been saved.") _renew_describe_results(renew_successes, renew_failures, renew_skipped, parse_failures) + if config.dry_run: + print("** DRY RUN (messages above refer to test certs only!") + print("** The certificates mentioned have not been saved.") def revoke(config, unused_plugins): # TODO: coop with renewal config From 9bc5523a3b44302c660cc2ddeeedf7138ebbc214 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 8 Feb 2016 19:06:16 -0800 Subject: [PATCH 154/208] Reorganize to make pylint happier --- letsencrypt/cli.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5ebf1ff25..5e6e1fade 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -884,9 +884,12 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def _renew_describe_results(renew_successes, renew_failures, renew_skipped, - parse_failures): +def _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures): status = lambda x, msg: " " + "\n ".join(i + " (" + msg +")" for i in x) + if config.dry_run: + print("** DRY RUN (messages below refer to test certs only!") + print("** The certificates mentioned have not been saved.") print() if renew_skipped: print("The following certs are not due for renewal yet:") @@ -912,6 +915,10 @@ def _renew_describe_results(renew_successes, renew_failures, renew_skipped, "were invalid: ") print(status(parse_failures, "parsefail")) + if config.dry_run: + print("** DRY RUN (messages above refer to test certs only!") + print("** The certificates mentioned have not been saved.") + def renew(config, unused_plugins): """Renew previously-obtained certificates.""" @@ -980,14 +987,8 @@ def renew(config, unused_plugins): renew_failures.append(renewal_candidate.fullchain) # Describe all the results - if config.dry_run: - print("** DRY RUN (messages below refer to test certs only!") - print("** The certificates mentioned have not been saved.") - _renew_describe_results(renew_successes, renew_failures, renew_skipped, - parse_failures) - if config.dry_run: - print("** DRY RUN (messages above refer to test certs only!") - print("** The certificates mentioned have not been saved.") + _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures) def revoke(config, unused_plugins): # TODO: coop with renewal config From 63c0718d869a95eadc2e05a38d5df375d71ee07c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 19:15:28 -0800 Subject: [PATCH 155/208] Accept --csr PEMFILE * Closes: #1082 #1935 * Also produce better errors if SANs are missing, though not yet fixing #1076 --- letsencrypt/cli.py | 28 +++++++++++++++++++++++----- letsencrypt/client.py | 7 ++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c9c58ea3b..d9497d8fe 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -684,7 +684,8 @@ def obtain_cert(config, plugins, lineage=None): # This is a special case; cert and chain are simply saved if config.csr is not None: assert lineage is None, "Did not expect a CSR with a RenewableCert" - certr, chain = le_client.obtain_certificate_from_csr(config.domains, config.actual_csr) + csr, typ = config.actual_csr + certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) if config.dry_run: logger.info( "Dry run: skipping saving certificate to %s", config.cert_path) @@ -1116,12 +1117,29 @@ class HelpfulArgumentParser(object): Process a --csr flag. This needs to happen early enought that the webroot plugin can know about the calls to _process_domain """ - csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") - # TODO: add CN to domains? - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + try: + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") + typ = OpenSSL.crypto.FILETYPE_ASN1 + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + except: + try: + e1 = traceback.format_exc() + typ = OpenSSL.crypto.FILETYPE_PEM + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="pem") + domains = crypto_util.get_sans_from_csr(csr.data, typ) + except: + logger.debug("DER CSR parse error %s", e1) + logger.debug("PEM CSR parse error %s", traceback.format_exc()) + raise errors.Error("Failed to CSR file: %s", parsed_args.csr[0]) + + if not domains: + # TODO: add CN to domains instead: + raise errors.Error( + "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" + % parsed_args.csr[0]) for d in domains: _process_domain(parsed_args, d) - parsed_args.actual_csr = csr + parsed_args.actual_csr = (csr, typ) csr_domains, config_domains = set(domains), set(parsed_args.domains) if csr_domains != config_domains: raise errors.ConfigurationError( diff --git a/letsencrypt/client.py b/letsencrypt/client.py index fd851c163..9dfa70e8d 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -195,7 +195,8 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr): + def obtain_certificate_from_csr(self, domains, csr, + typ=OpenSSL.crypto.FILETYPE_ASN1): """Obtain certificate. Internal function with precondition that `domains` are @@ -223,8 +224,8 @@ class Client(object): authzr = self.auth_handler.get_authorizations(domains) certr = self.acme.request_issuance( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.data)), + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(typ, csr.data)), authzr) return certr, self.acme.fetch_chain(certr) From b2e460f34bb5b29a545c6a4aa61d9cb56f9b6977 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 19:54:32 -0800 Subject: [PATCH 156/208] Address the comments of reviewers & lintmonsters --- letsencrypt/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index d9497d8fe..375495833 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1114,23 +1114,23 @@ class HelpfulArgumentParser(object): def handle_csr(self, parsed_args): """ - Process a --csr flag. This needs to happen early enought that the + Process a --csr flag. This needs to happen early enough that the webroot plugin can know about the calls to _process_domain """ try: csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") typ = OpenSSL.crypto.FILETYPE_ASN1 domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - except: + except OpenSSL.crypto.Error: try: e1 = traceback.format_exc() typ = OpenSSL.crypto.FILETYPE_PEM csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="pem") domains = crypto_util.get_sans_from_csr(csr.data, typ) - except: + except OpenSSL.crypto.Error: logger.debug("DER CSR parse error %s", e1) logger.debug("PEM CSR parse error %s", traceback.format_exc()) - raise errors.Error("Failed to CSR file: %s", parsed_args.csr[0]) + raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) if not domains: # TODO: add CN to domains instead: @@ -1143,7 +1143,7 @@ class HelpfulArgumentParser(object): csr_domains, config_domains = set(domains), set(parsed_args.domains) if csr_domains != config_domains: raise errors.ConfigurationError( - "Inconsistent domain requests:\ncsr: {0}\ncli config: {1}" + "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" .format(", ".join(csr_domains), ", ".join(config_domains)) ) From ff9d7a7b802f1e272ad379bdfec9010b9f701246 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 20:21:08 -0800 Subject: [PATCH 157/208] Restore old versions of some tests, port others --- letsencrypt/tests/cli_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index ff11b1dde..de64fce04 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -222,16 +222,16 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods if "nginx" in real_plugins: # Sending nginx a non-existent conf dir will simulate misconfiguration # (we can only do that if letsencrypt-nginx is actually present) - self._call(args) - # XXX: This probably now raises an exception (when nginx is - # present, but I don't know which one!) - # self.assertTrue("The nginx plugin is not working" in ret) - # self.assertTrue("MisconfigurationError" in ret) + ret, _, _, _ = self._call(args) + self.assertTrue("The nginx plugin is not working" in ret) + self.assertTrue("MisconfigurationError" in ret) args = ["certonly", "--webroot"] - # ret, _, _, _ = self._call(args) - self.assertRaises(errors.PluginSelectionError, self._call, args) - # self.assertTrue("--webroot-path must be set" in ret) + try: + self._call(args) + assert False, "Exception should have been raised" + except errors.PluginSelectionError as e: + self.assertTrue("--webroot-path must be set" in e.message) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") From b2de2cd1816d1d1646742822eb55925d220fa3e9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 20:21:40 -0800 Subject: [PATCH 158/208] Better dry run reporting --- letsencrypt/cli.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5e6e1fade..414c3b0ce 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -417,11 +417,12 @@ def _suggest_donation_if_appropriate(config): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) -def _report_successful_dry_run(): + +def _report_successful_dry_run(config): reporter_util = zope.component.getUtility(interfaces.IReporter) - reporter_util.add_message("A test certificate requested in dry run was " - "successfully issued.", - reporter_util.HIGH_PRIORITY, on_crash=False) + if config.verb != "renew": + reporter_util.add_message("The dry run was successful.", + reporter_util.HIGH_PRIORITY, on_crash=False) def _auth_from_domains(le_client, config, domains, lineage=None): @@ -709,7 +710,7 @@ def obtain_cert(config, plugins, lineage=None): _auth_from_domains(le_client, config, domains, lineage) if config.dry_run: - _report_successful_dry_run() + _report_successful_dry_run(config) elif config.verb == "renew": if installer is None: # Tell the user that the server was not restarted. From 28c54476533fad62c7bc5d90d123fbb74d90a2be Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 20:30:48 -0800 Subject: [PATCH 159/208] Port another test --- letsencrypt/cli.py | 3 +-- letsencrypt/tests/cli_test.py | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 414c3b0ce..064de1b3d 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -686,8 +686,7 @@ def obtain_cert(config, plugins, lineage=None): # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") except errors.PluginSelectionError as e: - logger.info( - "Could not choose appropriate plugin: %s", e) + logger.info("Could not choose appropriate plugin: %s", e) raise # TODO: Handle errors from _init_le_client? diff --git a/letsencrypt/tests/cli_test.py b/letsencrypt/tests/cli_test.py index de64fce04..07029ca66 100644 --- a/letsencrypt/tests/cli_test.py +++ b/letsencrypt/tests/cli_test.py @@ -329,10 +329,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods ret, _, _, _ = self._call(['-d', 'foo.bar', 'certonly', '--csr', CSR]) self.assertEqual(ret, '--domains and --csr are mutually exclusive') - # ret, _, _, _ = self._call(['-a', 'bad_auth', 'certonly']) - self.assertRaises(errors.PluginSelectionError, self._call, - ['-a', 'bad_auth', 'certonly']) - # self.assertEqual(ret, 'The requested bad_auth plugin does not appear to be installed') + try: + self._call(['-a', 'bad_auth', 'certonly']) + assert False, "Exception should have been raised" + except errors.PluginSelectionError as e: + self.assertTrue('The requested bad_auth plugin does not appear' in e.message) def test_check_config_sanity_domain(self): # Punycode From 57ee4f0b4684364f6f9cf62fc8a1589fe5bcd579 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 20:43:13 -0800 Subject: [PATCH 160/208] Nicen dry run renewal messages --- letsencrypt/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 064de1b3d..758c3e7f2 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -888,8 +888,8 @@ def _renew_describe_results(config, renew_successes, renew_failures, renew_skipped, parse_failures): status = lambda x, msg: " " + "\n ".join(i + " (" + msg +")" for i in x) if config.dry_run: - print("** DRY RUN (messages below refer to test certs only!") - print("** The certificates mentioned have not been saved.") + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates below have not been saved.)") print() if renew_skipped: print("The following certs are not due for renewal yet:") @@ -916,8 +916,8 @@ def _renew_describe_results(config, renew_successes, renew_failures, print(status(parse_failures, "parsefail")) if config.dry_run: - print("** DRY RUN (messages above refer to test certs only!") - print("** The certificates mentioned have not been saved.") + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates above have not been saved.)") def renew(config, unused_plugins): From e0cfd9f691fb9efd6197d41ecacfac4b1cf443be Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 21:10:34 -0800 Subject: [PATCH 161/208] Extra CSR sanity checking --- letsencrypt/cli.py | 15 +++++++++++---- letsencrypt/le_util.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 375495833..ac6e2c937 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1132,20 +1132,27 @@ class HelpfulArgumentParser(object): logger.debug("PEM CSR parse error %s", traceback.format_exc()) raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) + for d in domains: + _process_domain(parsed_args, d) + + for d in domains: + sanitised = le_util.enforce_domain_sanity(d): + if d.lower() != sanitised: + raise errors.ConfigurationError( + "CSR domain {0} needs to be sanitised to {1}.".format(d, sanitised)) + if not domains: # TODO: add CN to domains instead: raise errors.Error( "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" % parsed_args.csr[0]) - for d in domains: - _process_domain(parsed_args, d) + parsed_args.actual_csr = (csr, typ) csr_domains, config_domains = set(domains), set(parsed_args.domains) if csr_domains != config_domains: raise errors.ConfigurationError( "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" - .format(", ".join(csr_domains), ", ".join(config_domains)) - ) + .format(", ".join(csr_domains), ", ".join(config_domains))) def determine_verb(self): diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py index 35793849e..527c9bdae 100644 --- a/letsencrypt/le_util.py +++ b/letsencrypt/le_util.py @@ -308,7 +308,7 @@ def enforce_domain_sanity(domain): # Unicode try: - domain = domain.encode('ascii') + domain = domain.encode('ascii').lower() except UnicodeDecodeError: raise errors.ConfigurationError( "Internationalized domain names are not presently supported: {0}" From 4000aa762ef3235f576b013725d95e402ef8ed48 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 21:14:45 -0800 Subject: [PATCH 162/208] Fix snauf --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 7791d2819..533539684 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1202,7 +1202,7 @@ class HelpfulArgumentParser(object): _process_domain(parsed_args, d) for d in domains: - sanitised = le_util.enforce_domain_sanity(d): + sanitised = le_util.enforce_domain_sanity(d) if d.lower() != sanitised: raise errors.ConfigurationError( "CSR domain {0} needs to be sanitised to {1}.".format(d, sanitised)) From d6703f771ac158941360cce74642cc27b2e717e7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 8 Feb 2016 21:28:43 -0800 Subject: [PATCH 163/208] Lint & cleanup weirdness from #2392... --- letsencrypt/cli.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 533539684..00d45b700 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -943,7 +943,6 @@ def renew(config, unused_plugins): try: renewal_candidate = _reconstitute(lineage_config, renewal_file) except Exception as e: # pylint: disable=broad-except - # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) logger.debug("Traceback was:\n%s", traceback.format_exc()) @@ -954,24 +953,12 @@ def renew(config, unused_plugins): if renewal_candidate is None: parse_failures.append(renewal_file) else: - # _reconstitute succeeded in producing a RenewableCert, so we - # have something to work with from this particular config file. - # XXX: ensure that each call here replaces the previous one zope.component.provideUtility(lineage_config) - # Although obtain_cert itself also indirectly decides - # whether to renew or not, we need to check at this - # stage in order to avoid claiming that renewal - # succeeded when it wasn't even attempted (since - # obtain_cert wouldn't raise an error in that case). if _should_renew(lineage_config, renewal_candidate): - err = obtain_cert(lineage_config, - plugins_disco.PluginsRegistry.find_all(), - renewal_candidate) - if err is None: - renew_successes.append(renewal_candidate.fullchain) - else: - renew_failures.append(renewal_candidate.fullchain) + plugins = plugins_disco.PluginsRegistry.find_all() + obtain_cert(lineage_config, plugins, renewal_candidate) + renew_successes.append(renewal_candidate.fullchain) else: renew_skipped.append(renewal_candidate.fullchain) except Exception as e: # pylint: disable=broad-except @@ -1197,7 +1184,6 @@ class HelpfulArgumentParser(object): logger.debug("DER CSR parse error %s", e1) logger.debug("PEM CSR parse error %s", traceback.format_exc()) raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) - for d in domains: _process_domain(parsed_args, d) From 3603f482e5fe81cb5948699a77a2abaa16a8839d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:06:59 -0800 Subject: [PATCH 164/208] Resume using a fully-constructed config namespace --- letsencrypt/cli.py | 4 ++-- letsencrypt/configuration.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 35211c0bd..a385f5e05 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -744,8 +744,8 @@ def _set_by_cli(var): plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = _parser.args + [_parser.verb] - _set_by_cli.detector = prepare_and_parse_args(plugins, reconstructed_args, - detect_defaults=True) + default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) + _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) try: # Is detector.var something that isn't false? diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 2bbf1b019..979d5e985 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -34,7 +34,7 @@ class NamespaceConfig(object): """ zope.interface.implements(interfaces.IConfig) - def __init__(self, namespace): + def __init__(self, namespace, fake=False): self.namespace = namespace self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) @@ -42,7 +42,8 @@ class NamespaceConfig(object): self.namespace.logs_dir = os.path.abspath(self.namespace.logs_dir) # Check command line parameters sanity, and error out in case of problem. - check_config_sanity(self) + if not fake: + check_config_sanity(self) def __getattr__(self, name): return getattr(self.namespace, name) From 60392cce04cf25989e4dd5acb8633278650e6e5b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:40:09 -0800 Subject: [PATCH 165/208] Try to reconstruct all the plugin cli vars --- letsencrypt/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 83dec0c32..395ad2dd7 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -753,6 +753,9 @@ def _set_by_cli(var): reconstructed_args = _parser.args + [_parser.verb] default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) + # propagate plugin requests: eg --standalone modifies config.authenticator + plugin_reqs = cli_plugin_requests(_set_by_cli.detector) + _set_by_cli.detector.authenticator, _set_by_cli.detector.installer = plugin_reqs try: # Is detector.var something that isn't false? From 55f1840d834153019cb7df722439e1c8481cf1c3 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:41:52 -0800 Subject: [PATCH 166/208] Make static var less verbose --- letsencrypt/cli.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 395ad2dd7..27c9069c2 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -745,21 +745,22 @@ def _set_by_cli(var): (CLI or config file) including if the user explicitly set it to the default. Returns False if the variable was assigned a default value. """ - if _set_by_cli.detector is None: + detector = _set_by_cli.detector + if detector is None: # Setup on first run: `detector` is a weird version of config in which # the default value of every attribute is wrangled to be boolean-false plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = _parser.args + [_parser.verb] default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) - _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) + detector = _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) # propagate plugin requests: eg --standalone modifies config.authenticator - plugin_reqs = cli_plugin_requests(_set_by_cli.detector) - _set_by_cli.detector.authenticator, _set_by_cli.detector.installer = plugin_reqs + plugin_reqs = cli_plugin_requests(detector) + detector.authenticator, detector.installer = plugin_reqs try: # Is detector.var something that isn't false? - change_detected = _set_by_cli.detector.__getattr__(var) + change_detected = detector.__getattr__(var) except AttributeError: logger.warning("Missing default analysis for %r", var) return False @@ -768,7 +769,7 @@ def _set_by_cli(var): return True # Special case: vars like --no-redirect that get set True -> False # default to None; False means they were set - elif var in _set_by_cli.detector.store_false_vars and change_detected is not None: + elif var in detector.store_false_vars and change_detected is not None: return True else: return False From 4a7a0bd47ad5bb110b3416eca850eee38e01b2c4 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:52:10 -0800 Subject: [PATCH 167/208] Update detector's namespace correctly --- letsencrypt/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 27c9069c2..f4524d6b7 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -755,8 +755,13 @@ def _set_by_cli(var): default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) detector = _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) # propagate plugin requests: eg --standalone modifies config.authenticator - plugin_reqs = cli_plugin_requests(detector) - detector.authenticator, detector.installer = plugin_reqs + auth, inst = cli_plugin_requests(detector) + if auth: + detector.namespace.__setattr__("authenticator", auth) + if inst: + detector.namespace.__setattr__("installer", inst) + # more spammy than just debug + logger.log(-10, "Default Detector is %r", auth, inst, detector.namespace) try: # Is detector.var something that isn't false? From 5514776a7e42d246103c5ba029803e0f9791beb9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:53:48 -0800 Subject: [PATCH 168/208] lint / fix --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f4524d6b7..589a403e0 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -761,7 +761,7 @@ def _set_by_cli(var): if inst: detector.namespace.__setattr__("installer", inst) # more spammy than just debug - logger.log(-10, "Default Detector is %r", auth, inst, detector.namespace) + logger.log(-10, "Default Detector is %r",detector.namespace) try: # Is detector.var something that isn't false? From d0d63b65b85202cd90a8984f4b317683ebbb6d3c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 10:54:24 -0800 Subject: [PATCH 169/208] lintlint --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 589a403e0..302df26cc 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -761,7 +761,7 @@ def _set_by_cli(var): if inst: detector.namespace.__setattr__("installer", inst) # more spammy than just debug - logger.log(-10, "Default Detector is %r",detector.namespace) + logger.log(-10, "Default Detector is %r", detector.namespace) try: # Is detector.var something that isn't false? From 0fb3bf689db5bd29b6765985df9861350736f3a9 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 11:46:43 -0800 Subject: [PATCH 170/208] More debugging. --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 302df26cc..f5a00ba8f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -761,7 +761,7 @@ def _set_by_cli(var): if inst: detector.namespace.__setattr__("installer", inst) # more spammy than just debug - logger.log(-10, "Default Detector is %r", detector.namespace) + logger.debug("Default Detector is %r", detector.namespace) try: # Is detector.var something that isn't false? From 13aed36cd5ce6e3e4e974a6fbe0065d51c65181d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 12:22:36 -0800 Subject: [PATCH 171/208] "" is a reasonable server value in default_detection=True --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index f5a00ba8f..25eb821aa 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1195,7 +1195,7 @@ class HelpfulArgumentParser(object): parsed_args.domains.append(domain) if parsed_args.staging or parsed_args.dry_run: - if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): + if parsed_args.server not in ("", flag_default("server"), constants.STAGING_URI): conflicts = ["--staging"] if parsed_args.staging else [] conflicts += ["--dry-run"] if parsed_args.dry_run else [] if not self.detect_defaults: From 56be2e054cacb464df970235e918c55ffd6f5010 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 12:24:25 -0800 Subject: [PATCH 172/208] For testing purposes, implement a way to specify which renewal files are run --- letsencrypt/cli.py | 8 +++++++- letsencrypt/constants.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 25eb821aa..cce69ca47 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -918,7 +918,7 @@ def _reconstitute(config, full_path): def _renewal_conf_files(config): """Return /path/to/*.conf in the renewal conf directory""" - return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + return glob.glob(os.path.join(config.renewal_configs_dir, config.renewal_glob)) def _renew_describe_results(config, renew_successes, renew_failures, @@ -1356,6 +1356,9 @@ class HelpfulArgumentParser(object): for var in args: self.store_false_vars[var] = True + if "--server" in args: + print("Munged? server to", kwargs) + def add_deprecated_argument(self, argument_name, num_args): """Adds a deprecated argument with the name argument_name. @@ -1484,6 +1487,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "automation", "--expand", action="store_true", help="If an existing cert covers some subset of the requested names, " "always expand and replace it with the additional names.") + helpful.add( + "automation", "--renewal-glob", default=flag_default("renewal_glob"), + help="A pattern for which renewal files in /etc/letsencrypt/renewal/ to process") helpful.add( "automation", "--version", action="version", version="%(prog)s {0}".format(letsencrypt.__version__), diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index 402f5e9a1..dc543980e 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -22,6 +22,7 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", + renewal_glob="*.conf", no_verify_ssl=False, http01_port=challenges.HTTP01Response.PORT, tls_sni_01_port=challenges.TLSSNI01Response.PORT, From 04926a4662fa74566c12fd136425b1592aeb236e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 12:44:05 -0800 Subject: [PATCH 173/208] Fix some bugs: - modify_arg_for_default_detection was not actually succeeding in modifying the kwargs it was called with - it should be good enough for the default detector to notice --dry-run and pass that to the real conf, we don't need to play with .server as well... --- letsencrypt/cli.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index cce69ca47..24cea0e30 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1194,13 +1194,14 @@ class HelpfulArgumentParser(object): if domain not in parsed_args.domains: parsed_args.domains.append(domain) - if parsed_args.staging or parsed_args.dry_run: - if parsed_args.server not in ("", flag_default("server"), constants.STAGING_URI): - conflicts = ["--staging"] if parsed_args.staging else [] - conflicts += ["--dry-run"] if parsed_args.dry_run else [] - if not self.detect_defaults: - raise errors.Error("--server value conflicts with {0}".format( - " and ".join(conflicts))) + if not self.detect_defaults: + if parsed_args.staging or parsed_args.dry_run: + if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): + conflicts = ["--staging"] if parsed_args.staging else [] + conflicts += ["--dry-run"] if parsed_args.dry_run else [] + if not self.detect_defaults: + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) parsed_args.server = constants.STAGING_URI @@ -1316,7 +1317,7 @@ class HelpfulArgumentParser(object): """ if self.detect_defaults: - self.modify_arg_for_default_detection(self, *args, **kwargs) + kwargs = self.modify_arg_for_default_detection(self, *args, **kwargs) if self.visible_topics[topic]: if topic in self.groups: @@ -1336,6 +1337,8 @@ class HelpfulArgumentParser(object): :param list *args: the names of this argument flag :param dict **kwargs: various argparse settings for this argument + + :returns: a modified versions of kwargs """ # argument either doesn't have a default, or the default doesn't # isn't Pythonically false @@ -1356,8 +1359,7 @@ class HelpfulArgumentParser(object): for var in args: self.store_false_vars[var] = True - if "--server" in args: - print("Munged? server to", kwargs) + return kwargs def add_deprecated_argument(self, argument_name, num_args): From ed348f5528c3496203a1dd58317db2c711a9d4a7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 12:50:49 -0800 Subject: [PATCH 174/208] Actually all of this logic was fine, and we do need to bring in the staging server value --- letsencrypt/cli.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 24cea0e30..0111b6f10 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1194,14 +1194,13 @@ class HelpfulArgumentParser(object): if domain not in parsed_args.domains: parsed_args.domains.append(domain) - if not self.detect_defaults: - if parsed_args.staging or parsed_args.dry_run: - if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): - conflicts = ["--staging"] if parsed_args.staging else [] - conflicts += ["--dry-run"] if parsed_args.dry_run else [] - if not self.detect_defaults: - raise errors.Error("--server value conflicts with {0}".format( - " and ".join(conflicts))) + if parsed_args.staging or parsed_args.dry_run: + if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): + conflicts = ["--staging"] if parsed_args.staging else [] + conflicts += ["--dry-run"] if parsed_args.dry_run else [] + if not self.detect_defaults: + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) parsed_args.server = constants.STAGING_URI From 3a9f91a16929db0480bf2b761821a7ba4213a6cf Mon Sep 17 00:00:00 2001 From: Gian Carlo Pace Date: Tue, 9 Feb 2016 22:39:17 +0100 Subject: [PATCH 175/208] added a missing space that was causing an error in letsencrypt-auto script --- letsencrypt-auto-source/letsencrypt-auto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c87e4c000..24b62e342 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -372,7 +372,7 @@ Bootstrap() { elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon - elif [ -f /etc/os-release] && `grep -q openSUSE /etc/os-release` ; then + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then echo "Bootstrapping dependencies for openSUSE-based OSes..." BootstrapSuseCommon elif [ -f /etc/arch-release ]; then From 6ca84acc1cb9f76bb998f51165127f2887af9503 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 14:19:53 -0800 Subject: [PATCH 176/208] Fix more bugs * setting --dry-run or --staging implies changing account * --dry-run: be willing to make a staging account if the user only has a prod one --- letsencrypt/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 0111b6f10..1465aa789 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -416,7 +416,6 @@ def _suggest_donation_if_appropriate(config): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) - def _report_successful_dry_run(config): reporter_util = zope.component.getUtility(interfaces.IReporter) if config.verb != "renew": @@ -772,6 +771,10 @@ def _set_by_cli(var): if change_detected: return True + # Special case: we actually want account to be set to "" if the server + # the account was on has changed + elif var == "account" and (detector.server or detector.dry_run or detector.staging): + return True # Special case: vars like --no-redirect that get set True -> False # default to None; False means they were set elif var in detector.store_false_vars and change_detected is not None: @@ -1209,6 +1212,11 @@ class HelpfulArgumentParser(object): raise errors.Error("--dry-run currently only works with the " "'certonly' or 'renew' subcommands (%r)" % self.verb) parsed_args.break_my_certs = parsed_args.staging = True + if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): + # The user has a prod account, but might not have a staging + # one; we don't want to start trying to perform interactive registration + parsed_args.agree_tos = True + parsed_args.register_unsafely_without_email = True if parsed_args.csr: self.handle_csr(parsed_args) From 0ab54820c6f9b282f411cccc170123540e713ff2 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 14:25:18 -0800 Subject: [PATCH 177/208] Non-interactive menus were broken if labelled with help... --- letsencrypt/display/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 93b8f6d91..976a2afdf 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -446,7 +446,7 @@ class NoninteractiveDisplay(object): line=os.linesep, frame=side_frame, msg=message)) def menu(self, message, choices, ok_label=None, cancel_label=None, - default=None, cli_flag=None): + help_label=None, default=None, cli_flag=None): # pylint: disable=unused-argument,too-many-arguments """Avoid displaying a menu. From d34c6779e8535396bf8e50876341aaeebd190c7f Mon Sep 17 00:00:00 2001 From: Gian Carlo Pace Date: Tue, 9 Feb 2016 23:34:38 +0100 Subject: [PATCH 178/208] added a missing space in letsencrypt-auto.template as well --- letsencrypt-auto-source/letsencrypt-auto.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index de4844c9e..ad8c97a7f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -128,7 +128,7 @@ Bootstrap() { elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon - elif [ -f /etc/os-release] && `grep -q openSUSE /etc/os-release` ; then + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then echo "Bootstrapping dependencies for openSUSE-based OSes..." BootstrapSuseCommon elif [ -f /etc/arch-release ]; then From c8b89a9f5adda40c539033d555c174fb5396e9ce Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 14:50:32 -0800 Subject: [PATCH 179/208] Revert "For testing purposes, implement a way to specify which renewal files are run" This reverts commit 56be2e054cacb464df970235e918c55ffd6f5010. --- letsencrypt/cli.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 1465aa789..9ed6cdd8f 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -921,7 +921,7 @@ def _reconstitute(config, full_path): def _renewal_conf_files(config): """Return /path/to/*.conf in the renewal conf directory""" - return glob.glob(os.path.join(config.renewal_configs_dir, config.renewal_glob)) + return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def _renew_describe_results(config, renew_successes, renew_failures, @@ -1496,9 +1496,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "automation", "--expand", action="store_true", help="If an existing cert covers some subset of the requested names, " "always expand and replace it with the additional names.") - helpful.add( - "automation", "--renewal-glob", default=flag_default("renewal_glob"), - help="A pattern for which renewal files in /etc/letsencrypt/renewal/ to process") helpful.add( "automation", "--version", action="version", version="%(prog)s {0}".format(letsencrypt.__version__), From 8a5d40fc35fdaee47436271d1a7b760182df0393 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2016 15:41:42 -0800 Subject: [PATCH 180/208] Make add_deprecated_argument more readable --- letsencrypt-apache/letsencrypt_apache/configurator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 0ab16ff06..c61980f01 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -106,7 +106,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): add("handle-sites", default=constants.os_constant("handle_sites"), help="Let installer handle enabling sites for you." + "(Only Ubuntu/Debian currently)") - le_util.add_deprecated_argument(add, "init-script", 1) + le_util.add_deprecated_argument( + add, argument_name="init-script", nargs=1) def __init__(self, *args, **kwargs): """Initialize an Apache Configurator. From 8f283924cd8f672a6593a2276cc011403d036aaa Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2016 15:43:51 -0800 Subject: [PATCH 181/208] Add --apache-ctl as deprecated arg --- letsencrypt-apache/letsencrypt_apache/configurator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index c61980f01..cbc451ac9 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -106,6 +106,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): add("handle-sites", default=constants.os_constant("handle_sites"), help="Let installer handle enabling sites for you." + "(Only Ubuntu/Debian currently)") + le_util.add_deprecated_argument(add, argument_name="ctl", nargs=1) le_util.add_deprecated_argument( add, argument_name="init-script", nargs=1) From 0513af83f408612f317768180f351fafd76f1275 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 16:09:02 -0800 Subject: [PATCH 182/208] Test CLI flag setting from renewal integration tests Closes: #2411 --- tests/boulder-integration.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 29618b97f..32c292e90 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -35,7 +35,7 @@ common() { common --domains le1.wtf --standalone-supported-challenges tls-sni-01 auth common --domains le2.wtf --standalone-supported-challenges http-01 run -common -a manual -d le.wtf auth +common -a manual -d le.wtf auth --rsa-key-size 4096 export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf @@ -69,9 +69,21 @@ CheckCertCount 2 # This will renew because the expiry is less than 10 years from now sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" -common_no_force_renew renew +common_no_force_renew renew --rsa-key-size 2048 CheckCertCount 3 +# The 4096 bit setting should persist to the first renewal, but be overriden in the second + +size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes +if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then + echo key sizes violate assumptions: + ls -l "${root}/conf/archive/le.wtf/privkey"* + exit 1 +fi + # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" # revoke renewed From 3ca769e634d8f45b5c8c45baee34e5640ef31674 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Tue, 9 Feb 2016 16:09:30 -0800 Subject: [PATCH 183/208] Update logging to mention --force-renew --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c98b3f0d7..e0a07a94b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -282,7 +282,7 @@ def _treat_as_renewal(config, domains): def _should_renew(config, lineage): "Return true if any of the circumstances for automatic renewal apply." if config.renew_by_default: - logger.info("Auto-renewal forced with --renew-by-default...") + logger.info("Auto-renewal forced with --force-renewal...") return True if lineage.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") From b4b584155e99a0c823630fd58ec0797263737e30 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 17:40:01 -0800 Subject: [PATCH 184/208] Address stylistic review comments --- letsencrypt/cli.py | 16 +++++++--------- letsencrypt/constants.py | 1 - 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 9ed6cdd8f..aa69df3c6 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -755,11 +755,8 @@ def _set_by_cli(var): detector = _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) # propagate plugin requests: eg --standalone modifies config.authenticator auth, inst = cli_plugin_requests(detector) - if auth: - detector.namespace.__setattr__("authenticator", auth) - if inst: - detector.namespace.__setattr__("installer", inst) - # more spammy than just debug + detector.namespace.authenticator = auth if auth else "" + detector.namespace.installer = inst if inst else "" logger.debug("Default Detector is %r", detector.namespace) try: @@ -915,7 +912,7 @@ def _reconstitute(config, full_path): return None if not _set_by_cli("domains"): - setattr(config.namespace, "domains", domains) + config.namespace.domains = domains return renewal_candidate @@ -1158,9 +1155,10 @@ class HelpfulArgumentParser(object): self.parser._add_config_file_help = False # pylint: disable=protected-access self.silent_parser = SilentParser(self.parser) - # This setting attempts to force all default values to None; it - # is used to detect when values have been explicitly set by the user, - # including when they are set to their normal default value + # This setting attempts to force all default values to things that are + # pythonically false; it is used to detect when values have been + # explicitly set by the user, including when they are set to their + # normal default value self.detect_defaults = detect_defaults if detect_defaults: self.store_false_vars = {} # vars that use "store_false" diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index dc543980e..402f5e9a1 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -22,7 +22,6 @@ CLI_DEFAULTS = dict( config_dir="/etc/letsencrypt", work_dir="/var/lib/letsencrypt", logs_dir="/var/log/letsencrypt", - renewal_glob="*.conf", no_verify_ssl=False, http01_port=challenges.HTTP01Response.PORT, tls_sni_01_port=challenges.TLSSNI01Response.PORT, From 64600ff61c1bb1b28881e8a372078dd5fc883ed8 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 9 Feb 2016 17:52:30 -0800 Subject: [PATCH 185/208] Place a signpost about default detection for plugin args --- docs/contributing.rst | 3 +- letsencrypt/plugins/common.py | 56 +++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 8a27d565e..36dff01b9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -157,7 +157,7 @@ Plugin-architecture Let's Encrypt has a plugin architecture to facilitate support for different webservers, other TLS servers, and operating systems. The interfaces available for plugins to implement are defined in -`interfaces.py`_. +`interfaces.py`_ and `plugins/common.py`_. The most common kind of plugin is a "Configurator", which is likely to implement the `~letsencrypt.interfaces.IAuthenticator` and @@ -168,6 +168,7 @@ There are also `~letsencrypt.interfaces.IDisplay` plugins, which implement bindings to alternative UI libraries. .. _interfaces.py: https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py +.. _plugins/common.py: https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L34 Authenticators diff --git a/letsencrypt/plugins/common.py b/letsencrypt/plugins/common.py index f18b1fb3b..bf996ba5e 100644 --- a/letsencrypt/plugins/common.py +++ b/letsencrypt/plugins/common.py @@ -41,6 +41,36 @@ class Plugin(object): self.config = config self.name = name + @jose_util.abstractclassmethod + def add_parser_arguments(cls, add): + """Add plugin arguments to the CLI argument parser. + + :param callable add: Function that proxies calls to + `argparse.ArgumentParser.add_argument` prepending options + with unique plugin name prefix. + + NOTE: if you add argpase arguments such that users setting them can + create a config entry that python's bool() would consider false (ie, + the use might set the variable to "", [], 0, etc), please ensure that + cli._set_by_cli() works for your variable. + + """ + + @classmethod + def inject_parser_options(cls, parser, name): + """Inject parser options. + + See `~.IPlugin.inject_parser_options` for docs. + + """ + # dummy function, doesn't check if dest.startswith(self.dest_namespace) + def add(arg_name_no_prefix, *args, **kwargs): + # pylint: disable=missing-docstring + return parser.add_argument( + "--{0}{1}".format(option_namespace(name), arg_name_no_prefix), + *args, **kwargs) + return cls.add_parser_arguments(add) + @property def option_namespace(self): """ArgumentParser options namespace (prefix of all options).""" @@ -64,32 +94,6 @@ class Plugin(object): def conf(self, var): """Find a configuration value for variable ``var``.""" return getattr(self.config, self.dest(var)) - - @classmethod - def inject_parser_options(cls, parser, name): - """Inject parser options. - - See `~.IPlugin.inject_parser_options` for docs. - - """ - # dummy function, doesn't check if dest.startswith(self.dest_namespace) - def add(arg_name_no_prefix, *args, **kwargs): - # pylint: disable=missing-docstring - return parser.add_argument( - "--{0}{1}".format(option_namespace(name), arg_name_no_prefix), - *args, **kwargs) - return cls.add_parser_arguments(add) - - @jose_util.abstractclassmethod - def add_parser_arguments(cls, add): - """Add plugin arguments to the CLI argument parser. - - :param callable add: Function that proxies calls to - `argparse.ArgumentParser.add_argument` prepending options - with unique plugin name prefix. - - """ - # other From 8c970a890da5a56f075c641a7497645c2eb9c3c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2016 18:28:56 -0800 Subject: [PATCH 186/208] Revert "Resume using a fully-constructed config namespace" This reverts commit 3603f482e5fe81cb5948699a77a2abaa16a8839d. --- letsencrypt/cli.py | 4 ++-- letsencrypt/configuration.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index aa69df3c6..6655c33dd 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -751,8 +751,8 @@ def _set_by_cli(var): plugins = plugins_disco.PluginsRegistry.find_all() # reconstructed_args == sys.argv[1:], or whatever was passed to main() reconstructed_args = _parser.args + [_parser.verb] - default_args = prepare_and_parse_args(plugins, reconstructed_args, detect_defaults=True) - detector = _set_by_cli.detector = configuration.NamespaceConfig(default_args, fake=True) + detector = _set_by_cli.detector = prepare_and_parse_args( + plugins, reconstructed_args, detect_defaults=True) # propagate plugin requests: eg --standalone modifies config.authenticator auth, inst = cli_plugin_requests(detector) detector.namespace.authenticator = auth if auth else "" diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 979d5e985..2bbf1b019 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -34,7 +34,7 @@ class NamespaceConfig(object): """ zope.interface.implements(interfaces.IConfig) - def __init__(self, namespace, fake=False): + def __init__(self, namespace): self.namespace = namespace self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) @@ -42,8 +42,7 @@ class NamespaceConfig(object): self.namespace.logs_dir = os.path.abspath(self.namespace.logs_dir) # Check command line parameters sanity, and error out in case of problem. - if not fake: - check_config_sanity(self) + check_config_sanity(self) def __getattr__(self, name): return getattr(self.namespace, name) From 96d89cf44d4eebab6c3e7e785e440e3abe3875dc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2016 18:32:59 -0800 Subject: [PATCH 187/208] Disable too-many-locals for now --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 040ffa618..8cb4c0879 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -680,7 +680,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals def obtain_cert(config, plugins, lineage=None): """Implements "certonly": authenticate & obtain cert, but do not install it.""" - + # pylint: disable=too-many-locals try: # installers are used in auth mode to determine domain names installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") From 7aa2f4b5f6c56ae494a97b7fb48906c8190c291d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Feb 2016 18:34:44 -0800 Subject: [PATCH 188/208] Remove stay .namespace --- letsencrypt/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 6655c33dd..6f0b45888 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -755,9 +755,9 @@ def _set_by_cli(var): plugins, reconstructed_args, detect_defaults=True) # propagate plugin requests: eg --standalone modifies config.authenticator auth, inst = cli_plugin_requests(detector) - detector.namespace.authenticator = auth if auth else "" - detector.namespace.installer = inst if inst else "" - logger.debug("Default Detector is %r", detector.namespace) + detector.authenticator = auth if auth else "" + detector.installer = inst if inst else "" + logger.debug("Default Detector is %r", detector) try: # Is detector.var something that isn't false? From e808091a97c0b9f4ed3a2bad224434e2cf3877de Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 10:58:07 -0800 Subject: [PATCH 189/208] Better command-line docs for renew. --- letsencrypt/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index bbfb6b88f..e91478ad4 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1368,6 +1368,11 @@ def prepare_and_parse_args(plugins, args): help="Run without ever asking for user input. This may require " "additional command line flags; the client will try to explain " "which ones are required if it finds one missing") + helpful.add( + None, "--dry-run", action="store_true", dest="dry_run", + help="Perform a test run of the client, obtaining test (invalid) certs" + " but not saving them to disk. This can currently only be used" + " with the 'certonly' subcommand.") helpful.add( None, "--register-unsafely-without-email", action="store_true", help="Specifying this flag enables registering an account with no " @@ -1490,6 +1495,16 @@ def prepare_and_parse_args(plugins, args): help="Require that all configuration files are owned by the current " "user; only needed if your config is somewhere unsafe like /tmp/") + helpful.add_group( + "renew", description="The 'renew' subcommand will attempt to renew all " + "certificates (or more precisely, certificate lineages) you have previously " + "obtained, and print a summary of the results. " + "By default, 'renew' will reuse the options " + "used to create obtain or most recently successfully renew each certificate lineage. " + "You can try it with `--dry-run` first. " + "For more fine-grained control, you can renew individual lineages with" + "the `certonly` subcommand.") + helpful.add_deprecated_argument("--agree-dev-preview", 0) _create_subparsers(helpful) @@ -1586,10 +1601,6 @@ def _paths_parser(helpful): add("testing", "--test-cert", "--staging", action='store_true', dest='staging', help='Use the staging server to obtain test (invalid) certs; equivalent' ' to --server ' + constants.STAGING_URI) - add("testing", "--dry-run", action="store_true", dest="dry_run", - help="Perform a test run of the client, obtaining test (invalid) certs" - " but not saving them to disk. This can currently only be used" - " with the 'certonly' subcommand.") def _plugins_parsing(helpful, plugins): From 78cde8b7d9a3c04967413eb9ad7743bc4996cca5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 11:39:03 -0800 Subject: [PATCH 190/208] Update language to clarify that only expiring certs are renewed --- letsencrypt/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e91478ad4..cdd6e5c45 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1496,14 +1496,14 @@ def prepare_and_parse_args(plugins, args): "user; only needed if your config is somewhere unsafe like /tmp/") helpful.add_group( - "renew", description="The 'renew' subcommand will attempt to renew all " - "certificates (or more precisely, certificate lineages) you have previously " - "obtained, and print a summary of the results. " - "By default, 'renew' will reuse the options " - "used to create obtain or most recently successfully renew each certificate lineage. " - "You can try it with `--dry-run` first. " - "For more fine-grained control, you can renew individual lineages with" - "the `certonly` subcommand.") + "renew", description="The 'renew' subcommand will attempt to renew all" + " certificates (or more precisely, certificate lineages) you have" + " previously obtained if they are close to expiry, and print a" + " summary of the results. By default, 'renew' will reuse the options" + " used to create obtain or most recently successfully renew each" + " certificate lineage. You can try it with `--dry-run` first. For" + " more fine-grained control, you can renew individual lineages with" + " the `certonly` subcommand.") helpful.add_deprecated_argument("--agree-dev-preview", 0) From 9077ae76bbb873786bf88a0945b0f31316fdc824 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 12:53:36 -0800 Subject: [PATCH 191/208] Offline sigs are actually made with sha256 --- tools/offline-sigrequest.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh index ca349f629..07163dbdb 100755 --- a/tools/offline-sigrequest.sh +++ b/tools/offline-sigrequest.sh @@ -9,9 +9,9 @@ fi function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do - cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.5)"; \ + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ echo -n '(SayText "'; \ - sha1sum | cut -c1-40 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + sha1sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ echo '")' ) | festival done @@ -23,8 +23,8 @@ function offlinesign { # $1 <-- INPFILE ; $2 <---SIGFILE echo HASH FOR SIGNING: SIGFILEBALL="$2.lzma.base64" #echo "(place the resulting raw binary signature in $SIGFILEBALL)" - sha1sum $1 - echo metahash for confirmation only $(sha1sum $1 |cut -d' ' -f1 | tr -d '\n' | sha1sum | cut -c1-6) ... + sha256sum $1 + echo metahash for confirmation only $(sha256sum $1 |cut -d' ' -f1 | tr -d '\n' | sha256sum | cut -c1-6) ... echo sayhash $1 $SIGFILEBALL } From f35896b30583d5e662cc2447306265f279ba1484 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Feb 2016 13:00:14 -0800 Subject: [PATCH 192/208] Stopped breaking things --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 29f7f7fcb..8f34a6e81 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -766,7 +766,7 @@ def _set_by_cli(var): try: # Is detector.var something that isn't false? - change_detected = detector.__getattr__(var) + change_detected = getattr(detector, var) except AttributeError: logger.warning("Missing default analysis for %r", var) return False From 4b49419c07738ff3f218322049ef9e1d2952ad9b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 14:27:14 -0800 Subject: [PATCH 193/208] Allow renew to handle --webroot-path on the commandline - and generally clarify how renew handles webroot options --- letsencrypt/cli.py | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 8f34a6e81..7aecb550c 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -834,10 +834,15 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. - plugin_prefixes = [renewalparams["authenticator"]] + if renewalparams["authenticator"] == "webroot": + _restore_webroot_config(config, renewalparams) + plugin_prefixes = [] + else: + plugin_prefixes = [renewalparams["authenticator"]] + if renewalparams.get("installer", None) is not None: plugin_prefixes.append(renewalparams["installer"]) - for plugin_prefix in set(plugin_prefixes): + for plugin_prefix in set(plugin_prefixes) - set("webroot"): # webroot is special for config_item, config_value in renewalparams.iteritems(): if config_item.startswith(plugin_prefix + "_") and not _set_by_cli(config_item): # Avoid confusion when, for example, "csr = None" (avoid @@ -855,6 +860,22 @@ def _restore_plugin_configs(config, renewalparams): else: setattr(config.namespace, config_item, str(config_value)) +def _restore_webroot_config(config, renewalparams) + """ + webroot_map is, uniquely, a dict, and the general-purpose configuration + restoring logic is not able to correctly parse it from the serialized + form. + """ + if "webroot_map" in renewalparams: + # if the user does anything that would create a new webroot map on the + # CLI, don't use the old one + if not (_set_by_cli("webroot_map") or _set_by_cli("webroot_path")): + wrm = renewalparams["webroot_map"] + setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) + elif "webroot_path" in renewalparams: + logger.info("Ancient renewal conf file without webroot-map, restoring webroot-path") + setattr(config.namespace, "webroot_path", renewalparams["webroot_path"]) + def _reconstitute(config, full_path): """Try to instantiate a RenewableCert, updating config with relevant items. @@ -901,24 +922,15 @@ def _reconstitute(config, full_path): logger.debug("Traceback was:\n%s", traceback.format_exc()) return None - # webroot_map is, uniquely, a dict, and the general-purpose - # configuration restoring logic is not able to correctly parse it - # from the serialized form. - if "webroot_map" in renewalparams and not _set_by_cli("webroot_map"): - setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) - try: - domains = [le_util.enforce_domain_sanity(x) for x in - renewal_candidate.names()] + for d in renewal_candidate.names(): + _process_domain(config, d) except errors.ConfigurationError as error: logger.warning("Renewal configuration file %s references a cert " "that contains an invalid domain name. The problem " "was: %s. Skipping.", full_path, error) return None - if not _set_by_cli("domains"): - config.namespace.domains = domains - return renewal_candidate def _renewal_conf_files(config): @@ -1738,7 +1750,7 @@ def _plugins_parsing(helpful, plugins): # they are parsed in conjunction with --domains, they live here for # legibility. helpful.add_plugin_ags must be called first to add the # "webroot" topic - helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor, + helpful.add("webroot", "-w", "--webroot-path", default=[], action=WebrootPathProcessor, help="public_html / webroot path. This can be specified multiple times to " "handle different domains; each domain will have the webroot path that" " preceded it. For instance: `-w /var/www/example -d example.com -d " @@ -1765,8 +1777,7 @@ class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstrin Keep a record of --webroot-path / -w flags during processing, so that we know which apply to which -d flags """ - if args.webroot_path is None: # first -w flag encountered - args.webroot_path = [] + if not args.webroot_path: # first -w flag encountered # if any --domain flags preceded the first --webroot-path flag, # apply that webroot path to those; subsequent entries in # args.webroot_map are filled in by cli.DomainFlagProcessor From 7c7a07ceb2ccac11d55942bc9325ea82c1542108 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 14:30:35 -0800 Subject: [PATCH 194/208] rm redundancy --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 7aecb550c..ba5ee4c73 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -842,7 +842,7 @@ def _restore_plugin_configs(config, renewalparams): if renewalparams.get("installer", None) is not None: plugin_prefixes.append(renewalparams["installer"]) - for plugin_prefix in set(plugin_prefixes) - set("webroot"): # webroot is special + for plugin_prefix in set(plugin_prefixes): for config_item, config_value in renewalparams.iteritems(): if config_item.startswith(plugin_prefix + "_") and not _set_by_cli(config_item): # Avoid confusion when, for example, "csr = None" (avoid From 86bddfa9b4e8cbefad956cdbe1b1510332af17cd Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 10 Feb 2016 14:50:10 -0800 Subject: [PATCH 195/208] Fix syntax error (missing colon in function definition) --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index ba5ee4c73..c31f3b7c3 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -860,7 +860,7 @@ def _restore_plugin_configs(config, renewalparams): else: setattr(config.namespace, config_item, str(config_value)) -def _restore_webroot_config(config, renewalparams) +def _restore_webroot_config(config, renewalparams): """ webroot_map is, uniquely, a dict, and the general-purpose configuration restoring logic is not able to correctly parse it from the serialized From 6abcfddfe7d757d49f1c456a7e8a6237598e39c9 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 10 Feb 2016 15:05:18 -0800 Subject: [PATCH 196/208] Remove unused variable "wrm" --- letsencrypt/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index c31f3b7c3..4170f31cc 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -870,7 +870,6 @@ def _restore_webroot_config(config, renewalparams): # if the user does anything that would create a new webroot map on the # CLI, don't use the old one if not (_set_by_cli("webroot_map") or _set_by_cli("webroot_path")): - wrm = renewalparams["webroot_map"] setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) elif "webroot_path" in renewalparams: logger.info("Ancient renewal conf file without webroot-map, restoring webroot-path") From c328ec62008b61c27ff8bd16dd365784bdc2079a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 10 Feb 2016 16:35:00 -0800 Subject: [PATCH 197/208] Include centos-options-ssl-apache.conf in letsencrypt-apache --- letsencrypt-apache/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt-apache/MANIFEST.in b/letsencrypt-apache/MANIFEST.in index 933cc10ac..bdb67199f 100644 --- a/letsencrypt-apache/MANIFEST.in +++ b/letsencrypt-apache/MANIFEST.in @@ -2,5 +2,6 @@ include LICENSE.txt include README.rst recursive-include docs * recursive-include letsencrypt_apache/tests/testdata * +include letsencrypt_apache/centos-options-ssl-apache.conf include letsencrypt_apache/options-ssl-apache.conf recursive-include letsencrypt_apache/augeas_lens *.aug From ea31db75b7baa1e16e34b24ac8b8d748af29a11c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 16:35:14 -0800 Subject: [PATCH 198/208] Misc release script fixes --- tools/offline-sigrequest.sh | 2 +- tools/release.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh index 07163dbdb..7706796ef 100755 --- a/tools/offline-sigrequest.sh +++ b/tools/offline-sigrequest.sh @@ -11,7 +11,7 @@ function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ echo -n '(SayText "'; \ - sha1sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ echo '")' ) | festival done diff --git a/tools/release.sh b/tools/release.sh index 83b57657f..1472f856b 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -171,6 +171,7 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" done +gid add letsencrypt-auto-source git diff --cached git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" git tag --local-user "$RELEASE_GPG_KEY" \ From 24fa435f464adf6fe526b00039ff99bcf98b1e61 Mon Sep 17 00:00:00 2001 From: Minn Soe Date: Thu, 11 Feb 2016 00:38:24 +0000 Subject: [PATCH 199/208] Fix broken reference to script in old bootstrap directory --- tools/venv.sh | 2 +- tools/venv3.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/venv.sh b/tools/venv.sh index 11ab417dd..5a09efb0b 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -3,7 +3,7 @@ export VENV_ARGS="--python python2" -./bootstrap/dev/_venv_common.sh \ +./tools/_venv_common.sh \ -e acme[testing] \ -e .[dev,docs,testing] \ -e letsencrypt-apache \ diff --git a/tools/venv3.sh b/tools/venv3.sh index ccffffb83..158605f72 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -4,5 +4,5 @@ export VENV_NAME="${VENV_NAME:-venv3}" export VENV_ARGS="--python python3" -./bootstrap/dev/_venv_common.sh \ +./tools/_venv_common.sh \ -e acme[testing] \ From bf674489d5cc1a49000fc2ff8fc22fb4e98b1828 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Wed, 10 Feb 2016 18:10:10 -0800 Subject: [PATCH 200/208] make false falsy --- letsencrypt/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 4170f31cc..15205ee29 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -845,12 +845,12 @@ def _restore_plugin_configs(config, renewalparams): for plugin_prefix in set(plugin_prefixes): for config_item, config_value in renewalparams.iteritems(): if config_item.startswith(plugin_prefix + "_") and not _set_by_cli(config_item): - # Avoid confusion when, for example, "csr = None" (avoid - # trying to read the file called "None") - # Should we omit the item entirely rather than setting - # its value to None? - if config_value == "None": - setattr(config.namespace, config_item, None) + # Values None, True, and False need to be treated specially, + # As they don't get parsed correctly based on type + if config_value in ("None", "True", "False"): + # bool("False") == True + # pylint: disable=eval-used + setattr(config.namespace, config_item, eval(config_value)) continue for action in _parser.parser._actions: # pylint: disable=protected-access if action.type is not None and action.dest == config_item: From 4b86cabe5bb336b8926c0f764fb6433987f2cc8c Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 18:33:08 -0800 Subject: [PATCH 201/208] Fix git typo --- tools/release.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/release.sh b/tools/release.sh index 1472f856b..6ec83053f 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -171,11 +171,10 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" done -gid add letsencrypt-auto-source +git add letsencrypt-auto-source git diff --cached git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" \ - --sign --message "Release $version" "$tag" +git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" cd .. echo Now in $PWD From 74063851e3330b3a52ce592d1319e0e1f0795728 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 18:48:40 -0800 Subject: [PATCH 202/208] Release 0.4.0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 20 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/letsencrypt-auto-requirements.txt | 18 ++++++++-------- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index b5bec3476..9f228f4ed 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0.dev0' +version = '0.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index a6553d890..922cd0e8e 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0.dev0' +version = '0.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 24b62e342..9218bdc52 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.4.0.dev0" +LE_AUTO_VERSION="0.4.0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -638,17 +638,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: QMIkIvGF3mcJhGLAKRX7n5EVIPjOrfLtklN6ePjbJes -# sha256: fNFWiij6VxfG5o7u3oNbtrYKQ4q9vhzOLATfxNlozvQ -acme==0.3.0 +# sha256: ilvjjTWOS86xchl0WBZ0YOAw_0rmqdnjNsxb1hq2RD8 +# sha256: T37KMj0TnsuvHIzCCmoww2fpfpOBTj7cd4NAqucXcpw +acme==0.4.0 -# sha256: qdnzpoRf_44QXKoktNoAKs2RBAxUta2Sr6GS0t_tAKo -# sha256: ELWJaHNvBZIqVPJYkla8yXLtXIuamqAf6f_VAFv16Uk -letsencrypt==0.3.0 +# sha256: 33BQiANlNLGqGpirTfdCEElTF9YbpaKiYpTbK4zeGD8 +# sha256: lwsV1OdEzzlMeb08C_PRxaCXZ2vOk_1AI2755rZHmPM +letsencrypt==0.4.0 -# sha256: EypLpEw3-Tr8unw4aSFsHXgRiU8ZYLrJKOJohP2tC9M -# sha256: HYvP13GzA-DDJYwlfOoaraJO0zuYO48TCSAyTUAGCqA -letsencrypt-apache==0.3.0 +# sha256: D3YDaVFjLsMSEfjI5B5D5tn5FeWUtNHYXCObw3ih2tg +# sha256: VTgvsePYGRmI4IOSAnxoYFHd8KciD73bxIuIHtbVFd8 +letsencrypt-apache==0.4.0 # sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 # sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 4bb5f97ea686dd282f99591ef1d8416dd002942d..532a482073932f4be88c1e25642d18ad947e7e64 100644 GIT binary patch literal 256 zcmV+b0ssC6h;!6XyQogJex_KK=DFx0Q?~h#$ZiK8LqF z9UK0?`*Aq5PynjWNy*-8JZ$G>+S9o<8P@27c@y3`uBda8X`#O+CjMrKVzMiqiCsyS zbqYMkAp~3&FJG3hply|GI7?14!p?ySpSW8X9EZ1FWtJRi4)+#lw>8^eI!3 G_s+-+c7oaf literal 256 zcmV+b0ssEr`(`t5&`>Jt>x>2a9mQS9O6fDHCDL_8lB$tS0lnsR{$ToQQaX9Rhs?!? zX%XvgMxhK&1EB+aICOQ`&uqrPI6(_);_;FS#$BTKNzITO>|85R;}$Z6Pp{SA92rI= zvvxGgNXBR}fB(IGMM=B{V{!z6R{|+^0bk>ltYeHHi|`ZE>5vLZ>vHm!!hF=F#iJxo z0S`3=R_LJTAg+Y6PoDi1aaPX67Osp_7_RB6^A#jZ>bB)#GwK^?2cp8L>3XHP!UlI| zY)FPzE6XOJSbgU_0rnDP8Sw9TYg1Y?Q5$Bb+pxBcip}z$e>44+VYR5nu7$TZnAP{R GaMfnHo`A#v diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index c83396de2..574e567c3 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -201,17 +201,17 @@ zope.event==4.1.0 # sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I zope.interface==4.1.3 -# sha256: QMIkIvGF3mcJhGLAKRX7n5EVIPjOrfLtklN6ePjbJes -# sha256: fNFWiij6VxfG5o7u3oNbtrYKQ4q9vhzOLATfxNlozvQ -acme==0.3.0 +# sha256: ilvjjTWOS86xchl0WBZ0YOAw_0rmqdnjNsxb1hq2RD8 +# sha256: T37KMj0TnsuvHIzCCmoww2fpfpOBTj7cd4NAqucXcpw +acme==0.4.0 -# sha256: qdnzpoRf_44QXKoktNoAKs2RBAxUta2Sr6GS0t_tAKo -# sha256: ELWJaHNvBZIqVPJYkla8yXLtXIuamqAf6f_VAFv16Uk -letsencrypt==0.3.0 +# sha256: 33BQiANlNLGqGpirTfdCEElTF9YbpaKiYpTbK4zeGD8 +# sha256: lwsV1OdEzzlMeb08C_PRxaCXZ2vOk_1AI2755rZHmPM +letsencrypt==0.4.0 -# sha256: EypLpEw3-Tr8unw4aSFsHXgRiU8ZYLrJKOJohP2tC9M -# sha256: HYvP13GzA-DDJYwlfOoaraJO0zuYO48TCSAyTUAGCqA -letsencrypt-apache==0.3.0 +# sha256: D3YDaVFjLsMSEfjI5B5D5tn5FeWUtNHYXCObw3ih2tg +# sha256: VTgvsePYGRmI4IOSAnxoYFHd8KciD73bxIuIHtbVFd8 +letsencrypt-apache==0.4.0 # sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 # sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index b7f448e83..4af0cffcc 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0.dev0' +version = '0.4.0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index c1ff85185..8aab19a57 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0.dev0' +version = '0.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 1dd7d7eba..5e937310e 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.4.0.dev0' +__version__ = '0.4.0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 000f86c31..deaf9c9b5 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0.dev0' +version = '0.4.0' install_requires = [ 'setuptools', # pkg_resources From 563c1150447217273d4f8b24c22d7556bfe7c408 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 18:49:27 -0800 Subject: [PATCH 203/208] Bump version to 0.5.0 --- acme/setup.py | 2 +- letsencrypt-apache/setup.py | 2 +- letsencrypt-compatibility-test/setup.py | 2 +- letsencrypt-nginx/setup.py | 2 +- letsencrypt/__init__.py | 2 +- letshelp-letsencrypt/setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 9f228f4ed..8b7b040e5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index 922cd0e8e..a8e010f0e 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 4af0cffcc..67262ba72 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0' +version = '0.5.0.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index 8aab19a57..656d6e04f 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 5e937310e..0dbeb1567 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.4.0' +__version__ = '0.5.0.dev0' diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index deaf9c9b5..fff8dcfc3 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.4.0' +version = '0.5.0.dev0' install_requires = [ 'setuptools', # pkg_resources From 1f31cf1a3096cc4f42041f29e1c390831768905e Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 19:09:05 -0800 Subject: [PATCH 204/208] Quick test farm fix --- tests/letstest/multitester.py | 6 +++--- tools/venv.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 378670071..e27385002 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -241,21 +241,21 @@ def local_git_clone(repo_url): "clones master of repo_url" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s'% repo_url) + local('git clone %s letsencrypt'% repo_url) local('tar czf le.tar.gz letsencrypt') def local_git_branch(repo_url, branch_name): "clones branch of repo_url" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s --branch %s --single-branch'%(repo_url, branch_name)) + local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name)) local('tar czf le.tar.gz letsencrypt') def local_git_PR(repo_url, PRnumstr, merge_master=True): "clones specified pull request from repo_url and optionally merges into master" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s'% repo_url) + local('git clone %s letsencrypt'% repo_url) local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) local('cd letsencrypt && git co lePRtest') if merge_master: diff --git a/tools/venv.sh b/tools/venv.sh index 11ab417dd..5a09efb0b 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -3,7 +3,7 @@ export VENV_ARGS="--python python2" -./bootstrap/dev/_venv_common.sh \ +./tools/_venv_common.sh \ -e acme[testing] \ -e .[dev,docs,testing] \ -e letsencrypt-apache \ From 29737ab2caf90d23f17cd28a302d151991bc3f78 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 10 Feb 2016 19:24:25 -0800 Subject: [PATCH 205/208] More hacky fixes --- .../scripts/test_letsencrypt_auto_certonly_standalone.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 10d7c3b5e..ecef9814b 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -8,7 +8,8 @@ #private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) cd letsencrypt -./letsencrypt-auto certonly -v --standalone --debug \ +./letsencrypt-auto --os-packages-only +./letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect \ --register-unsafely-without-email \ From 180117facb62c78cf91c6ae169249602cd817b7d Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Wed, 10 Feb 2016 22:13:27 -0800 Subject: [PATCH 206/208] Some preliminary documentation updates to mention renew verb --- docs/using.rst | 60 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 9ee16dffd..c2962ea2e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -71,7 +71,9 @@ Plugin Auth Inst Notes =========== ==== ==== =============================================================== apache_ Y Y Automates obtaining and installing a cert with Apache 2.4 on Debian-based distributions with ``libaugeas0`` 1.0+. -standalone_ Y N Uses a "standalone" webserver to obtain a cert. +standalone_ Y N Uses a "standalone" webserver to obtain a cert. This is useful + on systems with no webserver, or when direct integration with + the local webserver is not supported or not desired. webroot_ Y N Obtains a cert by writing to the webroot directory of an already running webserver. manual_ Y N Helps you obtain a cert by giving you instructions to perform @@ -171,21 +173,59 @@ Renewal days). Make sure you renew the certificates at least once in 3 months. -In order to renew certificates simply call the ``letsencrypt`` (or +The ``letsencrypt`` client now supports a ``renew`` action to check +all installed certificates for impending expiry and attempt to renew +them. The simplest form is simply + +``letsencrypt renew`` + +This will attempt to renew any previously-obtained certificates that +expire in less than 30 days. The same plugin and options that were used +at the time the certificate was originally issued will be used for the +renewal attempt, unless you specify other plugins or options. + +If you're sure that UI doesn't prompt for any details you can add the +command to ``crontab`` (make it less than every 90 days to avoid problems, +say every month); note that the current version provides detailed output +describing either renewal success or failure. + +The ``--force-renew`` flag may be helpful for automating renewal; +it causes the expiration time of the certificate(s) to be ignored when +considering renewal, and attempts to renew each and every installed +certificate regardless of its age. + +Note that options provided to ``letsencrypt renew`` will apply to +*every* certificate for which renewal is attempted; for example, +``letsencrypt renew --rsa-key-size 4096`` would try to replace every +near-expiry certificate with an equivalent certificate using a 4096-bit +RSA public key. If a certificate is successfully renewed using +specified options, those options will be saved and used for future +renewals of that certificate. + + +An alternative form that provides for more fine-grained control over the +renewal process (while renewing specified certificates one at a time), +is ``letsencrypt certonly`` with the complete set of subject domains of +a specific certificate specified via `-d` flags, like + +``letsencrypt certonly -d example.com -d www.example.com`` + +(All of the domains covered by the certificate must be specified in +this case in order to renew and replace the old certificate rather +than obtaining a new one; don't forget any `www.` domains!) The +``certonly`` form attempts to renew one individual certificate. + letsencrypt-auto_) again, and use the same values when prompted. You can automate it slightly by passing necessary flags on the CLI (see `--help -all`), or even further using the :ref:`config-file`. The ``--force-renew`` -flag may be helpful for automating renewal; it causes the expiration time -of the certificate(s) to be ignored when considering renewal. If you're -sure that UI doesn't prompt for any details you can add the command to -``crontab`` (make it less than every 90 days to avoid problems, say -every month). +all`), or even further using the :ref:`config-file`. + Please note that the CA will send notification emails to the address you provide if you do not renew certificates that are about to expire. -Let's Encrypt is working hard on automating the renewal process. Until -the tool is ready, we are sorry for the inconvenience! +Let's Encrypt is working hard on improving the renewal process, and we +apologize for any inconveniences you encounter in integrating these +commands into your individual environment. .. _where-certs: From eb4e8bf59ebc13dbb9b92f4f4983b6906c0b506a Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Thu, 11 Feb 2016 18:42:27 -0500 Subject: [PATCH 207/208] Add a "success" message after installation. Fix #1621. Close #2214. --- letsencrypt-auto-source/letsencrypt-auto | 3 ++- letsencrypt-auto-source/letsencrypt-auto.template | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9218bdc52..30d907690 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -19,7 +19,7 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN=${VENV_PATH}/bin -LE_AUTO_VERSION="0.4.0" +LE_AUTO_VERSION="0.5.0.dev0" # This script takes the same arguments as the main letsencrypt program, but it # additionally responds to --verbose (more output) and --debug (allow support @@ -1632,6 +1632,7 @@ UNLIKELY_EOF echo "$PEEP_OUT" exit 1 fi + echo "Installation succeeded." fi echo "Requesting root privileges to run letsencrypt..." echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index ad8c97a7f..1fa12528f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -209,6 +209,7 @@ UNLIKELY_EOF echo "$PEEP_OUT" exit 1 fi + echo "Installation succeeded." fi echo "Requesting root privileges to run letsencrypt..." echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" From 73b81a35c282e520d7c597877c3443bd6a07485b Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 11 Feb 2016 17:57:46 -0800 Subject: [PATCH 208/208] More documentation edits; prioritize webroot over standalone --- docs/using.rst | 90 +++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index c2962ea2e..22e6b5e5e 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -49,7 +49,7 @@ or for full help, type: ``letsencrypt-auto`` is the recommended method of running the Let's Encrypt client beta releases on systems that don't have a packaged version. Debian, -Arch linux, FreeBSD, and OpenBSD now have native packages, so on those +Arch Linux, FreeBSD, and OpenBSD now have native packages, so on those systems you can just install ``letsencrypt`` (and perhaps ``letsencrypt-apache``). If you'd like to run the latest copy from Git, or run your own locally modified copy of the client, follow the instructions in @@ -93,36 +93,27 @@ This automates both obtaining *and* installing certs on an Apache webserver. To specify this plugin on the command line, simply include ``--apache``. -Standalone ----------- - -To obtain a cert using a "standalone" webserver, you can use the -standalone plugin by including ``certonly`` and ``--standalone`` -on the command line. This plugin needs to bind to port 80 or 443 in -order to perform domain validation, so you may need to stop your -existing webserver. To control which port the plugin uses, include -one of the options shown below on the command line. - - * ``--standalone-supported-challenges http-01`` to use port 80 - * ``--standalone-supported-challenges tls-sni-01`` to use port 443 - Webroot ------- -If you're running a webserver that you don't want to stop to use -standalone, you can use the webroot plugin to obtain a cert by -including ``certonly`` and ``--webroot`` on the command line. In -addition, you'll need to specify ``--webroot-path`` or ``-w`` with the root -directory of the files served by your webserver. For example, -``--webroot-path /var/www/html`` or -``--webroot-path /usr/share/nginx/html`` are two common webroot paths. +If you're running a local webserver for which you have the ability +to modify the content being served, and you'd prefer not to stop the +webserver during the certificate issuance process, you can use the webroot +plugin to obtain a cert by including ``certonly`` and ``--webroot`` on +the command line. In addition, you'll need to specify ``--webroot-path`` +or ``-w`` with the top-level directory ("web root") containing the files +served by your webserver. For example, ``--webroot-path /var/www/html`` +or ``--webroot-path /usr/share/nginx/html`` are two common webroot paths. -If you're getting a certificate for many domains at once, each domain will use -the most recent ``--webroot-path``. So for instance: +If you're getting a certificate for many domains at once, the plugin +needs to know where each domain's files are served from, which could +potentially be a separate directory for each domain. When requested a +certificate for multiple domains, each domain will use the most recently +specified ``--webroot-path``. So, for instance, ``letsencrypt certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/eg -d eg.is -d www.eg.is`` -Would obtain a single certificate for all of those names, using the +would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and ``/var/www/eg`` for the second two. @@ -137,8 +128,28 @@ made to your web server would look like: 66.133.109.36 - - [05/Jan/2016:20:11:24 -0500] "GET /.well-known/acme-challenge/HGr8U1IeTW4kY_Z6UIyaakzOkyQgPr_7ArlLgtZE8SX HTTP/1.1" 200 87 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" Note that to use the webroot plugin, your server must be configured to serve -files from hidden directories. +files from hidden directories. If ``/.well-known`` is treated specially by +your webserver configuration, you might need to modify the configuration +to ensure that files inside ``/.well-known/ache-challenge`` are served by +the webserver. +Standalone +---------- + +To obtain a cert using a "standalone" webserver, you can use the +standalone plugin by including ``certonly`` and ``--standalone`` +on the command line. This plugin needs to bind to port 80 or 443 in +order to perform domain validation, so you may need to stop your +existing webserver. To control which port the plugin uses, include +one of the options shown below on the command line. + + * ``--standalone-supported-challenges http-01`` to use port 80 + * ``--standalone-supported-challenges tls-sni-01`` to use port 443 + +The standalone plugin does not rely on any other server software running +on the machine where you obtain the certificate. It must still be possible +for that machine to accept inbound connections from the Internet on the +specified port using each requested domain name. Manual ------ @@ -148,7 +159,8 @@ other than your target webserver or perform the steps for domain validation yourself, you can use the manual plugin. While hidden from the UI, you can use the plugin to obtain a cert by specifying ``certonly`` and ``--manual`` on the command line. This requires you -to copy and paste commands into another terminal session. +to copy and paste commands into another terminal session, which may +be on a different computer. Nginx ----- @@ -159,7 +171,7 @@ is still experimental, however, and is not installed with letsencrypt-auto_. If installed, you can select this plugin on the command line by including ``--nginx``. -Third party plugins +Third-party plugins ------------------- These plugins are listed at @@ -184,15 +196,19 @@ expire in less than 30 days. The same plugin and options that were used at the time the certificate was originally issued will be used for the renewal attempt, unless you specify other plugins or options. -If you're sure that UI doesn't prompt for any details you can add the -command to ``crontab`` (make it less than every 90 days to avoid problems, -say every month); note that the current version provides detailed output -describing either renewal success or failure. +If you're sure that this command executes successfully without human +intervention, you can add the command to ``crontab`` (since certificates +are only renewed when they're determined to be near expiry, the command +can run on a regular basis, like every week or every day); note that +the current version provides detailed output describing either renewal +success or failure. The ``--force-renew`` flag may be helpful for automating renewal; it causes the expiration time of the certificate(s) to be ignored when considering renewal, and attempts to renew each and every installed -certificate regardless of its age. +certificate regardless of its age. (This form is not appropriate to run +daily because each certificate will be renewed every day, which will +quickly run into the certificate authority rate limit.) Note that options provided to ``letsencrypt renew`` will apply to *every* certificate for which renewal is attempted; for example, @@ -212,12 +228,10 @@ a specific certificate specified via `-d` flags, like (All of the domains covered by the certificate must be specified in this case in order to renew and replace the old certificate rather -than obtaining a new one; don't forget any `www.` domains!) The -``certonly`` form attempts to renew one individual certificate. - -letsencrypt-auto_) again, and use the same values when prompted. You can -automate it slightly by passing necessary flags on the CLI (see `--help -all`), or even further using the :ref:`config-file`. +than obtaining a new one; don't forget any `www.` domains! Specifying +a subset of the domains creates a new, separate certificate containing +only those domains, rather than replacing the original certificate.) +The ``certonly`` form attempts to renew one individual certificate. Please note that the CA will send notification emails to the address