From 2f7eb22891ba261f09e1995cb8fd555980e7fa1c Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Thu, 12 Jan 2017 13:01:29 +0100 Subject: [PATCH 001/128] Dockerfile refactored to use Alpine. --- Dockerfile | 88 +++++++++++++++--------------------------------------- 1 file changed, 24 insertions(+), 64 deletions(-) diff --git a/Dockerfile b/Dockerfile index d42b632d4..157578cd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,70 +1,30 @@ -# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297 -# it is more likely developers will already have ubuntu:trusty rather -# than e.g. debian:jessie and image size differences are negligible -FROM ubuntu:trusty +FROM python:2-alpine MAINTAINER Jakub Warmuz MAINTAINER William Budington -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) +EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt - WORKDIR /opt/certbot - -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - -ENV DEBIAN_FRONTEND=noninteractive - -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - - -COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ - -# all above files are necessary for setup.py and venv setup, however, -# package source code directory has to be copied separately to a -# subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ - - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv - -# PATH is set now so pipstrap upgrades the correct (v)env -ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. - ENTRYPOINT [ "certbot" ] +COPY . src + +RUN apk add --no-cache --virtual .certbot-deps \ + dialog \ + augeas-libs \ + libffi \ + libssl1.0 \ + wget \ + ca-certificates \ + binutils +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + linux-headers \ + openssl-dev \ + musl-dev \ + libffi-dev \ + && pip install --no-cache-dir \ + --editable /opt/certbot/src/acme \ + --editable /opt/certbot/src \ + --editable /opt/certbot/src/certbot-apache \ + --editable /opt/certbot/src/certbot-nginx \ + && apk del .build-deps From 6a8113fa87ff8f55aa16395b9208ac0afe820a34 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 16 Feb 2017 10:24:48 -0800 Subject: [PATCH 002/128] Remove reference to #certbot on OFTC The #letsencrypt channel on Freenode is much more active, and is the defacto place for questions about Certbot. Users posting questions on #certbot on OFTC are not getting prompt answers. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f986703ac..ab12562df 100644 --- a/README.rst +++ b/README.rst @@ -88,7 +88,7 @@ Main Website: https://certbot.eff.org Let's Encrypt Website: https://letsencrypt.org -IRC Channel: #letsencrypt on `Freenode`_ or #certbot on `OFTC`_ +IRC Channel: #letsencrypt on `Freenode`_ Community: https://community.letsencrypt.org From a92ca8e97c38a8adbdf5088af058ec50e55b2602 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 22 Feb 2017 18:48:01 -0800 Subject: [PATCH 003/128] Add default timeout to ClientNetwork. (#4217) In https://community.letsencrypt.org/t/letsencrypt-cli-hangs-on-certificate-request/27211, a community member pointed out that Certbot seems to hang when there are routing problems. --- acme/acme/client.py | 1 + acme/acme/client_test.py | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index d01555c75..168574d58 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -607,6 +607,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes kwargs['verify'] = self.verify_ssl kwargs.setdefault('headers', {}) kwargs['headers'].setdefault('User-Agent', self.user_agent) + kwargs.setdefault('timeout', 45) # timeout after 45 seconds response = self.session.request(method, url, *args, **kwargs) # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 0f4506203..b7bd0740c 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -532,7 +532,7 @@ class ClientNetworkTest(unittest.TestCase): 'HEAD', 'http://example.com/', 'foo', bar='baz')) self.net.session.request.assert_called_once_with( 'HEAD', 'http://example.com/', 'foo', - headers=mock.ANY, verify=mock.ANY, bar='baz') + headers=mock.ANY, verify=mock.ANY, timeout=mock.ANY, bar='baz') @mock.patch('acme.client.logger') def test_send_request_get_der(self, mock_logger): @@ -542,7 +542,8 @@ class ClientNetworkTest(unittest.TestCase): headers={"Content-Type": "application/pkix-cert"}, content=b"hi") # pylint: disable=protected-access - self.net._send_request('HEAD', 'http://example.com/', 'foo', bar='baz') + self.net._send_request('HEAD', 'http://example.com/', 'foo', + timeout=mock.ANY, bar='baz') mock_logger.debug.assert_called_once_with( 'Received response:\nHTTP %d\n%s\n\n%s', 200, 'Content-Type: application/pkix-cert', b'aGk=') @@ -555,7 +556,7 @@ class ClientNetworkTest(unittest.TestCase): 'POST', 'http://example.com/', 'foo', data='qux', bar='baz')) self.net.session.request.assert_called_once_with( 'POST', 'http://example.com/', 'foo', - headers=mock.ANY, verify=mock.ANY, data='qux', bar='baz') + headers=mock.ANY, verify=mock.ANY, timeout=mock.ANY, data='qux', bar='baz') def test_send_request_verify_ssl(self): # pylint: disable=protected-access @@ -568,7 +569,8 @@ class ClientNetworkTest(unittest.TestCase): self.response, self.net._send_request('GET', 'http://example.com/')) self.net.session.request.assert_called_once_with( - 'GET', 'http://example.com/', verify=verify, headers=mock.ANY) + 'GET', 'http://example.com/', verify=verify, + timeout=mock.ANY, headers=mock.ANY) def test_send_request_user_agent(self): self.net.session = mock.MagicMock() @@ -577,13 +579,23 @@ class ClientNetworkTest(unittest.TestCase): headers={'bar': 'baz'}) self.net.session.request.assert_called_once_with( 'GET', 'http://example.com/', verify=mock.ANY, + timeout=mock.ANY, headers={'User-Agent': 'acme-python-test', 'bar': 'baz'}) self.net._send_request('GET', 'http://example.com/', headers={'User-Agent': 'foo2'}) self.net.session.request.assert_called_with( 'GET', 'http://example.com/', - verify=mock.ANY, headers={'User-Agent': 'foo2'}) + verify=mock.ANY, timeout=mock.ANY, headers={'User-Agent': 'foo2'}) + + def test_send_request_timeout(self): + self.net.session = mock.MagicMock() + # pylint: disable=protected-access + self.net._send_request('GET', 'http://example.com/', + headers={'bar': 'baz'}) + self.net.session.request.assert_called_once_with( + mock.ANY, mock.ANY, verify=mock.ANY, headers=mock.ANY, + timeout=45) def test_del(self): sess = mock.MagicMock() From b1a4280519b1a2d271e0e770a87a2a37915a381a Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 23 Feb 2017 03:50:56 +0100 Subject: [PATCH 004/128] Show error details for the nginx config parser (#4221) Nginx parser errors now include helpful details on where parsing has actually failed. Related: https://github.com/certbot/certbot/issues/3798 --- certbot-nginx/certbot_nginx/parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 1a2c85c2c..c586aa459 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -205,8 +205,8 @@ class NginxParser(object): trees.append(parsed) except IOError: logger.warning("Could not open file: %s", item) - except pyparsing.ParseException: - logger.debug("Could not parse file: %s", item) + except pyparsing.ParseException as err: + logger.debug("Could not parse file: %s due to %s", item, err) return trees def _parse_ssl_options(self, ssl_options): @@ -216,8 +216,8 @@ class NginxParser(object): return nginxparser.load(_file).spaced except IOError: logger.warn("Missing NGINX TLS options file: %s", ssl_options) - except pyparsing.ParseBaseException: - logger.debug("Could not parse file: %s", ssl_options) + except pyparsing.ParseBaseException as err: + logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] def _set_locations(self, ssl_options): From 52ce335ff095c8bfde7ee624c57fafef673c19af Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 23 Feb 2017 18:31:23 -0800 Subject: [PATCH 005/128] lineage_for_certname should return None if there is no existing renewal file (#4243) * lineage_for_certname should return None if there is no existing renewal file * add unit test * add regression test to integration test * revent boulder-start to boulder-fetch --- certbot/cert_manager.py | 5 ++++- certbot/tests/cert_manager_test.py | 10 ++++++++++ tests/boulder-integration.sh | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 71a5fe6fa..35d539a16 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -100,7 +100,10 @@ def lineage_for_certname(cli_config, certname): configs_dir = cli_config.renewal_configs_dir # Verify the directory is there util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - renewal_file = storage.renewal_file_for_certname(cli_config, certname) + try: + renewal_file = storage.renewal_file_for_certname(cli_config, certname) + except errors.CertStorageError: + return None try: return storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 1fa68d195..473970870 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -290,6 +290,16 @@ class LineageForCertnameTest(BaseCertManagerTest): None) self.assertTrue(mock_make_or_verify_dir.called) + @mock.patch('certbot.util.make_or_verify_dir') + @mock.patch('certbot.storage.renewal_file_for_certname') + def test_no_renewal_file(self, mock_renewal_conf_file, + mock_make_or_verify_dir): + mock_renewal_conf_file.side_effect = errors.CertStorageError() + from certbot import cert_manager + self.assertEqual(cert_manager.lineage_for_certname(self.cli_config, "example.com"), + None) + self.assertTrue(mock_make_or_verify_dir.called) + class DomainsForCertnameTest(BaseCertManagerTest): """Tests for certbot.cert_manager.domains_for_certname""" diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2ddd3c04b..ca6f48e60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -100,6 +100,8 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 \ common certonly -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni \ --manual-auth-hook ./tests/manual-dns-auth.sh +common certonly --cert-name newname -d newname.le.wtf + export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf ./examples/generate-csr.sh le3.wtf From 28cbd6e7d385ce0bd4c28e49e422d6eaf51f9cdb Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 24 Feb 2017 06:28:36 +0200 Subject: [PATCH 006/128] Fix for case sensitivity when looking for vhosts (#4193) --- certbot-apache/certbot_apache/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index a956341eb..5639dae83 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -580,7 +580,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ("/files%s//*[label()=~regexp('%s')]" % (vhost_path, parser.case_i("VirtualHost")))) paths = [path for path in paths if - os.path.basename(path) == "VirtualHost"] + os.path.basename(path.lower()) == "virtualhost"] for path in paths: new_vhost = self._create_vhost(path) if not new_vhost: From e02d641490181fb7c7722ec85b52052fe07f3534 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 24 Feb 2017 21:40:03 +0200 Subject: [PATCH 007/128] Docker changes for easier testing (#4249) These changes allow developers to run tests directly from the host machine using Docker, and to enable ipdb inside the container. docker-compose.yml is upgraded to version 2 format. This means that you need docker-engine version >= 1.10.0 instead of previous requirement of version >= 1.9.1. The reason for this is to be able to use custom Dockerfile (Dockerfile-dev in this case) in build context. ipdb has been added to dev dependencies to be able to be able to debug the code without installing it on every docker run. This is also what we recommend for debugging in the developer documentation, so there really is no reason not to install it with the dev dependencies. setuptools is being upgraded to a newer version to be able to run coverage tests. This was using the older version of setuptools for some reason, and without the upgrade, coverage tests would fail horribly. Upgrading remedies the situation. Few examples: Run unit tests for certbot-apache `docker-compose run --rm --service-ports development bash -c 'cd src;nosetests -v certbot-apache'` Run coverage tests `docker-compose run --rm --service-ports development bash -c 'cd src;./tox.cover.sh'` Run linter `docker-compose run --rm --service-ports development bash -c 'cd src;tox -e lint'` --- Dockerfile-dev | 3 ++- docker-compose.yml | 26 +++++++++++++++----------- setup.py | 1 + 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 098fae523..dbb45f75e 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -58,7 +58,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ -e /opt/certbot/src/certbot-nginx \ -e /opt/certbot/src/letshelp-certbot \ -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] + -e /opt/certbot/src[dev,docs] && \ + /opt/certbot/venv/bin/pip install -U setuptools # install in editable mode (-e) to save space: it's not possible to # "rm -rf /opt/certbot/src" (it's stays in the underlaying image); diff --git a/docker-compose.yml b/docker-compose.yml index 8b2a8e9a3..00d3d4c72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,18 @@ -production: - build: . - ports: - - "443:443" +version: '2' +services: + production: + build: . + ports: + - "443:443" # For development, mount git root to /opt/certbot/src in order to # make the dev workflow more vagrant-like. -development: - build: . - ports: - - "443:443" - volumes: - - .:/opt/certbot/src - - /opt/certbot/venv + development: + build: + context: . + dockerfile: Dockerfile-dev + ports: + - "443:443" + volumes: + - .:/opt/certbot/src + - /opt/certbot/venv diff --git a/setup.py b/setup.py index 529459cf0..0c47b973f 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', 'coverage', + 'ipdb', 'nose', 'pylint==1.4.2', # upstream #248 'tox', From d066f8b38b432000d4217c1cd39bc292938796ce Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Fri, 24 Feb 2017 13:08:25 -0800 Subject: [PATCH 008/128] created an issue template (#4201) * created an issue template * bmw changes --- ISSUE_TEMPLATE.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..e4e56f93d --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,16 @@ +## My operating system is (include version): + + +## My web server is (include version): + + +## How did you install Certbot: + + +## What command did you run and what output did it produce? + + +## Can you provide a Certbot error log showing the issue? +###### It is stored by default in `/var/log/letsencrypt` - feel free to redact domain names, e-mail and IP addresses as you see fit + + From 7d02e129f9e0fd9abed850621d78e91dd7684ff8 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 25 Feb 2017 10:21:21 +0800 Subject: [PATCH 009/128] Fix certbot-apache tests on Python 3 (#4172) --- certbot-apache/certbot_apache/obj.py | 10 ++++++++++ certbot-apache/certbot_apache/parser.py | 14 +++++++++---- .../certbot_apache/tests/configurator_test.py | 20 ++++++++++--------- .../certbot_apache/tests/display_ops_test.py | 2 +- .../certbot_apache/tests/tls_sni_01_test.py | 2 +- certbot-apache/certbot_apache/tls_sni_01.py | 2 +- tox.ini | 8 ++++++++ 7 files changed, 42 insertions(+), 16 deletions(-) diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index b29b0e0ee..30cb24844 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -25,6 +25,11 @@ class Addr(common.Addr): def __repr__(self): return "certbot_apache.obj.Addr(" + repr(self.tup) + ")" + def __hash__(self): + # Python 3 requires explicit overridden for __hash__ if __eq__ or + # __cmp__ is overridden. See https://bugs.python.org/issue2235 + return super(Addr, self).__hash__() + def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" # pylint: disable=protected-access @@ -174,6 +179,11 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def __ne__(self, other): return not self.__eq__(other) + def __hash__(self): + return hash((self.filep, self.path, + tuple(self.addrs), tuple(self.get_names()), + self.ssl, self.enabled, self.modmacro)) + def conflicts(self, addrs): """See if vhost conflicts with any of the addrs. diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 6bb6ff170..275a01e7f 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -1,10 +1,12 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" import fnmatch -import itertools import logging import os import re import subprocess +import sys + +import six from certbot import errors @@ -87,7 +89,7 @@ class ApacheParser(object): while len(self.modules) != prev_size: prev_size = len(self.modules) - for match_name, match_filename in itertools.izip( + for match_name, match_filename in six.moves.zip( iterator, iterator): self.modules.add(self.get_arg(match_name)) self.modules.add( @@ -460,8 +462,12 @@ class ApacheParser(object): :rtype: str """ - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] + if sys.version_info < (3, 6): + # This strips off final /Z(?ms) + return fnmatch.translate(clean_fn_match)[:-7] + else: # pragma: no cover + # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z + return fnmatch.translate(clean_fn_match)[4:-3] def _parse_file(self, filepath): """Parse file with Augeas diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 01361c8f0..937694267 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -6,6 +6,8 @@ import socket import unittest import mock +# six is used in mock.patch() +import six # pylint: disable=unused-import from acme import challenges @@ -517,12 +519,12 @@ class MultipleVhostsTest(util.ApacheTest): # Test self.config.prepare_server_https("8080", temp=True) self.assertEqual(mock_add_dir.call_count, 3) - self.assertEqual(mock_add_dir.call_args_list[0][0][2], - ["1.2.3.4:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[1][0][2], - ["[::1]:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[2][0][2], - ["1.1.1.1:8080", "https"]) + call_args_list = [mock_add_dir.call_args_list[i][0][2] for i in range(3)] + self.assertEqual( + sorted(call_args_list), + sorted([["1.2.3.4:8080", "https"], + ["[::1]:8080", "https"], + ["1.1.1.1:8080", "https"]])) # mock_get.side_effect = ["1.2.3.4:80", "[::1]:80"] # mock_find.return_value = ["test1", "test2", "test3"] @@ -662,7 +664,7 @@ class MultipleVhostsTest(util.ApacheTest): # This calls open self.config.reverter.register_file_creation = mock.Mock() mock_open.side_effect = IOError - with mock.patch("__builtin__.open", mock_open): + with mock.patch("six.moves.builtins.open", mock_open): self.assertRaises( errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) @@ -1208,13 +1210,13 @@ class MultipleVhostsTest(util.ApacheTest): achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"), + token=b"jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"), "pending"), domain="encryption-example.demo", account_key=account_key) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), + token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), "pending"), domain="certbot.demo", account_key=account_key) diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index ec6eee3f2..f8b75022e 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -38,7 +38,7 @@ class SelectVhostTest(unittest.TestCase): try: self._call(self.vhosts) except errors.MissingCommandlineFlag as e: - self.assertTrue("vhost ambiguity" in e.message) + self.assertTrue("vhost ambiguity" in str(e)) @certbot_util.patch_get_utility() def test_more_info_cancel(self, mock_util): diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 5e369e3db..62464d5d0 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -105,7 +105,7 @@ class TlsSniPerformTest(util.ApacheTest): for achall in self.achalls: self.sni.add_chall(achall) z_domain = achall.response(self.auth_key).z_domain - z_domains.append(set([z_domain])) + z_domains.append(set([z_domain.decode('ascii')])) self.sni._mod_config() # pylint: disable=protected-access self.sni.configurator.save() diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index d9e294119..65a66d2fd 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -184,7 +184,7 @@ class ApacheTlsSni01(common.TLSSNI01): # https://docs.python.org/2.7/reference/lexical_analysis.html return self.VHOST_TEMPLATE.format( vhost=ips, - server_name=achall.response(achall.account_key).z_domain, + server_name=achall.response(achall.account_key).z_domain.decode('ascii'), ssl_options_conf_path=self.configurator.mod_ssl_conf, cert_path=self.get_cert_path(achall), key_path=self.get_key_path(achall), diff --git a/tox.ini b/tox.ini index e6317e665..232010d40 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,8 @@ commands = nosetests -v acme --processes=-1 pip install -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 + pip install -e certbot-apache + nosetests -v certbot_apache --processes=-1 --process-timeout=80 [testenv:py34] commands = @@ -53,6 +55,8 @@ commands = nosetests -v acme --processes=-1 pip install -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 + pip install -e certbot-apache + nosetests -v certbot_apache --processes=-1 --process-timeout=80 [testenv:py35] commands = @@ -60,6 +64,8 @@ commands = nosetests -v acme --processes=-1 pip install -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 + pip install -e certbot-apache + nosetests -v certbot_apache --processes=-1 --process-timeout=80 [testenv:py36] commands = @@ -67,6 +73,8 @@ commands = nosetests -v acme --processes=-1 pip install -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 + pip install -e certbot-apache + nosetests -v certbot_apache --processes=-1 --process-timeout=80 [testenv:py27_install] basepython = python2.7 From bb35126a2cf76d18187f9c7eec746cd1917c6c37 Mon Sep 17 00:00:00 2001 From: Osiris Inferi Date: Mon, 27 Feb 2017 20:39:42 +0100 Subject: [PATCH 010/128] Deprecated standalone-preferred-challenges --- docs/using.rst | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 628043ff9..c162e2fde 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -151,14 +151,17 @@ order to perform domain validation, so you may need to stop your existing webserver. To control which port the plugin uses, include one of the options shown below on the command line. - * ``--standalone-supported-challenges http-01`` to use port 80 - * ``--standalone-supported-challenges tls-sni-01`` to use port 443 + * ``--preferred-challenges http`` to use port 80 + * ``--preferred-challenges tls-sni`` to use port 443 The standalone plugin does not rely on any other server software running on the machine where you obtain the certificate. It must still be possible for that machine to accept inbound connections from the Internet on the specified port using each requested domain name. +.. note:: The ``--standalone-supported-challenges`` option has been + deprecated since ``certbot`` version 0.9.0. + Manual ------ @@ -170,6 +173,23 @@ the UI, you can use the plugin to obtain a cert by specifying to copy and paste commands into another terminal session, which may be on a different computer. +The manual plugin can use either the ``http`` or the ``dns`` challenge. You +can use the ``--preferred-challenges`` option to chose the challenge of your +preference. +The ``http`` challenge will ask you to place a file with a specific name and +specific content in the ``/.well-known/acme-challenge/`` directory directly +in the top-level directory (“web root”) containing the files served by your +webserver. In essence it's the same as the webroot_ plugin, but not automated. +When using the ``dns`` plugin, ``certbot`` will ask you to place a TXT DNS +record with specific contents under the domain name consisting of the hostname +for which you want a certificate issued, prepended by ``_acme-challenge``. + +For example, for the domain ``example.com``, a zone file entry would look like: + +:: + + _acme-challenge.example.com. 300 IN TXT "gfj9Xq...Rg85nM" + Additionally you can specify scripts to prepare for validation and perform the authentication procedure and/or clean up after it by using the ``--manual-auth-hook`` and ``--manual-cleanup-hook`` flags. This is described in From e5909d379c5e0144481c7c848ccf1c13b71c2a73 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 27 Feb 2017 13:35:29 -0800 Subject: [PATCH 011/128] Don't crash on listen unix: (#4259) Fixes #4225. * don't crash on listen unix: * correctly merge #4221 --- certbot-nginx/certbot_nginx/parser.py | 7 ++++--- certbot-nginx/certbot_nginx/tests/parser_test.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index c586aa459..eddc7b9b0 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -586,9 +586,10 @@ def _parse_server_raw(server): continue if directive[0] == 'listen': addr = obj.Addr.fromstring(directive[1]) - parsed_server['addrs'].add(addr) - if addr.ssl: - parsed_server['ssl'] = True + if addr: + parsed_server['addrs'].add(addr) + if addr.ssl: + parsed_server['ssl'] = True elif directive[0] == 'server_name': parsed_server['names'].update( _get_servernames(directive[1])) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 921cc3c5a..6a3f2f1de 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -323,6 +323,12 @@ class NginxParserTest(util.NginxTest): ]) self.assertTrue(server['ssl']) + def test_parse_server_raw_unix(self): + server = parser._parse_server_raw([ #pylint: disable=protected-access + ['listen', 'unix:/var/run/nginx.sock'] + ]) + self.assertEqual(len(server['addrs']), 0) + def test_parse_server_global_ssl_applied(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) server = nparser.parse_server([ From 80055ec770ee0e896a52b19c6420c4f2cf6a59fb Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 27 Feb 2017 15:15:19 -0800 Subject: [PATCH 012/128] Cleanup issue template (#4256) --- ISSUE_TEMPLATE.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index e4e56f93d..e648a21b4 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,16 +1,14 @@ ## My operating system is (include version): -## My web server is (include version): +## I installed Certbot with (certbot-auto, OS package manager, pip, etc): -## How did you install Certbot: +## I ran this command and it produced this output: -## What command did you run and what output did it produce? - - -## Can you provide a Certbot error log showing the issue? -###### It is stored by default in `/var/log/letsencrypt` - feel free to redact domain names, e-mail and IP addresses as you see fit +## Certbot's behavior differed from what I expected because: +## Here is a Certbot log showing the issue (if available): +###### Logs are stored in `/var/log/letsencrypt` by default. Feel free to redact domains, e-mail and IP addresses as you see fit. From 402ad8b35311460babb7195095b10d02a0b14e48 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 27 Feb 2017 17:17:08 -0800 Subject: [PATCH 013/128] bump requests requirement to >=2.10 (#4248) --- acme/setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/acme/setup.py b/acme/setup.py index d1e91e5ec..9b43278af 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -15,7 +15,11 @@ install_requires = [ 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + # requests>=2.10 is required to fix + # https://github.com/shazow/urllib3/issues/556. This requirement can be + # relaxed to 'requests[security]>=2.4.1', however, less useful errors + # will be raised for some network/SSL errors. + 'requests[security]>=2.10', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', From 44a6ec29c58e6bcceb05543ea8c5503fc8c5b772 Mon Sep 17 00:00:00 2001 From: Damien Tournoud Date: Tue, 28 Feb 2017 03:13:06 +0100 Subject: [PATCH 014/128] Fix direct usages of the root logger (#4236) Some code uses `logging.debug` and `logging.info` instead of the file-specific logger in `logger.debug` and `logger.info`. --- acme/acme/challenges.py | 4 ++-- acme/acme/client.py | 6 +++--- acme/acme/client_test.py | 2 +- certbot-nginx/certbot_nginx/configurator.py | 2 +- certbot/ocsp.py | 6 +++--- certbot/tests/ocsp_test.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 83b9b9edd..3b1e90166 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -425,7 +425,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): # TODO: domain is not necessary if host is provided if "host" not in kwargs: host = socket.gethostbyname(domain) - logging.debug('%s resolved to %s', domain, host) + logger.debug('%s resolved to %s', domain, host) kwargs["host"] = host kwargs.setdefault("port", self.PORT) @@ -445,7 +445,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ # pylint: disable=protected-access sans = crypto_util._pyopenssl_cert_or_req_san(cert) - logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans) return self.z_domain.decode() in sans def simple_verify(self, chall, domain, account_public_key, diff --git a/acme/acme/client.py b/acme/acme/client.py index 168574d58..23eabe4b9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -600,10 +600,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """ if method == "POST": - logging.debug('Sending POST request to %s:\n%s', + logger.debug('Sending POST request to %s:\n%s', url, kwargs['data']) else: - logging.debug('Sending %s request to %s.', method, url) + logger.debug('Sending %s request to %s.', method, url) kwargs['verify'] = self.verify_ssl kwargs.setdefault('headers', {}) kwargs['headers'].setdefault('User-Agent', self.user_agent) @@ -651,7 +651,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _get_nonce(self, url): if not self._nonces: - logging.debug('Requesting fresh nonce') + logger.debug('Requesting fresh nonce') self._add_nonce(self.head(url)) return self._nonces.pop() diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index b7bd0740c..3621a0824 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -544,7 +544,7 @@ class ClientNetworkTest(unittest.TestCase): # pylint: disable=protected-access self.net._send_request('HEAD', 'http://example.com/', 'foo', timeout=mock.ANY, bar='baz') - mock_logger.debug.assert_called_once_with( + mock_logger.debug.assert_called_with( 'Received response:\nHTTP %d\n%s\n\n%s', 200, 'Content-Type: application/pkix-cert', b'aGk=') diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 6d51ca641..7348def2f 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -630,7 +630,7 @@ class NginxConfigurator(common.Plugin): stderr=subprocess.PIPE) text = proc.communicate()[1] # nginx prints output to stderr except (OSError, ValueError) as error: - logging.debug(error, exc_info=True) + logger.debug(error, exc_info=True) raise errors.PluginError( "Unable to run %s -V" % self.conf('ctl')) diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 8921dbb88..d34110f88 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -16,7 +16,7 @@ class RevocationChecker(object): self.broken = False if not util.exe_exists("openssl"): - logging.info("openssl not installed, can't check revocation") + logger.info("openssl not installed, can't check revocation") self.broken = True return @@ -61,7 +61,7 @@ class RevocationChecker(object): logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) try: - output, err = util.run_script(cmd, log=logging.debug) + output, err = util.run_script(cmd, log=logger.debug) except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False @@ -80,7 +80,7 @@ class RevocationChecker(object): try: url, _err = util.run_script( ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logging.debug) + log=logger.debug) except errors.SubprocessError: logger.info("Cannot extract OCSP URI from %s", cert_path) return None, None diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 549e83ca8..91dd6f8d6 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -28,7 +28,7 @@ class OCSPTest(unittest.TestCase): def tearDown(self): pass - @mock.patch('certbot.ocsp.logging.info') + @mock.patch('certbot.ocsp.logger.info') @mock.patch('certbot.ocsp.Popen') @mock.patch('certbot.util.exe_exists') def test_init(self, mock_exists, mock_popen, mock_log): From 0d8a4b4ebdf12d6664550cd52a40c9f70e864c03 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 28 Feb 2017 15:17:07 -0800 Subject: [PATCH 015/128] Make mod-check more flexible (#4268) * fixes #4166 * Run mod-check from anywhere * pass TRAVIS_BRANCH through in tox --- tests/modification-check.sh | 61 ++++++++++++++++--------------------- tox.ini | 4 ++- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/tests/modification-check.sh b/tests/modification-check.sh index 5168f5ab5..6f412ba47 100755 --- a/tests/modification-check.sh +++ b/tests/modification-check.sh @@ -1,46 +1,39 @@ #!/bin/bash temp_dir=`mktemp -d` +trap "rm -rf $temp_dir" EXIT -# Script should be run from Certbot's root directory - -SCRIPT_PATH=`dirname $0` -SCRIPT_PATH=`readlink -f $SCRIPT_PATH` +# cd to repo root +cd $(dirname $(dirname $(readlink -f $0))) FLAG=false -# Compare root letsencrypt-auto and certbot-auto with published versions - -cp letsencrypt-auto ${temp_dir}/letsencrypt-to-be-checked -cp certbot-auto ${temp_dir}/certbot-to-be-checked - -cp letsencrypt-auto-source/pieces/fetch.py ${temp_dir}/fetch.py -cd ${temp_dir} - -LATEST_VERSION=`python fetch.py --latest-version` -python fetch.py --le-auto-script v${LATEST_VERSION} - -cmp -s letsencrypt-auto letsencrypt-to-be-checked - -if [ $? != 0 ]; then - echo "Root letsencrypt-auto has changed." - FLAG=true +if ! cmp -s certbot-auto letsencrypt-auto; then + echo "Root certbot-auto and letsencrypt-auto differ." + FLAG=true else - echo "Root letsencrypt-auto is unchanged." + cp certbot-auto "$temp_dir/local-auto" + cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" + cd $temp_dir + + # Compare file against current version in the target branch + BRANCH=${TRAVIS_BRANCH:-master} + URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" + curl -sS $URL > certbot-auto + if cmp -s certbot-auto local-auto; then + echo "Root *-auto were unchanged." + else + # Compare file against the latest released version + python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" + if cmp -s letsencrypt-auto local-auto; then + echo "Root *-auto were updated to the latest version." + else + echo "Root *-auto have unexpected changes." + FLAG=true + fi + fi + cd ~- fi -cmp -s letsencrypt-auto certbot-to-be-checked - -if [ $? != 0 ]; then - echo "Root certbot-auto has changed." - FLAG=true -else - echo "Root certbot-auto is unchanged." -fi - -# Cleanup -rm ${temp_dir}/* -cd ${SCRIPT_PATH}/../ - # Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea diff --git a/tox.ini b/tox.ini index 232010d40..ea1423415 100644 --- a/tox.ini +++ b/tox.ini @@ -151,7 +151,9 @@ commands = docker run --rm -t -i lea whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS_BRANCH [testenv:le_auto_wheezy] # At the moment, this tests under Python 2.7 only, as only that version is From 11ec1eb91148f6420e10d2296b8a507c90337792 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 2 Mar 2017 10:31:15 -0800 Subject: [PATCH 016/128] Revert "Remove Link rel=next for authzs and new-certs." (#4277) --- acme/acme/client.py | 40 ++++++++++++++++++++++-------- acme/acme/client_test.py | 40 ++++++++++++++++++++++++------ acme/acme/jose/json_util.py | 2 +- acme/acme/messages.py | 6 ++++- acme/acme/messages_test.py | 6 ++++- acme/examples/example_client.py | 3 ++- certbot/auth_handler.py | 3 ++- certbot/tests/account_test.py | 2 +- certbot/tests/acme_util.py | 1 + certbot/tests/auth_handler_test.py | 3 ++- certbot/tests/display/ops_test.py | 4 +-- 11 files changed, 84 insertions(+), 26 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 23eabe4b9..ddcba7635 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -71,13 +71,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes self.directory = directory @classmethod - def _regr_from_response(cls, response, uri=None, terms_of_service=None): + def _regr_from_response(cls, response, uri=None, new_authzr_uri=None, + terms_of_service=None): if 'terms-of-service' in response.links: terms_of_service = response.links['terms-of-service']['url'] + if 'next' in response.links: + new_authzr_uri = response.links['next']['url'] + + if new_authzr_uri is None: + raise errors.ClientError('"next" link missing') return messages.RegistrationResource( body=messages.Registration.from_json(response.json()), uri=response.headers.get('Location', uri), + new_authzr_uri=new_authzr_uri, terms_of_service=terms_of_service) def register(self, new_reg=None): @@ -110,7 +117,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # (c.f. acme-spec #94) return self._regr_from_response( - response, uri=regr.uri, + response, uri=regr.uri, new_authzr_uri=regr.new_authzr_uri, terms_of_service=regr.terms_of_service) def update_registration(self, regr, update=None): @@ -167,30 +174,43 @@ class Client(object): # pylint: disable=too-many-instance-attributes return self.update_registration( regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier, + uri=None, new_cert_uri=None): + # pylint: disable=no-self-use + if new_cert_uri is None: + try: + new_cert_uri = response.links['next']['url'] + except KeyError: + raise errors.ClientError('"next" link missing') + authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), - uri=response.headers.get('Location', uri)) + uri=response.headers.get('Location', uri), + new_cert_uri=new_cert_uri) if authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier): + def request_challenges(self, identifier, new_authzr_uri=None): """Request challenges. :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: ``new-authorization`` URI. If omitted, + will default to value found in ``directory``. :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) + response = self.net.post(self.directory.new_authz + if new_authzr_uri is None else new_authzr_uri, + new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) - def request_domain_challenges(self, domain): + def request_domain_challenges(self, domain, new_authzr_uri=None): """Request challenges for domain names. This is simply a convenience function that wraps around @@ -205,7 +225,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain)) + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) def answer_challenge(self, challb, response): """Answer challenge. @@ -280,7 +300,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ response = self.net.get(authzr.uri) updated_authzr = self._authzr_from_response( - response, authzr.body.identifier, authzr.uri) + response, authzr.body.identifier, authzr.uri, authzr.new_cert_uri) # TODO: check and raise UnexpectedUpdate return updated_authzr, response @@ -304,7 +324,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument response = self.net.post( - self.directory.new_cert, + authzrs[0].new_cert_uri, # TODO: acme-spec #90 req, content_type=content_type, headers={'Accept': content_type}) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 3621a0824..b3db21ac9 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,8 +40,6 @@ class ClientTest(unittest.TestCase): 'https://www.letsencrypt-demo.org/acme/revoke-cert', messages.NewAuthorization: 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', }) from acme.client import Client @@ -58,6 +56,7 @@ class ClientTest(unittest.TestCase): self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', + new_authzr_uri='https://www.letsencrypt-demo.org/acme/new-reg', terms_of_service='https://www.letsencrypt-demo.org/tos') # Authorization @@ -73,7 +72,8 @@ class ClientTest(unittest.TestCase): typ=messages.IDENTIFIER_FQDN, value='example.com'), challenges=(challb,), combinations=None) self.authzr = messages.AuthorizationResource( - body=self.authz, uri=authzr_uri) + body=self.authz, uri=authzr_uri, + new_cert_uri='https://www.letsencrypt-demo.org/acme/new-cert') # Request issuance self.certr = messages.CertificateResource( @@ -98,12 +98,18 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.response.headers['Location'] = self.regr.uri self.response.links.update({ + 'next': {'url': self.regr.new_authzr_uri}, 'terms-of-service': {'url': self.regr.terms_of_service}, }) self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments + def test_register_missing_next(self): + self.response.status_code = http_client.CREATED + self.assertRaises( + errors.ClientError, self.client.register, self.new_reg) + def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -136,6 +142,13 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) + def test_query_registration_updates_new_authzr_uri(self): + self.response.json.return_value = self.regr.body.to_json() + self.response.links = {'next': {'url': 'UPDATED'}} + self.assertEqual( + 'UPDATED', + self.client.query_registration(self.regr).new_authzr_uri) + def test_agree_to_tos(self): self.client.update_registration = mock.Mock() self.client.agree_to_tos(self.regr) @@ -146,6 +159,9 @@ class ClientTest(unittest.TestCase): self.response.status_code = http_client.CREATED self.response.headers['Location'] = self.authzr.uri self.response.json.return_value = self.authz.to_json() + self.response.links = { + 'next': {'url': self.authzr.new_cert_uri}, + } def test_request_challenges(self): self._prepare_response_for_request_challenges() @@ -156,9 +172,8 @@ class ClientTest(unittest.TestCase): def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() - self.client.request_challenges(self.identifier) - self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + self.client.request_challenges(self.identifier, 'URI') + self.net.post.assert_called_once_with('URI', mock.ANY) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -166,7 +181,12 @@ class ClientTest(unittest.TestCase): identifier=self.identifier.update(value='foo')).to_json() self.assertRaises( errors.UnexpectedUpdate, self.client.request_challenges, - self.identifier) + self.identifier, self.authzr.uri) + + def test_request_challenges_missing_next(self): + self.response.status_code = http_client.CREATED + self.assertRaises(errors.ClientError, self.client.request_challenges, + self.identifier) def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() @@ -174,6 +194,12 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier), self.client.request_domain_challenges('example.com')) + def test_request_domain_challenges_custom_uri(self): + self.client.request_challenges = mock.MagicMock() + self.assertEqual( + self.client.request_challenges(self.identifier, 'URI'), + self.client.request_domain_challenges('example.com', 'URI')) + def test_answer_challenge(self): self.response.links['up'] = {'url': self.challr.authzr_uri} self.response.json.return_value = self.challr.body.to_json() diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index 4baadda5e..d474f4aac 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -267,7 +267,7 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): if missing: raise errors.DeserializationError( - 'The following fields are required: {0}'.format( + 'The following field are required: {0}'.format( ','.join(missing))) @classmethod diff --git a/acme/acme/messages.py b/acme/acme/messages.py index c3df4998c..54cd25c94 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -191,7 +191,7 @@ class Directory(jose.JSONDeSerializable): try: return self[name.replace('_', '-')] except KeyError as error: - raise AttributeError(str(error) + ': ' + name) + raise AttributeError(str(error)) def __getitem__(self, name): try: @@ -315,10 +315,12 @@ class RegistrationResource(ResourceWithURI): """Registration Resource. :ivar acme.messages.Registration body: + :ivar unicode new_authzr_uri: URI found in the 'next' ``Link`` header :ivar unicode terms_of_service: URL for the CA TOS. """ body = jose.Field('body', decoder=Registration.from_json) + new_authzr_uri = jose.Field('new_authzr_uri') terms_of_service = jose.Field('terms_of_service', omitempty=True) @@ -423,9 +425,11 @@ class AuthorizationResource(ResourceWithURI): """Authorization Resource. :ivar acme.messages.Authorization body: + :ivar unicode new_cert_uri: URI found in the 'next' ``Link`` header """ body = jose.Field('body', decoder=Authorization.from_json) + new_cert_uri = jose.Field('new_cert_uri') @Directory.register diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index e84c3e992..b3454f25b 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -225,12 +225,14 @@ class RegistrationResourceTest(unittest.TestCase): from acme.messages import RegistrationResource self.regr = RegistrationResource( body=mock.sentinel.body, uri=mock.sentinel.uri, + new_authzr_uri=mock.sentinel.new_authzr_uri, terms_of_service=mock.sentinel.terms_of_service) def test_to_partial_json(self): self.assertEqual(self.regr.to_json(), { 'body': mock.sentinel.body, 'uri': mock.sentinel.uri, + 'new_authzr_uri': mock.sentinel.new_authzr_uri, 'terms_of_service': mock.sentinel.terms_of_service, }) @@ -344,7 +346,9 @@ class AuthorizationResourceTest(unittest.TestCase): from acme.messages import AuthorizationResource authzr = AuthorizationResource( uri=mock.sentinel.uri, - body=mock.sentinel.body) + body=mock.sentinel.body, + new_cert_uri=mock.sentinel.new_cert_uri, + ) self.assertTrue(isinstance(authzr, jose.JSONDeSerializable)) diff --git a/acme/examples/example_client.py b/acme/examples/example_client.py index 1386491b1..261b37603 100644 --- a/acme/examples/example_client.py +++ b/acme/examples/example_client.py @@ -32,7 +32,8 @@ acme.agree_to_tos(regr) logging.debug(regr) authzr = acme.request_challenges( - identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN)) + identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN), + new_authzr_uri=regr.new_authzr_uri) logging.debug(authzr) authzr, authzr_response = acme.poll(authzr) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 53346a77c..6e9ab25a7 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -63,7 +63,8 @@ class AuthHandler(object): """ for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + self.authzr[domain] = self.acme.request_domain_challenges( + domain, self.account.regr.new_authzr_uri) self._choose_challenges(domains) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7367717bf..8ed591c98 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -110,7 +110,7 @@ class AccountFileStorageTest(unittest.TestCase): from certbot.account import Account self.acc = Account( regr=messages.RegistrationResource( - uri=None, body=messages.Registration()), + uri=None, new_authzr_uri=None, body=messages.Registration()), key=KEY) def tearDown(self): diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index f0549666a..5e6b190a7 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -96,5 +96,6 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): # pylint: disable=star-args return messages.AuthorizationResource( uri="https://trusted.ca/new-authz-resource", + new_cert_uri="https://trusted.ca/new-cert", body=messages.Authorization(**authz_kwargs) ) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 9d22843db..046eb5ef1 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -309,6 +309,7 @@ class PollChallengesTest(unittest.TestCase): new_authzr = messages.AuthorizationResource( uri=authzr.uri, + new_cert_uri=authzr.new_cert_uri, body=messages.Authorization( identifier=authzr.body.identifier, challenges=new_challbs, @@ -436,7 +437,7 @@ def gen_auth_resp(chall_list): for chall in chall_list] -def gen_dom_authzr(domain, challs, combos=True): +def gen_dom_authzr(domain, unused_new_authzr_uri, challs, combos=True): """Generates new authzr for domains.""" return acme_util.gen_authzr( messages.STATUS_PENDING, domain, challs, diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index f2a9b3d07..f6de33a92 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -104,10 +104,10 @@ class ChooseAccountTest(unittest.TestCase): self.key = KEY self.acc1 = account.Account(messages.RegistrationResource( - uri=None, body=messages.Registration.from_data( + uri=None, new_authzr_uri=None, body=messages.Registration.from_data( email="email1@g.com")), self.key) self.acc2 = account.Account(messages.RegistrationResource( - uri=None, body=messages.Registration.from_data( + uri=None, new_authzr_uri=None, body=messages.Registration.from_data( email="email2@g.com", phone="phone")), self.key) @classmethod From 5e6a6f51d3627d2ec297d5e1bdb4c2ddfb113654 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 2 Mar 2017 10:31:37 -0800 Subject: [PATCH 017/128] Fix test_leauto_upgrades.sh (#4278) * fix-test-leauto-upgrades * redirect stderr * redirect stderr part 2 --- .../letstest/scripts/test_leauto_upgrades.sh | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index f7a74821b..b46080eff 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -4,13 +4,6 @@ # are dynamically set at execution cd letsencrypt -#git checkout v0.1.0 use --branch instead -SAVE="$PIP_EXTRA_INDEX_URL" -unset PIP_EXTRA_INDEX_URL -export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" - -#OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" - if ! command -v git ; then if [ "$OS_TYPE" = "ubuntu" ] ; then @@ -22,15 +15,18 @@ if ! command -v git ; then fi fi BRANCH=`git rev-parse --abbrev-ref HEAD` -git checkout -f v0.1.0 -./letsencrypt-auto -v --debug --version -unset PIP_INDEX_URL - -export PIP_EXTRA_INDEX_URL="$SAVE" +# 0.4.1 is the oldest version of letsencrypt-auto that can be used because +# it's the first version that both pins package versions and properly supports +# --no-self-upgrade. +git checkout -f v0.4.1 +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.4.1 ; then + echo initial installation appeared to fail + exit 1 +fi git checkout -f "$BRANCH" EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade | grep $EXPECTED_VERSION ; then +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then echo upgrade appeared to fail exit 1 fi From b040717e4d6841342991067bc722c5cb5b990656 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 2 Mar 2017 10:31:55 -0800 Subject: [PATCH 018/128] Changelog (#4252) * made a changelog * fix date for 0.6.0 * fix brad nits * fix typo --- CHANGELOG.md | 326 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..4cf02e954 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,326 @@ +# 0.11.1 +## 02/01/2017 + +* Resolve a problem where Certbot would crash while parsing command line +arguments in some cases. +* Fix a typo. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Aclosed + +# 0.11.0 +## 02/01/2017 + +* Providing `--quiet` to `certbot-auto` now silences package manager output. +* The UI has been improved in the standalone plugin. When using the +plugin while running Certbot interactively and a required port is bound +by another process, Certbot will give you the option to retry to grab +the port rather than immediately exiting. +* You are now able to deactivate your account with the Let's Encrypt +server using the `unregister` subcommand. +* When revoking a certificate using the `revoke` subcommand, you now +have the option to provide the reason the certificate is being revoked +to Let's Encrypt with `--reason`. +* Removal of the optional `dnspython` dependency in our `acme` package. +Now the library does not support client side verification of the DNS +challenge. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.11.0+is%3Aclosed + +# 0.10.2 +## 01/25/2017 + +* We now save `--preferred-challenges` values for renewal. Previously +these values were discarded causing a different challenge type to be +used when renewing certs in some cases. +* If Certbot receives a request with a `badNonce` error, we +automatically retry the request. Since nonces from Let's Encrypt expire, +this helps people performing the DNS challenge with the `manual` plugin +who may have to wait an extended period of time for their DNS changes to +propagate. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed + +# 0.10.1 +## 01/13/2017 + +* Resolve problems where when asking Certbot to update a certificate at +an existing path to include different domain names, the old names would +continue to be used. +* Fix issues successfully running our unit test suite on some systems. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.1+is%3Aclosed + +# 0.10.0 +## 01/11/2017 + +* The ability to customize and automatically complete DNS and HTTP +domain validation challenges with the manual plugin. The flags +`--manual-auth-hook` and `--manual-cleanup-hook` can now be provided +when using the manual plugin to execute commands provided by the user to +perform and clean up challenges provided by the CA. This is best used in +complicated setups where the DNS challenge must be used or Certbot's +existing plugins cannot be used to perform HTTP challenges. For more +information on how this works, see `certbot --help manual`. +* A `--cert-name` flag for specifying the name to use for the +certificate in Certbot's configuration directory. Using this flag in +combination with `-d/--domains`, a user can easily request a new +certificate with different domains and save it with the name provided by +`--cert-name`. Additionally, `--cert-name` can be used to select a +certificate with the `certonly` and `run` subcommands so a full list of +domains in the certificate does not have to be provided. +* The subcommand `certificates` for listing the certificates managed by +Certbot and their properties. +* A `delete` subcommand for removing certificates managed by Certbot +from the configuration directory. +* Support for requesting internationalized domain names (IDNs). +* Removal of the ncurses interface. This change solves problems people +were having on many systems, reduces the number of Certbot dependencies, +and simplifies our code. Certbot's only interface now is the text +interface which was available by providing `-t/--text` to earlier +versions of Certbot. +* Hooks provided to Certbot are now saved to be reused during renewal. +If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` +flags when obtaining a certificate, the provided commands will +automatically be saved and executed again when renewing the certificate. +A pre-hook and/or post-hook can also be given to the `certbot renew` +command either on the command line or in a [configuration +file](https://certbot.eff.org/docs/using.html#configuration-file) to run +an additional command before/after any certificate is renewed. Hooks +will only be run if a certificate is renewed. +* Recategorized `-h/--help` output to improve documentation and +discoverability. +* Busybox support in certbot-auto. +* Many small bug fixes. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.0is%3Aclosed + +# 0.9.3 +## 10/13/2016 + +* Adopt more conservative behavior about reporting a needed port as +unavailable when using the standalone plugin. +* The Apache plugin uses information about your OS to help determine the +layout of your Apache configuration directory. We added a patch to +ensure this code behaves the same way when testing on different systems +as the tests were failing in some cases. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/27?closed=1 + +# 0.9.2 +## 10/12/2016 + +* Ensuring we properly copy `ssl on;` directives as necessary when +performing domain validation in the Nginx plugin. +* Verifying that our optional dependencies version matches what is +required by Certbot. +* A fix for problems where symlinks were becoming files when they were +packaged, causing errors during testing and OS packaging. +* Stop requiring that all possibly required ports are available when +using the standalone plugin. Only verify the ports are available when +you know they are necessary. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/26?closed=1 + +# 0.9.1 +## 10/06/2016 + +* This version of Certbot simply fixes a bug that was introduced in version +0.9.0 where the command line flag -q/--quiet wasn't respected in some cases. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/25?closed=1 + +# 0.9.0 +## 10/05/2016 + +* An alpha version of the Nginx plugin. This plugin fully automates the +process of obtaining and installing certificates with Nginx. +Additionally, it is able to automatically configure security +enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use +this plugin, you must have the `certbot-nginx` package installed (which +is installed automatically when using `certbot-auto`) and provide +`--nginx` on the command line. This plugin is still in its early stages +so we recommend you use it with some caution and make sure you have a +backup of your Nginx configuration. +* Support for the `DNS` challenge in the `acme` library as well as `DNS` +support in Certbot's `manual` plugin. This allows you to create DNS +records to prove to Let's Encrypt you control the requested the domain +name. To use this feature, include `--manual --preferred-challenges dns` +on the command line. +* Help with enabling Extra Packages for Enterprise Linux (EPEL) on +CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, +the EPEL repository has to be enabled. `certbot-auto` will now prompt +users asking them if they would like the script to enable this for them +automatically. This is done without prompting users when using +`letsencrypt-auto` or if `-n/--non-interactive/--noninteractive` is +included on the command line. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Aclosed + +# 0.8.1 +## 06/14/2016 + +* Preserving a certificate's common name when using `renew` +* Save webroot values for renewal when they are entered interactively +* Problems with an invalid user-agent string on OS X +* Gracefully reporting the Apache plugin isn't usable when Augeas is not installed +* Experimental support for Mageia has been added to `certbot-auto` + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ + +# 0.8.0 +## 06/02/2016 + +* The main new feature in this release is the `register` subcommand which +can be used to register an account with the Let's Encrypt CA. +* Additionally, you can run `certbot register --update-registration` to +change the e-mail address associated with your registration. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ + +# 0.7.0 +## 05/27/2016 + +* `--must-staple` to request certificates from Let's Encrypt with the +OCSP must staple extension +* automatic configuration of OSCP stapling for Apache +* requesting certificates for domains found in the common name of a +custom CSR +* a number of bug fixes + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue + +# 0.6.0 +## 05/12/2016 + +* Renamed the client from `letsencrypt` to `certbot` +* Fixed a small json deserialization error +* Versioned the datetime dependency in setup.py +* Preserve domain order in generated CSRs +* Some minor bug fixes + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is%3Aclosed%20 + +# 0.5.0 +## 04/05/2016 + +* The ability to use the webroot plugin interactively. +* The flags --pre-hook, --post-hook, and --renew-hook which can be used +with the renew subcommand to register shell commands to run in +response to renewal events. Pre-hook commands will be run before +any certs are renewed, post-hook commands will be run after any +certs are renewed, and renew-hook commands will be run after each +cert is renewed. If no certs are due for renewal, no command is run. +* Cleaner renewal configuration files. In /etc/letsencrypt/renewal by +default, these files can be used to control what parameters are used +when renewing a specific certificate. +* A -q/--quiet flag which silences all output except errors. +* An --allow-subset-of-domains flag which can be used with the renew +command to prevent renewal failures for a subset of the requested +domains from causing the client to exit. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue + +# 0.4.2 +## 03/03/2016 + +* Resolves problems encountered when compiling letsencrypt +against the new OpenSSL release. +* A patch fixing problems of using letsencrypt renew with configuration files +from private beta has been added. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 + +# 0.4.1 +## 02/29/2016 + +* Fixes Apache parsing errors with some configurations +* Fixes Werkzeug dependency problems on some Red Hat systems +* Fixes bootstrapping failures when using letsencrypt-auto with --no-self-upgrade +* Fixes problems with parsing renewal config files from private beta + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 + +# 0.4.0 +## 02/10/2016 + +* The new verb/subcommand `renew` can be used to renew your existing +certificates as they approach expiration. Running `letsencrypt renew` +will examine all existing certificate lineages and determine if any are +less than 30 days from expiration. If so, the client will use the +settings provided when you previously obtained the certificate to renew +it. The subcommand finishes by printing a summary of which renewals were +successful, failed, or not yet due. +* A `--dry-run` flag has been added to help with testing configuration +without affecting production rate limits. Currently supported by the +`renew` and `certonly` subcommands, providing `--dry-run` on the command +line will obtain certificates from the staging server without saving the +resulting certificates to disk. +* Major improvements have been added to letsencrypt-auto. This script +has been rewritten to include full support for Python 2.6, the ability +for letsencrypt-auto to update itself, and improvements to the +stability, security, and performance of the script. +* Support for Apache 2.2 has been added to the Apache plugin. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 + +# 0.3.0 +## 01/27/2016 + +* A non-interactive mode which can be enabled by including `-n` or +`--non-interactive` on the command line. This can be used to +guarantee the client will not prompt when run automatically using +cron/systemd. +* Preparation for the new letsencrypt-auto script. Over the past +couple months, we've been working on increasing the reliability and +security of letsencrypt-auto. A number of changes landed in this +release to prepare for the new version of this script. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 + +# 0.2.0 +## 01/14/2016 + +* Apache plugin support for non-Debian based systems. Support has been +added for modern Red Hat based systems such as Fedora 23, Red Hat 7, +and CentOS 7 running Apache 2.4. In theory, this plugin should be +able to be configured to run on any Unix-like OS running Apache 2.4. +* Relaxed PyOpenSSL version requirements. This adds support for systems +with PyOpenSSL versions 0.13 or 0.14. +* Resolves issues with the Apache plugin enabling an HTTP to HTTPS +redirect on some systems. +* Improved error messages from the client. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 + +# 0.1.1 +## 12/15/2015 + +* Fix a confusing UI path that caused some users to repeatedly renew +their certs while experimenting with the client, in some cases +hitting issuance rate limits +* Fixes numerous Apache configuration parser fixes +* Avoids attempting to issue for unqualified domain names like +"localhost" +* Fixes --webroot permission handling for non-root users + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 From 5e671682caac181072141747747a97b9ca2f48cb Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 2 Mar 2017 15:26:24 -0800 Subject: [PATCH 019/128] Candidate 0.12.0 (#4286) * Release 0.12.0 * Bump version to 0.13.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 26 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 14 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/letsencrypt-auto-requirements.txt | 24 ++++++++-------- 12 files changed, 64 insertions(+), 64 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 9b43278af..f169f59a7 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.13.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 87ea1a281..56a48abc6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.13.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index ac721638f..54cc429cf 100755 --- a/certbot-auto +++ b/certbot-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.11.1" +LE_AUTO_VERSION="0.12.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -833,18 +833,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.12.0 \ + --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ + --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf +certbot==0.12.0 \ + --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ + --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 +certbot-apache==0.12.0 \ + --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ + --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 +certbot-nginx==0.12.0 \ + --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ + --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e2d226a72..73d3b704b 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.13.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 6de2dc6bd..24c2564b9 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.13.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 6451eb0d5..0c667378d 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.12.0.dev0' +__version__ = '0.13.0.dev0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4cb408002..9ef9d9e6c 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -86,7 +86,7 @@ optional arguments: statistics about success rates by OS and plugin. If you wish to hide your server OS version from the Let's Encrypt server, set this to "". (default: - CertbotACMEClient/0.11.1 (Ubuntu 16.04.1 LTS) + CertbotACMEClient/0.12.0 (Ubuntu 16.04.2 LTS) Authenticator/XXX Installer/YYY) automation: diff --git a/letsencrypt-auto b/letsencrypt-auto index ac721638f..54cc429cf 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.11.1" +LE_AUTO_VERSION="0.12.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -833,18 +833,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.12.0 \ + --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ + --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf +certbot==0.12.0 \ + --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ + --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 +certbot-apache==0.12.0 \ + --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ + --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 +certbot-nginx==0.12.0 \ + --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ + --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 44e2246b7..417d43387 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 -iQEcBAABAgAGBQJYkqlBAAoJEE0XyZXNl3XyXUAH/RqwKDOpceChSdH/aIk891HX -VRDBQxIjZ6EB1iyebfihyZd4a5zGJ9ocMj1GxThMyLgSKbgkSRtjE/+ymDWsL0Us -Y8w9fw76BAImaJZEvjkrpqD2bSYdijnF479hBa/huZHKcQhb/sqxkNJO9SO1uj8z -bnF0UjJNjgn1hm2yHNMWwyEX7xCN/Vxiq/Zwqi7HdPus99sInJA7+04nwXaUtash -87MHpCjIiHh3axOCOjJbAWzIfsDUKeaBHgeYO+2ldOPWVQ0Amp7ghXjohryBkiux -dqhhAuvBTmNqPrbPAjdJ7Kd74NOGDo3HvAUiuXIckDWqxX2Q34w5pwxelZcIEnI= -=vmVS +iQEcBAABAgAGBQJYuJdQAAoJEE0XyZXNl3Xyw+oH/1AQ90P3397rKB0jP+5MchtR +Nz4ScKL86x9s+o/OzAN76gLhJNj/gOVWoyeK8wVkJ07MpbGyLBiYFsXPZWYUcJ77 +LRj4sGAxJatptHG+PnzIquAf+swynqVu0QdBv8ImKwYrqOlULR+Kr8QZE95Ena51 +JPkbm5o0ipSbByIpraAYabCOHj7SrsFQtMx+tPTd7xaliO8VkguzLQt93QQC7CNj +JIO/yURnfKzutTOe3OPzBzbb6e2yhHcHZcSyv8S0DCIAoB08N9Bs8aAbVwmD89Fq +fwYxLZherXRZ2VtJ2Sf/hUP2ZrEH/mvCkKjzznZokFGJXLvTEc8fC/O6c/q/nLw= +=YiSx -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 81e3862c2..cc248a36a 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.12.0.dev0" +LE_AUTO_VERSION="0.13.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -833,18 +833,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.12.0 \ + --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ + --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf +certbot==0.12.0 \ + --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ + --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 +certbot-apache==0.12.0 \ + --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ + --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 +certbot-nginx==0.12.0 \ + --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ + --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7aea42dd07338009aeb411d16b1ede2e47242c4a..711300dda497b4f0de469e0d5f9fdeffcc0ed791 100644 GIT binary patch literal 256 zcmV+b0ssCPKrmxhSd~Gl5(vW|<6opm^3z*2@_s;^(O#c8A(BZ&bT<6@KKsFX>*P^+ z5OMKIYIwR>5CX?W?*GvZf6nD9_+<;knP+ME`_3x{Odk_e&tozlwSx$&=Gd}j4{T;d zN=0Y**I$U*TkAErI2%6WFOxeoL@LtX^L?L>(^-}P0vSB0ac_x}l1H9mca`Z>jI7HK ze*M3p)+_=!N|Mdxjmt%{u8Za)+;9^WpG(BR|)b z`MuJ4TFdF^IQv0a>mH6LZ6Uwm>Ao3CY`8B~Piue4_sWH{fV;aQFnC4iPVzu|)T9g< GmtWvEc7D?U literal 256 zcmV+b0ssE0LMkrq1<;I$4(Z|boMIIj?&t!1cp>;Yh?HTs)6}s^=SfcWN;bWgd`EXf zEhtU0e_$I^0tEloE_&&fmjX6Hni@Oow|k9sJt&yBk3)F3fvE|SyJe#y@%&-O3e7`~ z73zWb(bivi7Ot+2)kS>^Ui$C!RU< Date: Thu, 2 Mar 2017 15:46:16 -0800 Subject: [PATCH 020/128] update changelog (#4287) --- CHANGELOG.md | 360 ++++++++++++++++++++++++++------------------------- 1 file changed, 185 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cf02e954..cc1ad82ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# 0.12.0 +## 03/02/2017 + +* Allow non-camelcase Apache VirtualHost names +* Allow more log messages to be silenced +* Fix a regression around using `--cert-name` when getting new certificates + +More information about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.12.0 + # 0.11.1 ## 02/01/2017 @@ -31,70 +41,70 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.11.0+is%3Ac # 0.10.2 ## 01/25/2017 -* We now save `--preferred-challenges` values for renewal. Previously -these values were discarded causing a different challenge type to be -used when renewing certs in some cases. -* If Certbot receives a request with a `badNonce` error, we -automatically retry the request. Since nonces from Let's Encrypt expire, -this helps people performing the DNS challenge with the `manual` plugin -who may have to wait an extended period of time for their DNS changes to -propagate. +* We now save `--preferred-challenges` values for renewal. Previously +these values were discarded causing a different challenge type to be +used when renewing certs in some cases. +* If Certbot receives a request with a `badNonce` error, we +automatically retry the request. Since nonces from Let's Encrypt expire, +this helps people performing the DNS challenge with the `manual` plugin +who may have to wait an extended period of time for their DNS changes to +propagate. -More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed # 0.10.1 ## 01/13/2017 -* Resolve problems where when asking Certbot to update a certificate at -an existing path to include different domain names, the old names would -continue to be used. -* Fix issues successfully running our unit test suite on some systems. +* Resolve problems where when asking Certbot to update a certificate at +an existing path to include different domain names, the old names would +continue to be used. +* Fix issues successfully running our unit test suite on some systems. More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.1+is%3Aclosed +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.1+is%3Aclosed # 0.10.0 ## 01/11/2017 -* The ability to customize and automatically complete DNS and HTTP -domain validation challenges with the manual plugin. The flags -`--manual-auth-hook` and `--manual-cleanup-hook` can now be provided -when using the manual plugin to execute commands provided by the user to -perform and clean up challenges provided by the CA. This is best used in -complicated setups where the DNS challenge must be used or Certbot's -existing plugins cannot be used to perform HTTP challenges. For more -information on how this works, see `certbot --help manual`. -* A `--cert-name` flag for specifying the name to use for the -certificate in Certbot's configuration directory. Using this flag in -combination with `-d/--domains`, a user can easily request a new -certificate with different domains and save it with the name provided by -`--cert-name`. Additionally, `--cert-name` can be used to select a -certificate with the `certonly` and `run` subcommands so a full list of -domains in the certificate does not have to be provided. -* The subcommand `certificates` for listing the certificates managed by -Certbot and their properties. -* A `delete` subcommand for removing certificates managed by Certbot -from the configuration directory. -* Support for requesting internationalized domain names (IDNs). -* Removal of the ncurses interface. This change solves problems people -were having on many systems, reduces the number of Certbot dependencies, -and simplifies our code. Certbot's only interface now is the text -interface which was available by providing `-t/--text` to earlier -versions of Certbot. -* Hooks provided to Certbot are now saved to be reused during renewal. -If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` -flags when obtaining a certificate, the provided commands will -automatically be saved and executed again when renewing the certificate. -A pre-hook and/or post-hook can also be given to the `certbot renew` -command either on the command line or in a [configuration -file](https://certbot.eff.org/docs/using.html#configuration-file) to run -an additional command before/after any certificate is renewed. Hooks -will only be run if a certificate is renewed. -* Recategorized `-h/--help` output to improve documentation and -discoverability. -* Busybox support in certbot-auto. -* Many small bug fixes. +* The ability to customize and automatically complete DNS and HTTP +domain validation challenges with the manual plugin. The flags +`--manual-auth-hook` and `--manual-cleanup-hook` can now be provided +when using the manual plugin to execute commands provided by the user to +perform and clean up challenges provided by the CA. This is best used in +complicated setups where the DNS challenge must be used or Certbot's +existing plugins cannot be used to perform HTTP challenges. For more +information on how this works, see `certbot --help manual`. +* A `--cert-name` flag for specifying the name to use for the +certificate in Certbot's configuration directory. Using this flag in +combination with `-d/--domains`, a user can easily request a new +certificate with different domains and save it with the name provided by +`--cert-name`. Additionally, `--cert-name` can be used to select a +certificate with the `certonly` and `run` subcommands so a full list of +domains in the certificate does not have to be provided. +* The subcommand `certificates` for listing the certificates managed by +Certbot and their properties. +* A `delete` subcommand for removing certificates managed by Certbot +from the configuration directory. +* Support for requesting internationalized domain names (IDNs). +* Removal of the ncurses interface. This change solves problems people +were having on many systems, reduces the number of Certbot dependencies, +and simplifies our code. Certbot's only interface now is the text +interface which was available by providing `-t/--text` to earlier +versions of Certbot. +* Hooks provided to Certbot are now saved to be reused during renewal. +If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` +flags when obtaining a certificate, the provided commands will +automatically be saved and executed again when renewing the certificate. +A pre-hook and/or post-hook can also be given to the `certbot renew` +command either on the command line or in a [configuration +file](https://certbot.eff.org/docs/using.html#configuration-file) to run +an additional command before/after any certificate is renewed. Hooks +will only be run if a certificate is renewed. +* Recategorized `-h/--help` output to improve documentation and +discoverability. +* Busybox support in certbot-auto. +* Many small bug fixes. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.0is%3Aclosed @@ -102,31 +112,31 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.0is%3Acl # 0.9.3 ## 10/13/2016 -* Adopt more conservative behavior about reporting a needed port as -unavailable when using the standalone plugin. -* The Apache plugin uses information about your OS to help determine the -layout of your Apache configuration directory. We added a patch to -ensure this code behaves the same way when testing on different systems -as the tests were failing in some cases. +* Adopt more conservative behavior about reporting a needed port as +unavailable when using the standalone plugin. +* The Apache plugin uses information about your OS to help determine the +layout of your Apache configuration directory. We added a patch to +ensure this code behaves the same way when testing on different systems +as the tests were failing in some cases. More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/milestone/27?closed=1 +https://github.com/certbot/certbot/milestone/27?closed=1 # 0.9.2 ## 10/12/2016 -* Ensuring we properly copy `ssl on;` directives as necessary when -performing domain validation in the Nginx plugin. -* Verifying that our optional dependencies version matches what is -required by Certbot. -* A fix for problems where symlinks were becoming files when they were -packaged, causing errors during testing and OS packaging. -* Stop requiring that all possibly required ports are available when -using the standalone plugin. Only verify the ports are available when -you know they are necessary. +* Ensuring we properly copy `ssl on;` directives as necessary when +performing domain validation in the Nginx plugin. +* Verifying that our optional dependencies version matches what is +required by Certbot. +* A fix for problems where symlinks were becoming files when they were +packaged, causing errors during testing and OS packaging. +* Stop requiring that all possibly required ports are available when +using the standalone plugin. Only verify the ports are available when +you know they are necessary. More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/milestone/26?closed=1 +https://github.com/certbot/certbot/milestone/26?closed=1 # 0.9.1 ## 10/06/2016 @@ -135,43 +145,43 @@ https://github.com/certbot/certbot/milestone/26?closed=1 0.9.0 where the command line flag -q/--quiet wasn't respected in some cases. More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/milestone/25?closed=1 +https://github.com/certbot/certbot/milestone/25?closed=1 # 0.9.0 ## 10/05/2016 -* An alpha version of the Nginx plugin. This plugin fully automates the -process of obtaining and installing certificates with Nginx. -Additionally, it is able to automatically configure security -enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use -this plugin, you must have the `certbot-nginx` package installed (which -is installed automatically when using `certbot-auto`) and provide -`--nginx` on the command line. This plugin is still in its early stages -so we recommend you use it with some caution and make sure you have a -backup of your Nginx configuration. -* Support for the `DNS` challenge in the `acme` library as well as `DNS` -support in Certbot's `manual` plugin. This allows you to create DNS -records to prove to Let's Encrypt you control the requested the domain -name. To use this feature, include `--manual --preferred-challenges dns` -on the command line. -* Help with enabling Extra Packages for Enterprise Linux (EPEL) on -CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, -the EPEL repository has to be enabled. `certbot-auto` will now prompt -users asking them if they would like the script to enable this for them -automatically. This is done without prompting users when using -`letsencrypt-auto` or if `-n/--non-interactive/--noninteractive` is -included on the command line. +* An alpha version of the Nginx plugin. This plugin fully automates the +process of obtaining and installing certificates with Nginx. +Additionally, it is able to automatically configure security +enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use +this plugin, you must have the `certbot-nginx` package installed (which +is installed automatically when using `certbot-auto`) and provide +`--nginx` on the command line. This plugin is still in its early stages +so we recommend you use it with some caution and make sure you have a +backup of your Nginx configuration. +* Support for the `DNS` challenge in the `acme` library as well as `DNS` +support in Certbot's `manual` plugin. This allows you to create DNS +records to prove to Let's Encrypt you control the requested the domain +name. To use this feature, include `--manual --preferred-challenges dns` +on the command line. +* Help with enabling Extra Packages for Enterprise Linux (EPEL) on +CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, +the EPEL repository has to be enabled. `certbot-auto` will now prompt +users asking them if they would like the script to enable this for them +automatically. This is done without prompting users when using +`letsencrypt-auto` or if `-n/--non-interactive/--noninteractive` is +included on the command line. More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Aclosed +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Aclosed # 0.8.1 ## 06/14/2016 -* Preserving a certificate's common name when using `renew` -* Save webroot values for renewal when they are entered interactively -* Problems with an invalid user-agent string on OS X -* Gracefully reporting the Apache plugin isn't usable when Augeas is not installed +* Preserving a certificate's common name when using `renew` +* Save webroot values for renewal when they are entered interactively +* Problems with an invalid user-agent string on OS X +* Gracefully reporting the Apache plugin isn't usable when Augeas is not installed * Experimental support for Mageia has been added to `certbot-auto` More details about these changes can be found on our GitHub repo: @@ -180,10 +190,10 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ # 0.8.0 ## 06/02/2016 -* The main new feature in this release is the `register` subcommand which -can be used to register an account with the Let's Encrypt CA. -* Additionally, you can run `certbot register --update-registration` to -change the e-mail address associated with your registration. +* The main new feature in this release is the `register` subcommand which +can be used to register an account with the Let's Encrypt CA. +* Additionally, you can run `certbot register --update-registration` to +change the e-mail address associated with your registration. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ @@ -191,18 +201,18 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ # 0.7.0 ## 05/27/2016 -* `--must-staple` to request certificates from Let's Encrypt with the -OCSP must staple extension -* automatic configuration of OSCP stapling for Apache -* requesting certificates for domains found in the common name of a -custom CSR -* a number of bug fixes +* `--must-staple` to request certificates from Let's Encrypt with the +OCSP must staple extension +* automatic configuration of OSCP stapling for Apache +* requesting certificates for domains found in the common name of a +custom CSR +* a number of bug fixes More details about these changes can be found on our GitHub repo: -https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue +https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue # 0.6.0 -## 05/12/2016 +## 05/12/2016 * Renamed the client from `letsencrypt` to `certbot` * Fixed a small json deserialization error @@ -216,34 +226,34 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is% # 0.5.0 ## 04/05/2016 -* The ability to use the webroot plugin interactively. -* The flags --pre-hook, --post-hook, and --renew-hook which can be used -with the renew subcommand to register shell commands to run in -response to renewal events. Pre-hook commands will be run before -any certs are renewed, post-hook commands will be run after any -certs are renewed, and renew-hook commands will be run after each -cert is renewed. If no certs are due for renewal, no command is run. -* Cleaner renewal configuration files. In /etc/letsencrypt/renewal by -default, these files can be used to control what parameters are used -when renewing a specific certificate. -* A -q/--quiet flag which silences all output except errors. -* An --allow-subset-of-domains flag which can be used with the renew -command to prevent renewal failures for a subset of the requested -domains from causing the client to exit. +* The ability to use the webroot plugin interactively. +* The flags --pre-hook, --post-hook, and --renew-hook which can be used +with the renew subcommand to register shell commands to run in +response to renewal events. Pre-hook commands will be run before +any certs are renewed, post-hook commands will be run after any +certs are renewed, and renew-hook commands will be run after each +cert is renewed. If no certs are due for renewal, no command is run. +* Cleaner renewal configuration files. In /etc/letsencrypt/renewal by +default, these files can be used to control what parameters are used +when renewing a specific certificate. +* A -q/--quiet flag which silences all output except errors. +* An --allow-subset-of-domains flag which can be used with the renew +command to prevent renewal failures for a subset of the requested +domains from causing the client to exit. More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue # 0.4.2 ## 03/03/2016 -* Resolves problems encountered when compiling letsencrypt -against the new OpenSSL release. +* Resolves problems encountered when compiling letsencrypt +against the new OpenSSL release. * A patch fixing problems of using letsencrypt renew with configuration files -from private beta has been added. +from private beta has been added. More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 # 0.4.1 ## 02/29/2016 @@ -254,73 +264,73 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 * Fixes problems with parsing renewal config files from private beta More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 +https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 # 0.4.0 ## 02/10/2016 -* The new verb/subcommand `renew` can be used to renew your existing -certificates as they approach expiration. Running `letsencrypt renew` -will examine all existing certificate lineages and determine if any are -less than 30 days from expiration. If so, the client will use the -settings provided when you previously obtained the certificate to renew -it. The subcommand finishes by printing a summary of which renewals were -successful, failed, or not yet due. -* A `--dry-run` flag has been added to help with testing configuration -without affecting production rate limits. Currently supported by the -`renew` and `certonly` subcommands, providing `--dry-run` on the command -line will obtain certificates from the staging server without saving the -resulting certificates to disk. -* Major improvements have been added to letsencrypt-auto. This script -has been rewritten to include full support for Python 2.6, the ability -for letsencrypt-auto to update itself, and improvements to the -stability, security, and performance of the script. -* Support for Apache 2.2 has been added to the Apache plugin. +* The new verb/subcommand `renew` can be used to renew your existing +certificates as they approach expiration. Running `letsencrypt renew` +will examine all existing certificate lineages and determine if any are +less than 30 days from expiration. If so, the client will use the +settings provided when you previously obtained the certificate to renew +it. The subcommand finishes by printing a summary of which renewals were +successful, failed, or not yet due. +* A `--dry-run` flag has been added to help with testing configuration +without affecting production rate limits. Currently supported by the +`renew` and `certonly` subcommands, providing `--dry-run` on the command +line will obtain certificates from the staging server without saving the +resulting certificates to disk. +* Major improvements have been added to letsencrypt-auto. This script +has been rewritten to include full support for Python 2.6, the ability +for letsencrypt-auto to update itself, and improvements to the +stability, security, and performance of the script. +* Support for Apache 2.2 has been added to the Apache plugin. More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 - +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 + # 0.3.0 ## 01/27/2016 -* A non-interactive mode which can be enabled by including `-n` or -`--non-interactive` on the command line. This can be used to -guarantee the client will not prompt when run automatically using -cron/systemd. -* Preparation for the new letsencrypt-auto script. Over the past -couple months, we've been working on increasing the reliability and -security of letsencrypt-auto. A number of changes landed in this -release to prepare for the new version of this script. +* A non-interactive mode which can be enabled by including `-n` or +`--non-interactive` on the command line. This can be used to +guarantee the client will not prompt when run automatically using +cron/systemd. +* Preparation for the new letsencrypt-auto script. Over the past +couple months, we've been working on increasing the reliability and +security of letsencrypt-auto. A number of changes landed in this +release to prepare for the new version of this script. More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 # 0.2.0 ## 01/14/2016 -* Apache plugin support for non-Debian based systems. Support has been -added for modern Red Hat based systems such as Fedora 23, Red Hat 7, -and CentOS 7 running Apache 2.4. In theory, this plugin should be -able to be configured to run on any Unix-like OS running Apache 2.4. -* Relaxed PyOpenSSL version requirements. This adds support for systems -with PyOpenSSL versions 0.13 or 0.14. -* Resolves issues with the Apache plugin enabling an HTTP to HTTPS -redirect on some systems. -* Improved error messages from the client. +* Apache plugin support for non-Debian based systems. Support has been +added for modern Red Hat based systems such as Fedora 23, Red Hat 7, +and CentOS 7 running Apache 2.4. In theory, this plugin should be +able to be configured to run on any Unix-like OS running Apache 2.4. +* Relaxed PyOpenSSL version requirements. This adds support for systems +with PyOpenSSL versions 0.13 or 0.14. +* Resolves issues with the Apache plugin enabling an HTTP to HTTPS +redirect on some systems. +* Improved error messages from the client. More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 # 0.1.1 ## 12/15/2015 -* Fix a confusing UI path that caused some users to repeatedly renew -their certs while experimenting with the client, in some cases -hitting issuance rate limits -* Fixes numerous Apache configuration parser fixes -* Avoids attempting to issue for unqualified domain names like -"localhost" -* Fixes --webroot permission handling for non-root users +* Fix a confusing UI path that caused some users to repeatedly renew +their certs while experimenting with the client, in some cases +hitting issuance rate limits +* Fixes numerous Apache configuration parser fixes +* Avoids attempting to issue for unqualified domain names like +"localhost" +* Fixes --webroot permission handling for non-root users More details about these changes can be found on our GitHub repo: -https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 From 26a7023b8dd985674dbf9ff2d35d4065ca7cae9d Mon Sep 17 00:00:00 2001 From: Sagi Kedmi Date: Fri, 3 Mar 2017 02:49:34 +0200 Subject: [PATCH 021/128] Change QSA to NE in HTTPS redirection (#4204) * Change QSA to NE in HTTPS redirection * Seamless transition to new HTTPS redirection RewriteRule --- certbot-apache/certbot_apache/configurator.py | 41 +++++++++++++----- certbot-apache/certbot_apache/constants.py | 8 +++- .../certbot_apache/tests/configurator_test.py | 43 ++++++++++++++++--- 3 files changed, 73 insertions(+), 19 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 5639dae83..cdfc01626 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1315,18 +1315,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # even with save() and load() if not self._is_rewrite_engine_on(general_vh): self.parser.add_dir(general_vh.path, "RewriteEngine", "on") + names = ssl_vhost.get_names() for idx, name in enumerate(names): args = ["%{SERVER_NAME}", "={0}".format(name), "[OR]"] if idx == len(names) - 1: args.pop() self.parser.add_dir(general_vh.path, "RewriteCond", args) - if self.get_version() >= (2, 3, 9): - self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS_WITH_END) - else: - self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS) + + self._set_https_redirection_rewrite_rule(general_vh) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % (general_vh.filep, ssl_vhost.filep)) @@ -1336,12 +1333,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.info("Redirecting vhost in %s to ssl vhost in %s", general_vh.filep, ssl_vhost.filep) + def _set_https_redirection_rewrite_rule(self, vhost): + if self.get_version() >= (2, 3, 9): + self.parser.add_dir(vhost.path, "RewriteRule", + constants.REWRITE_HTTPS_ARGS_WITH_END) + else: + self.parser.add_dir(vhost.path, "RewriteRule", + constants.REWRITE_HTTPS_ARGS) + + def _verify_no_certbot_redirect(self, vhost): """Checks to see if a redirect was already installed by certbot. Checks to see if virtualhost already contains a rewrite rule that is identical to Certbot's redirection rewrite rule. + For graceful transition to new rewrite rules for HTTPS redireciton we + delete certbot's old rewrite rules and set the new one instead. + :param vhost: vhost to check :type vhost: :class:`~certbot_apache.obj.VirtualHost` @@ -1355,19 +1364,29 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. rewrite_args_dict = defaultdict(list) - pat = r'.*(directive\[\d+\]).*' + pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) if m: - dir_id = m.group(1) - rewrite_args_dict[dir_id].append(match) + dir_path = m.group(1) + rewrite_args_dict[dir_path].append(match) if rewrite_args_dict: redirect_args = [constants.REWRITE_HTTPS_ARGS, constants.REWRITE_HTTPS_ARGS_WITH_END] - for matches in rewrite_args_dict.values(): - if [self.aug.get(x) for x in matches] in redirect_args: + for dir_path, args_paths in rewrite_args_dict.items(): + arg_vals = [self.aug.get(x) for x in args_paths] + + # Search for past redirection rule, delete it, set the new one + if arg_vals in constants.OLD_REWRITE_HTTPS_ARGS: + self.aug.remove(dir_path) + self._set_https_redirection_rewrite_rule(vhost) + self.save() + raise errors.PluginEnhancementAlreadyPresent( + "Certbot has already enabled redirection") + + if arg_vals in redirect_args: raise errors.PluginEnhancementAlreadyPresent( "Certbot has already enabled redirection") diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index dcc635c4b..3cfeb4dd6 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -136,15 +136,19 @@ AUGEAS_LENS_DIR = pkg_resources.resource_filename( """Path to the Augeas lens directory""" REWRITE_HTTPS_ARGS = [ - "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,NE,R=permanent]"] """Apache version<2.3.9 rewrite rule arguments used for redirections to https vhost""" REWRITE_HTTPS_ARGS_WITH_END = [ - "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,NE,R=permanent]"] """Apache version >= 2.3.9 rewrite rule arguments used for redirections to https vhost""" +OLD_REWRITE_HTTPS_ARGS = [ + ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"], + ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"]] + HSTS_ARGS = ["always", "set", "Strict-Transport-Security", "\"max-age=31536000\""] """Apache header arguments for HSTS""" diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 937694267..45e701bd5 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -18,6 +18,7 @@ from certbot.tests import acme_util from certbot.tests import util as certbot_util from certbot_apache import configurator +from certbot_apache import constants from certbot_apache import parser from certbot_apache import obj @@ -1047,6 +1048,36 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue("rewrite_module" in self.config.parser.modules) + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_redirect_with_old_https_redirection(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + mock_exe.return_value = True + self.config.get_version = mock.Mock(return_value=(2, 2, 0)) + + ssl_vhost = self.config.choose_vhost("certbot.demo") + + # pylint: disable=protected-access + http_vhost = self.config._get_http_vhost(ssl_vhost) + + # Create an old (previously suppoorted) https redirectoin rewrite rule + self.config.parser.add_dir( + http_vhost.path, "RewriteRule", + ["^", + "https://%{SERVER_NAME}%{REQUEST_URI}", + "[L,QSA,R=permanent]"]) + + self.config.save() + + try: + self.config.enhance("certbot.demo", "redirect") + except errors.PluginEnhancementAlreadyPresent: + args_paths = self.config.parser.find_dir( + "RewriteRule", None, http_vhost.path, False) + arg_vals = [self.config.aug.get(x) for x in args_paths] + self.assertEqual(arg_vals, constants.REWRITE_HTTPS_ARGS) + + def test_redirect_with_conflict(self): self.config.parser.modules.add("rewrite_module") ssl_vh = obj.VirtualHost( @@ -1134,7 +1165,7 @@ class MultipleVhostsTest(util.ApacheTest): http_vhost.path, "RewriteRule", ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,QSA,R=permanent]"]) + "[L,NE,R=permanent]"]) self.config.save() ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -1145,7 +1176,7 @@ class MultipleVhostsTest(util.ApacheTest): conf_text = open(ssl_vhost.filep).read() commented_rewrite_rule = ("# RewriteRule ^ " "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,QSA,R=permanent]") + "[L,NE,R=permanent]") self.assertTrue(commented_rewrite_rule in conf_text) mock_get_utility().add_message.assert_called_once_with(mock.ANY, @@ -1164,7 +1195,7 @@ class MultipleVhostsTest(util.ApacheTest): "RewriteCond", ["%{DOCUMENT_ROOT}/%{REQUEST_FILENAME}", "!-f"]) self.config.parser.add_dir( http_vhost.path, "RewriteRule", - ["^(.*)$", "b://u%{REQUEST_URI}", "[P,QSA,L]"]) + ["^(.*)$", "b://u%{REQUEST_URI}", "[P,NE,L]"]) # Add a chunk that should be commented out. self.config.parser.add_dir(http_vhost.path, @@ -1175,7 +1206,7 @@ class MultipleVhostsTest(util.ApacheTest): http_vhost.path, "RewriteRule", ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,QSA,R=permanent]"]) + "[L,NE,R=permanent]"]) self.config.save() @@ -1186,13 +1217,13 @@ class MultipleVhostsTest(util.ApacheTest): not_commented_cond1 = ("RewriteCond " "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f") not_commented_rewrite_rule = ("RewriteRule " - "^(.*)$ b://u%{REQUEST_URI} [P,QSA,L]") + "^(.*)$ b://u%{REQUEST_URI} [P,NE,L]") commented_cond1 = "# RewriteCond %{HTTPS} !=on" commented_cond2 = "# RewriteCond %{HTTPS} !^$" commented_rewrite_rule = ("# RewriteRule ^ " "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,QSA,R=permanent]") + "[L,NE,R=permanent]") self.assertTrue(not_commented_cond1 in conf_line_set) self.assertTrue(not_commented_rewrite_rule in conf_line_set) From 53117b0ce0f9e781ae9a44570cfc7dc3891bb918 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 2 Mar 2017 17:27:29 -0800 Subject: [PATCH 022/128] Remove UnexpectedUpdate exceptions. (#4197) * Remove UnexpectedUpdate exceptions. These exceptions trigger when the server sends the client back an object with a field that doesn't exactly match what the client previously sent. This causes unnecessary breakage in various cases, doesn't prevent any problems, and isn't required by spec. * Back out all UnexpectedUpdate removals except registration update. --- acme/acme/client.py | 3 --- acme/acme/client_test.py | 10 ---------- certbot/client.py | 2 -- 3 files changed, 15 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index ddcba7635..6c5ed79a2 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -134,8 +134,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) - if updated_regr != regr: - raise errors.UnexpectedUpdate(regr) return updated_regr def deactivate_registration(self, regr): @@ -301,7 +299,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes response = self.net.get(authzr.uri) updated_authzr = self._authzr_from_response( response, authzr.body.identifier, authzr.uri, authzr.new_cert_uri) - # TODO: check and raise UnexpectedUpdate return updated_authzr, response def request_issuance(self, csr, authzrs): diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index b3db21ac9..7e7ffe779 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -121,8 +121,6 @@ class ClientTest(unittest.TestCase): # TODO: split here and separate test self.response.json.return_value = self.regr.body.update( contact=()).to_json() - self.assertRaises( - errors.UnexpectedUpdate, self.client.update_registration, self.regr) def test_deactivate_account(self): self.response.headers['Location'] = self.regr.uri @@ -130,14 +128,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.deactivate_registration(self.regr)) - def test_deactivate_account_bad_registration_returned(self): - self.response.headers['Location'] = self.regr.uri - self.response.json.return_value = "some wrong registration thing" - self.assertRaises( - errors.UnexpectedUpdate, - self.client.deactivate_registration, - self.regr) - def test_query_registration(self): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) diff --git a/certbot/client.py b/certbot/client.py index 95882a9fc..a342c1bf3 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -155,8 +155,6 @@ def perform_registration(acme, config): :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` - - :raises .UnexpectedUpdate: """ try: return acme.register(messages.NewRegistration.from_data(email=config.email)) From 93908a33bcdf79db297c38445cc31266ea066a31 Mon Sep 17 00:00:00 2001 From: yomna Date: Thu, 2 Mar 2017 17:28:45 -0800 Subject: [PATCH 023/128] [#3451] small changes to the standalone documentation (#4247) --- docs/using.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 628043ff9..1582c5ab7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -144,6 +144,10 @@ the ``--nginx`` flag on the commandline. Standalone ---------- +Use standalone mode to obtain a cert if you don't want to use (or don't currently have) +existing server software. The standalone plugin does not rely on any other server +software running on the machine where you obtain the cert. + To obtain a cert using a "standalone" webserver, you can use the standalone plugin by including ``certonly`` and ``--standalone`` on the command line. This plugin needs to bind to port 80 or 443 in @@ -154,10 +158,8 @@ one of the options shown below on the command line. * ``--standalone-supported-challenges http-01`` to use port 80 * ``--standalone-supported-challenges tls-sni-01`` to use port 443 -The standalone plugin does not rely on any other server software running -on the machine where you obtain the certificate. It must still be possible -for that machine to accept inbound connections from the Internet on the -specified port using each requested domain name. +It must still be possible for your machine to accept inbound connections from +the Internet on the specified port using each requested domain name. Manual ------ From 12a6e49cf11d3b7de623c6279ffbf1387328688f Mon Sep 17 00:00:00 2001 From: Blake Griffith Date: Thu, 2 Mar 2017 21:16:19 -0800 Subject: [PATCH 024/128] Remove use of sha1 (#4271) These are not security critical uses of sha1 but they should still be removed. --- acme/acme/challenges.py | 2 +- certbot/tests/crypto_util_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 3b1e90166..ac4e3d60a 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -445,7 +445,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ # pylint: disable=protected-access sans = crypto_util._pyopenssl_cert_or_req_san(cert) - logger.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), sans) return self.z_domain.decode() in sans def simple_verify(self, chall, domain, account_public_key, diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index a580574a4..946e772c1 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -336,8 +336,8 @@ class CertLoaderTest(unittest.TestCase): from certbot.crypto_util import pyopenssl_load_certificate cert, file_type = pyopenssl_load_certificate(CERT) - self.assertEqual(cert.digest('sha1'), - OpenSSL.crypto.load_certificate(file_type, CERT).digest('sha1')) + self.assertEqual(cert.digest('sha256'), + OpenSSL.crypto.load_certificate(file_type, CERT).digest('sha256')) def test_load_invalid_cert(self): from certbot.crypto_util import pyopenssl_load_certificate From 1507b6b7316b0e0ba5aabc56cbaf64904ae15d59 Mon Sep 17 00:00:00 2001 From: yonjah Date: Sat, 4 Mar 2017 00:34:30 +0800 Subject: [PATCH 025/128] Added documentation about renew exit status #Fixes #4090 (#4234) * Added documentation about renew exit status #Fixes #4090 * recommend using post-hook instead of renew-hook --- docs/using.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/using.rst b/docs/using.rst index 1582c5ab7..104c688e7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -411,6 +411,13 @@ Certbot is working hard to improve the renewal process, and we apologize for any inconvenience you encounter in integrating these commands into your individual environment. +.. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. + This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. + If you write a custom script and expect to run a command only after a cert was actually renewed + you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal + and when renewal is not necessary. + + Modifying the Renewal Configuration File ---------------------------------------- From 80326511bb1f0f7392f2c5bd259a74e3c6c0c195 Mon Sep 17 00:00:00 2001 From: Alex Bowers Date: Fri, 3 Mar 2017 18:28:05 +0000 Subject: [PATCH 026/128] Improve error reporting for hooks (#4235) * Improve error reporting for hooks * My bad * Whitespace. --- certbot/hooks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/certbot/hooks.py b/certbot/hooks.py index 5cda478cc..ada3d3aaa 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -44,8 +44,12 @@ def validate_hook(shell_cmd, hook_name): cmd = shell_cmd.split(None, 1)[0] if not _prog(cmd): path = os.environ["PATH"] - msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format( - cmd, path, hook_name) + if os.path.exists(cmd): + msg = "{1}-hook command {0} exists, but is not executable.".format(cmd, hook_name) + else: + msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format( + cmd, path, hook_name) + raise errors.HookCommandNotFound(msg) def pre_hook(config): From ea578870da558c86546fb6b9b34d95ffea6f12a3 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 3 Mar 2017 10:37:31 -0800 Subject: [PATCH 027/128] ipdb can now be run without pip installing. (#4257) --- docs/contributing.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 5f939e947..de9904936 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -69,10 +69,8 @@ either in the same directory as ``foo.py`` or in the ``tests`` subdirectory (if there isn't, make one). While you are working on your code and tests, run ``python foo_test.py`` to run the relevant tests. -For debugging, we recommend running ``pip install ipdb`` and putting -``import ipdb; ipdb.set_trace()`` statements inside the source -code. Alternatively, you can use Python's standard library `pdb`, -but you won't get TAB completion. +For debugging, we recommend putting +``import ipdb; ipdb.set_trace()`` statements inside the source code. Once you are done with your code changes, and the tests in ``foo_test.py`` pass, run all of the unittests for Certbot with ``tox -e py27`` (this uses Python From 2862ade0b12c3970ba36c9cefa8f4b89b7a82e0f Mon Sep 17 00:00:00 2001 From: Shiloh Heurich Date: Fri, 3 Mar 2017 13:44:12 -0500 Subject: [PATCH 028/128] docs(ciphers): newer keylength.com recommendations (#4266) --- docs/ciphers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 31ce45963..1b320cdf9 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -255,7 +255,7 @@ I have access to an English-language summary of the recommendations. Keylength.com ~~~~~~~~~~~~~ -Damien Giry collects recommendations by academic researchers and standards organizations about keylengths for particular cryptoperiods, years, or security levels. The keylength recommendations of the various sources are summarized in a chart. This site has been updated over time and includes expert guidance from eight sources published between 2000 and 2015. +Damien Giry collects recommendations by academic researchers and standards organizations about keylengths for particular cryptoperiods, years, or security levels. The keylength recommendations of the various sources are summarized in a chart. This site has been updated over time and includes expert guidance from eight sources published between 2000 and 2017. http://www.keylength.com/ From 4a2582dda4f4e9ff0090179eb10752377cdd47e9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 6 Mar 2017 17:31:28 -0800 Subject: [PATCH 029/128] Remove broken known issues link (#4294) --- docs/contributing.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index de9904936..e05c302b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -283,8 +283,7 @@ Steps: including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. 5. If your code touches communication with an ACME server/Boulder, you - should run the integration tests, see `integration`_. See `Known Issues`_ - for some common failures that have nothing to do with your code. + should run the integration tests, see `integration`_. 6. Submit the PR. 7. Did your tests pass on Travis? If they didn't, fix any errors. From 82736e21d4966f2c68f2d3dfcd88856580af55bf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 6 Mar 2017 17:32:49 -0800 Subject: [PATCH 030/128] Improve path_surgery warning (#4293) Stops output like: Failed to find certbot.log in PATH: ... renew-hook command certbot.log exists, but is not executable. --- certbot/plugins/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index e45c26735..f0e2f4c5b 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -33,6 +33,6 @@ def path_surgery(cmd): return True else: expanded = " expanded" if any(added) else "" - logger.warning("Failed to find %s in%s PATH: %s", cmd, + logger.warning("Failed to find executable %s in%s PATH: %s", cmd, expanded, path) return False From 5d75906b2761a4aefa908afe0a252fe5759067fb Mon Sep 17 00:00:00 2001 From: sedrubal Date: Tue, 7 Mar 2017 02:34:03 +0100 Subject: [PATCH 031/128] Fix print for python3 in certbot-auto (#4263) Use printfunction from __future__ in order to get letsencrypt installed on a python3 only system. --- letsencrypt-auto-source/letsencrypt-auto | 7 +++++-- letsencrypt-auto-source/pieces/fetch.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index cc248a36a..35c7a530c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1093,6 +1093,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1197,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index 365a5a36a..e7ebb9e0a 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -10,6 +10,9 @@ On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -111,12 +114,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 From ee4e7c4f71ab181067c878f8fb095611fd19d6c0 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Mon, 6 Mar 2017 18:34:14 -0800 Subject: [PATCH 032/128] Improve packaging guide. Correct tagging format. Add request for random offsets for renewal. Make all bulleted lists consistent. Remove obsolete `letsencrypt` package for Fedora. Remove discouraged letshelp-certbot package. --- docs/packaging.rst | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index 1a1b83f50..c3cdfb579 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -16,21 +16,19 @@ The following scripts are used in the process: - https://github.com/letsencrypt/letsencrypt/blob/master/tools/release.sh -We currently version with the following scheme: +We use git tags to identify releases, using `Semantic Versioning`_. For +example: `v0.11.1`. -- ``0.1.0`` -- ``0.2.0dev`` for developement in ``master`` -- ``0.2.0`` (only temporarily in ``master``) -- ... +.. _`Semantic Versioning`: http://semver.org/ Notes for package maintainers ============================= -0. Please use our releases, not ``master``! +0. Please use our tagged releases, not ``master``! 1. Do not package ``certbot-compatibility-test`` or ``letshelp-certbot`` - it's only used internally. -2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. +2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. Add a random per-system offset to avoid having a large number of clients hit Let's Encrypt's servers simultaneously. 3. ``jws`` is an internal script for ``acme`` module and it doesn't have to be packaged - it's mostly for debugging: you can use it as ``echo foo | jws sign | jws verify``. @@ -44,34 +42,33 @@ Arch ---- From our official releases: + - https://www.archlinux.org/packages/community/any/python2-acme - https://www.archlinux.org/packages/community/any/certbot - https://www.archlinux.org/packages/community/any/certbot-apache - https://www.archlinux.org/packages/community/any/certbot-nginx -- https://www.archlinux.org/packages/community/any/letshelp-certbot From ``master``: https://aur.archlinux.org/packages/certbot-git Debian (and its derivatives, including Ubuntu) ------ -https://packages.debian.org/sid/certbot -https://packages.debian.org/sid/python-certbot -https://packages.debian.org/sid/python-certbot-apache +- https://packages.debian.org/sid/certbot +- https://packages.debian.org/sid/python-certbot +- https://packages.debian.org/sid/python-certbot-apache Fedora ------ In Fedora 23+. -- https://admin.fedoraproject.org/pkgdb/package/letsencrypt/ - https://admin.fedoraproject.org/pkgdb/package/certbot/ - https://admin.fedoraproject.org/pkgdb/package/python-acme/ FreeBSD ------- -https://svnweb.freebsd.org/ports/head/security/py-certbot/ +- https://svnweb.freebsd.org/ports/head/security/py-certbot/ GNU Guix -------- From b23a2593902df4c5b401b57e6b0ec17bb1fd3e90 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 11:40:17 -0800 Subject: [PATCH 033/128] Don't install unnecessary packages in Dockerfile --- Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 157578cd5..fb5695a77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ FROM python:2-alpine -MAINTAINER Jakub Warmuz -MAINTAINER William Budington EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt @@ -9,11 +7,8 @@ ENTRYPOINT [ "certbot" ] COPY . src RUN apk add --no-cache --virtual .certbot-deps \ - dialog \ - augeas-libs \ libffi \ libssl1.0 \ - wget \ ca-certificates \ binutils RUN apk add --no-cache --virtual .build-deps \ @@ -25,6 +20,4 @@ RUN apk add --no-cache --virtual .build-deps \ && pip install --no-cache-dir \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ - --editable /opt/certbot/src/certbot-apache \ - --editable /opt/certbot/src/certbot-nginx \ && apk del .build-deps From 6f8bf74c53c7a479209eeb7da83dc74b7f783f15 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 12:00:21 -0800 Subject: [PATCH 034/128] sort directives that never change in Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fb5695a77..fcc11e8a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ FROM python:2-alpine +ENTRYPOINT [ "certbot" ] EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot -ENTRYPOINT [ "certbot" ] + COPY . src RUN apk add --no-cache --virtual .certbot-deps \ From f94c4eeff1394dba89c194e7f28b6a42725c7ed3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 12:13:07 -0800 Subject: [PATCH 035/128] only copy necessary files in Dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fcc11e8a5..11d53788b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,9 @@ EXPOSE 80 443 VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot -COPY . src +COPY CHANGES.rst README.rst setup.py src/ +COPY acme src/acme +COPY certbot src/certbot RUN apk add --no-cache --virtual .certbot-deps \ libffi \ From fb9b62a282cd1fdb8d279a9710cc3bea95a7f744 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 12:17:23 -0800 Subject: [PATCH 036/128] add Dockerfile-old --- Dockerfile-old | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 Dockerfile-old diff --git a/Dockerfile-old b/Dockerfile-old new file mode 100644 index 000000000..d42b632d4 --- /dev/null +++ b/Dockerfile-old @@ -0,0 +1,70 @@ +# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297 +# it is more likely developers will already have ubuntu:trusty rather +# than e.g. debian:jessie and image size differences are negligible +FROM ubuntu:trusty +MAINTAINER Jakub Warmuz +MAINTAINER William Budington + +# Note: this only exposes the port to other docker containers. You +# still have to bind to 443@host at runtime, as per the ACME spec. +EXPOSE 443 + +# TODO: make sure --config-dir and --work-dir cannot be changed +# through the CLI (certbot-docker wrapper that uses standalone +# authenticator and text mode only?) +VOLUME /etc/letsencrypt /var/lib/letsencrypt + +WORKDIR /opt/certbot + +# no need to mkdir anything: +# https://docs.docker.com/reference/builder/#copy +# If doesn't exist, it is created along with all missing +# directories in its path. + +ENV DEBIAN_FRONTEND=noninteractive + +COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# the above is not likely to change, so by putting it further up the +# Dockerfile we make sure we cache as much as possible + + +COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ + +# all above files are necessary for setup.py and venv setup, however, +# package source code directory has to be copied separately to a +# subdirectory... +# https://docs.docker.com/reference/builder/#copy: "If is a +# directory, the entire contents of the directory are copied, +# including filesystem metadata. Note: The directory itself is not +# copied, just its contents." Order again matters, three files are far +# more likely to be cached than the whole project directory + +COPY certbot /opt/certbot/src/certbot/ +COPY acme /opt/certbot/src/acme/ +COPY certbot-apache /opt/certbot/src/certbot-apache/ +COPY certbot-nginx /opt/certbot/src/certbot-nginx/ + + +RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv + +# PATH is set now so pipstrap upgrades the correct (v)env +ENV PATH /opt/certbot/venv/bin:$PATH +RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \ + /opt/certbot/venv/bin/pip install \ + -e /opt/certbot/src/acme \ + -e /opt/certbot/src \ + -e /opt/certbot/src/certbot-apache \ + -e /opt/certbot/src/certbot-nginx + +# install in editable mode (-e) to save space: it's not possible to +# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); +# this might also help in debugging: you can "docker run --entrypoint +# bash" and investigate, apply patches, etc. + +ENTRYPOINT [ "certbot" ] From eb1e3c4cb3c87d48563f80eb197fb06e817c4934 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 13:56:43 -0800 Subject: [PATCH 037/128] add docker-warning script --- tools/docker-warning.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 tools/docker-warning.sh diff --git a/tools/docker-warning.sh b/tools/docker-warning.sh new file mode 100755 index 000000000..e4f5f40ee --- /dev/null +++ b/tools/docker-warning.sh @@ -0,0 +1,4 @@ +#!/bin/sh -e +echo "Warning: This Docker image will soon be switching to Alpine Linux." >&2 +echo "You can switch now using the certbot/certbot repo on Docker Hub." >&2 +exec /opt/certbot/venv/bin/certbot $@ From ca42e992228fc20ed88dfe3d2da5146c1e76b8a8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 3 Mar 2017 13:57:03 -0800 Subject: [PATCH 038/128] add certbot wrapper to Dockerfile-old --- Dockerfile-old | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile-old b/Dockerfile-old index d42b632d4..7bce82e0c 100644 --- a/Dockerfile-old +++ b/Dockerfile-old @@ -67,4 +67,9 @@ RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \ # this might also help in debugging: you can "docker run --entrypoint # bash" and investigate, apply patches, etc. +# set up certbot/letsencrypt wrapper to warn people about Dockerfile changes +COPY tools/docker-warning.sh /opt/certbot/bin/certbot +RUN ln -s /opt/certbot/bin/certbot /opt/certbot/bin/letsencrypt +ENV PATH /opt/certbot/bin:$PATH + ENTRYPOINT [ "certbot" ] From 6669b95a4e4a805aee274a45c5c19ba275ff34a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=B8=89?= Date: Wed, 8 Mar 2017 03:19:55 +0800 Subject: [PATCH 039/128] Updated the deprecated arguments (#4306) --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 104c688e7..e579a70b7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -155,8 +155,8 @@ order to perform domain validation, so you may need to stop your existing webserver. To control which port the plugin uses, include one of the options shown below on the command line. - * ``--standalone-supported-challenges http-01`` to use port 80 - * ``--standalone-supported-challenges tls-sni-01`` to use port 443 + * ``--preferred-challenges http-01`` to use port 80 + * ``--preferred-challenges tls-sni-01`` to use port 443 It must still be possible for your machine to accept inbound connections from the Internet on the specified port using each requested domain name. From 8f101034960ffc1e47879314585898efda234e60 Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 8 Mar 2017 20:10:12 -0500 Subject: [PATCH 040/128] Make argparse dependency unconditional. (#2249) The primary motivation is to avoid a branch, giving bugs one fewer place to hide. But, as a bonus, more people get a more bugfixed version of argparse. (To use the example from the argparse docs, people stuck on Python 3.2.3 can get bugfixes that made it into the stdlib only in 3.2.4.) --- acme/setup.py | 3 +-- setup.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index f169f59a7..48210108a 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -8,6 +8,7 @@ version = '0.13.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ + 'argparse', # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', @@ -30,8 +31,6 @@ install_requires = [ # Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ - # only some distros recognize stdlib argparse as already satisfying - 'argparse', 'mock<1.1.0', ]) else: diff --git a/setup.py b/setup.py index 0c47b973f..8ce4c51c9 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ version = meta['version'] # https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), + 'argparse', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. @@ -58,8 +59,6 @@ install_requires = [ # Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ - # only some distros recognize stdlib argparse as already satisfying - 'argparse', 'mock<1.1.0', ]) else: From 662c323b55711f0f32a99cb32e846ab6961dce2d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 10 Mar 2017 10:58:03 -0800 Subject: [PATCH 041/128] Build wheels correctly for Python 2.6 (#4313) * stop conditionally pinning mock version in acme * stop conditionally pinning mock version in certbot * stop conditionally pinning mock version in apache * stop conditionally pinning mock version in nginx * stop conditionally pinning mock version in letshelp * stop conditionally pinning mock version in compatibility-test --- acme/setup.py | 10 +--------- certbot-apache/setup.py | 6 +----- certbot-compatibility-test/setup.py | 6 +----- certbot-nginx/setup.py | 6 +----- letshelp-certbot/setup.py | 5 +---- setup.py | 10 +--------- 6 files changed, 6 insertions(+), 37 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 48210108a..df9937032 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -13,6 +13,7 @@ install_requires = [ # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', # Connection.set_tlsext_host_name (>=0.13) + 'mock', 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', @@ -27,15 +28,6 @@ install_requires = [ 'six', ] -# env markers in extras_require cause problems with older pip: #517 -# Keep in sync with conditional_requirements.py. -if sys.version_info < (2, 7): - install_requires.extend([ - 'mock<1.1.0', - ]) -else: - install_requires.append('mock') - dev_extras = [ 'nose', 'tox', diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 56a48abc6..db8cb11db 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -10,6 +10,7 @@ version = '0.13.0.dev0' install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), + 'mock', 'python-augeas', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: @@ -18,11 +19,6 @@ install_requires = [ 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 73d3b704b..238d7a2d5 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -9,16 +9,12 @@ version = '0.13.0.dev0' install_requires = [ 'certbot', 'certbot-apache', + 'mock', 'six', 'requests', 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 24c2564b9..bdc45b9b4 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -10,6 +10,7 @@ version = '0.13.0.dev0' install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), + 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? # For pkg_resources. >=1.0 so pip resolves it to a version cryptography @@ -18,11 +19,6 @@ install_requires = [ 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b616da688..b26ab41fe 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -7,12 +7,9 @@ from setuptools import find_packages version = '0.7.0.dev0' install_requires = [ + 'mock', 'setuptools', # pkg_resources ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags diff --git a/setup.py b/setup.py index 8ce4c51c9..0e8d19a22 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate + 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'PyOpenSSL', 'pyrfc3339', @@ -55,15 +56,6 @@ install_requires = [ 'zope.interface', ] -# env markers in extras_require cause problems with older pip: #517 -# Keep in sync with conditional_requirements.py. -if sys.version_info < (2, 7): - install_requires.extend([ - 'mock<1.1.0', - ]) -else: - install_requires.append('mock') - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', From 91f4b2b571d0ad3b0b09d150206dce2c8e9b9b0d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 10 Mar 2017 17:27:09 -0800 Subject: [PATCH 042/128] Improve macOS contributor instructions (#4327) * remove instructions about removed Vagrantfile * rewrite docker instructions * say where docker-compose can be run * give better intro for macOS devs * prompt people for permission to install OS packages * reword awkward sentence * Change WORKDIR to /opt/certbot/src This change is OK because all paths used in Dockerfile-dev are absolute paths. * remove 'cd src' instructions for Dockerfile-dev * Improve docker-compose testing instructions --- Dockerfile-dev | 2 +- docs/contributing.rst | 79 +++++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index dbb45f75e..2a89b2ff5 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -13,7 +13,7 @@ EXPOSE 443 # authenticator and text mode only?) VOLUME /etc/letsencrypt /var/lib/letsencrypt -WORKDIR /opt/certbot +WORKDIR /opt/certbot/src # no need to mkdir anything: # https://docs.docker.com/reference/builder/#copy diff --git a/docs/contributing.rst b/docs/contributing.rst index e05c302b8..5cdf86147 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -14,15 +14,24 @@ Getting Started Running a local copy of the client ---------------------------------- -Running the client in developer mode from your local tree is a little -different than running ``certbot-auto``. To get set up, do these things -once: +Running the client in developer mode from your local tree is a little different +than running Certbot as a user. To get set up, clone our git repository by +running: .. code-block:: shell git clone https://github.com/certbot/certbot + +If you're on macOS, we recommend you skip the rest of this section and instead +run Certbot in Docker. You can find instructions for how to do this :ref:`here +`. If you're running on Linux, you can run the following commands to +install dependencies and set up a virtual environment where you can run +Certbot. You only need to do this once. + +.. code-block:: shell + cd certbot - ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only + ./certbot-auto --os-packages-only ./tools/venv.sh Then in each shell where you're working on the client, do: @@ -343,56 +352,36 @@ This should generate documentation in the ``docs/_build/html`` directory. -Other methods for running the client -==================================== +.. _docker: -Vagrant -------- +Running the client with Docker +============================== -If you are a Vagrant user, Certbot comes with a Vagrantfile that -automates setting up a development environment in an Ubuntu 14.04 -LTS VM. To set it up, simply run ``vagrant up``. The repository is -synced to ``/vagrant``, so you can get started with: +You can use Docker Compose to quickly set up an environment for running and +testing Certbot. This is especially useful for macOS users. To install Docker +Compose, follow the instructions at https://docs.docker.com/compose/install/. -.. code-block:: shell +.. note:: Linux users can simply run ``pip install docker-compose`` to get + Docker Compose after installing Docker Engine and activating your shell as + described in the :ref:`Getting Started ` section. - vagrant ssh - cd /vagrant - sudo ./venv/bin/certbot +Now you can develop on your host machine, but run Certbot and test your changes +in Docker. When using ``docker-compose`` make sure you are inside your clone of +the Certbot repository. As an example, you can run the following command to +check for linting errors:: -Support for other Linux distributions coming soon. + docker-compose run --rm --service-ports development bash -c 'tox -e lint' -.. note:: - Unfortunately, Python distutils and, by extension, setup.py and - tox, use hard linking quite extensively. Hard linking is not - supported by the default sync filesystem in Vagrant. As a result, - all actions with these commands are *significantly slower* in - Vagrant. One potential fix is to `use NFS`_ (`related issue`_). +You can also leave a terminal open running a shell in the Docker container and +modify Certbot code in another window. The Certbot repo on your host machine is +mounted inside of the container so any changes you make immediately take +effect. To do this, run:: -.. _use NFS: http://docs.vagrantup.com/v2/synced-folders/nfs.html -.. _related issue: https://github.com/ClusterHQ/flocker/issues/516 + docker-compose run --rm --service-ports development bash +Now running the check for linting errors described above is as easy as:: -Docker ------- - -OSX users will probably find it easiest to set up a Docker container for -development. Certbot comes with a Dockerfile (``Dockerfile-dev``) -for doing so. To use Docker on OSX, install and setup docker-machine using the -instructions at https://docs.docker.com/installation/mac/. - -To build the development Docker image:: - - docker build -t certbot -f Dockerfile-dev . - -Now run tests inside the Docker image: - -.. code-block:: shell - - docker run -it certbot bash - cd src - tox -e py27 - + tox -e lint .. _prerequisites: From 0e735e360c197a1a77e97d806ff4a4a9b90a073a Mon Sep 17 00:00:00 2001 From: Amjad Mashaal Date: Sat, 11 Mar 2017 04:09:39 +0200 Subject: [PATCH 043/128] remove unnecessary whitespace from tools/release.sh --- tools/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/release.sh b/tools/release.sh index 75a4af29c..81582cef0 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -88,9 +88,9 @@ SetVersion() { sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py - + # interactive user input - git add -p certbot $SUBPKGS certbot-compatibility-test + git add -p certbot $SUBPKGS certbot-compatibility-test } From da1459df0d3db4339f62456d7dac3abf5df03723 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 14 Mar 2017 16:36:39 -0700 Subject: [PATCH 044/128] use docker hub in install guide --- docs/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index aa59e44ec..548c8f2a3 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -120,7 +120,7 @@ to, `install Docker`_, then issue the following command: sudo docker run -it --rm -p 443:443 -p 80:80 --name certbot \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest certonly + certbot/certbot certonly Running Certbot with the ``certonly`` command will obtain a certificate and place it in the directory ``/etc/letsencrypt/live`` on your system. Because Certonly cannot install the certificate from From 018a304cd6335c7cb6586faadd89b331cffc485c Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 14 Mar 2017 21:44:57 -0700 Subject: [PATCH 045/128] Remove Link rel=next for authzs and new-certs. (#4303) An early version of the spec indicated that clients should process issuance sequentially, following Link rel=next from an account URL to an authz URL, to a new-cert URL. However, the spec has long since moved to putting these URLs in the directory. Certbot nominally supports either; This change consolidates on always using the directory, simplifying things and making the transition to the latest ACME spec easier. * Revert "Revert "Remove Link rel=next for authzs and new-certs." (#4277)" This reverts commit 11ec1eb91148f6420e10d2296b8a507c90337792. * Save new_authzr_uri with account for older clients. * Add test that new_authzr_uri exists in regr. * Restore backwards compatibility for new_authzr_uri. * Fix account_test. * Add test for deprecated URI argument to request_challenges. * Review feedback. * Fix test * Add omitempty to new_cert_uri. --- acme/acme/client.py | 38 +++++++----------------- acme/acme/client_test.py | 47 +++++++++--------------------- acme/acme/jose/json_util.py | 2 +- acme/acme/messages.py | 10 +++---- acme/acme/messages_test.py | 6 +--- acme/examples/example_client.py | 3 +- certbot/account.py | 33 ++++++++++++++++----- certbot/auth_handler.py | 3 +- certbot/client.py | 2 +- certbot/interfaces.py | 2 +- certbot/main.py | 2 +- certbot/tests/account_test.py | 37 ++++++++++++++++------- certbot/tests/acme_util.py | 1 - certbot/tests/auth_handler_test.py | 3 +- certbot/tests/display/ops_test.py | 4 +-- certbot/tests/main_test.py | 9 ++++-- 16 files changed, 97 insertions(+), 105 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 6c5ed79a2..d6166960d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -71,20 +71,13 @@ class Client(object): # pylint: disable=too-many-instance-attributes self.directory = directory @classmethod - def _regr_from_response(cls, response, uri=None, new_authzr_uri=None, - terms_of_service=None): + def _regr_from_response(cls, response, uri=None, terms_of_service=None): if 'terms-of-service' in response.links: terms_of_service = response.links['terms-of-service']['url'] - if 'next' in response.links: - new_authzr_uri = response.links['next']['url'] - - if new_authzr_uri is None: - raise errors.ClientError('"next" link missing') return messages.RegistrationResource( body=messages.Registration.from_json(response.json()), uri=response.headers.get('Location', uri), - new_authzr_uri=new_authzr_uri, terms_of_service=terms_of_service) def register(self, new_reg=None): @@ -117,7 +110,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # (c.f. acme-spec #94) return self._regr_from_response( - response, uri=regr.uri, new_authzr_uri=regr.new_authzr_uri, + response, uri=regr.uri, terms_of_service=regr.terms_of_service) def update_registration(self, regr, update=None): @@ -172,19 +165,10 @@ class Client(object): # pylint: disable=too-many-instance-attributes return self.update_registration( regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier, - uri=None, new_cert_uri=None): - # pylint: disable=no-self-use - if new_cert_uri is None: - try: - new_cert_uri = response.links['next']['url'] - except KeyError: - raise errors.ClientError('"next" link missing') - + def _authzr_from_response(self, response, identifier, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), - uri=response.headers.get('Location', uri), - new_cert_uri=new_cert_uri) + uri=response.headers.get('Location', uri)) if authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr @@ -193,17 +177,16 @@ class Client(object): # pylint: disable=too-many-instance-attributes """Request challenges. :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: ``new-authorization`` URI. If omitted, - will default to value found in ``directory``. + :param str new_authzr_uri: Deprecated. Do not use. :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz - if new_authzr_uri is None else new_authzr_uri, - new_authz) + response = self.net.post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) @@ -217,6 +200,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes documentation. :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. :returns: Authorization Resource. :rtype: `.AuthorizationResource` @@ -298,7 +282,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ response = self.net.get(authzr.uri) updated_authzr = self._authzr_from_response( - response, authzr.body.identifier, authzr.uri, authzr.new_cert_uri) + response, authzr.body.identifier, authzr.uri) return updated_authzr, response def request_issuance(self, csr, authzrs): @@ -321,7 +305,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument response = self.net.post( - authzrs[0].new_cert_uri, # TODO: acme-spec #90 + self.directory.new_cert, req, content_type=content_type, headers={'Accept': content_type}) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 7e7ffe779..09bb38c00 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -40,6 +40,8 @@ class ClientTest(unittest.TestCase): 'https://www.letsencrypt-demo.org/acme/revoke-cert', messages.NewAuthorization: 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', }) from acme.client import Client @@ -56,7 +58,6 @@ class ClientTest(unittest.TestCase): self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - new_authzr_uri='https://www.letsencrypt-demo.org/acme/new-reg', terms_of_service='https://www.letsencrypt-demo.org/tos') # Authorization @@ -72,8 +73,7 @@ class ClientTest(unittest.TestCase): typ=messages.IDENTIFIER_FQDN, value='example.com'), challenges=(challb,), combinations=None) self.authzr = messages.AuthorizationResource( - body=self.authz, uri=authzr_uri, - new_cert_uri='https://www.letsencrypt-demo.org/acme/new-cert') + body=self.authz, uri=authzr_uri) # Request issuance self.certr = messages.CertificateResource( @@ -98,18 +98,12 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.response.headers['Location'] = self.regr.uri self.response.links.update({ - 'next': {'url': self.regr.new_authzr_uri}, 'terms-of-service': {'url': self.regr.terms_of_service}, }) self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments - def test_register_missing_next(self): - self.response.status_code = http_client.CREATED - self.assertRaises( - errors.ClientError, self.client.register, self.new_reg) - def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -132,13 +126,6 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) - def test_query_registration_updates_new_authzr_uri(self): - self.response.json.return_value = self.regr.body.to_json() - self.response.links = {'next': {'url': 'UPDATED'}} - self.assertEqual( - 'UPDATED', - self.client.query_registration(self.regr).new_authzr_uri) - def test_agree_to_tos(self): self.client.update_registration = mock.Mock() self.client.agree_to_tos(self.regr) @@ -149,9 +136,6 @@ class ClientTest(unittest.TestCase): self.response.status_code = http_client.CREATED self.response.headers['Location'] = self.authzr.uri self.response.json.return_value = self.authz.to_json() - self.response.links = { - 'next': {'url': self.authzr.new_cert_uri}, - } def test_request_challenges(self): self._prepare_response_for_request_challenges() @@ -160,10 +144,18 @@ class ClientTest(unittest.TestCase): self.directory.new_authz, messages.NewAuthorization(identifier=self.identifier)) + def test_request_challenges_deprecated_arg(self): + self._prepare_response_for_request_challenges() + self.client.request_challenges(self.identifier, new_authzr_uri="hi") + self.net.post.assert_called_once_with( + self.directory.new_authz, + messages.NewAuthorization(identifier=self.identifier)) + def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() - self.client.request_challenges(self.identifier, 'URI') - self.net.post.assert_called_once_with('URI', mock.ANY) + self.client.request_challenges(self.identifier) + self.net.post.assert_called_once_with( + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -171,12 +163,7 @@ class ClientTest(unittest.TestCase): identifier=self.identifier.update(value='foo')).to_json() self.assertRaises( errors.UnexpectedUpdate, self.client.request_challenges, - self.identifier, self.authzr.uri) - - def test_request_challenges_missing_next(self): - self.response.status_code = http_client.CREATED - self.assertRaises(errors.ClientError, self.client.request_challenges, - self.identifier) + self.identifier) def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() @@ -184,12 +171,6 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier), self.client.request_domain_challenges('example.com')) - def test_request_domain_challenges_custom_uri(self): - self.client.request_challenges = mock.MagicMock() - self.assertEqual( - self.client.request_challenges(self.identifier, 'URI'), - self.client.request_domain_challenges('example.com', 'URI')) - def test_answer_challenge(self): self.response.links['up'] = {'url': self.challr.authzr_uri} self.response.json.return_value = self.challr.body.to_json() diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py index d474f4aac..4baadda5e 100644 --- a/acme/acme/jose/json_util.py +++ b/acme/acme/jose/json_util.py @@ -267,7 +267,7 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): if missing: raise errors.DeserializationError( - 'The following field are required: {0}'.format( + 'The following fields are required: {0}'.format( ','.join(missing))) @classmethod diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 54cd25c94..f7670dd72 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -191,7 +191,7 @@ class Directory(jose.JSONDeSerializable): try: return self[name.replace('_', '-')] except KeyError as error: - raise AttributeError(str(error)) + raise AttributeError(str(error) + ': ' + name) def __getitem__(self, name): try: @@ -315,12 +315,12 @@ class RegistrationResource(ResourceWithURI): """Registration Resource. :ivar acme.messages.Registration body: - :ivar unicode new_authzr_uri: URI found in the 'next' ``Link`` header + :ivar unicode new_authzr_uri: Deprecated. Do not use. :ivar unicode terms_of_service: URL for the CA TOS. """ body = jose.Field('body', decoder=Registration.from_json) - new_authzr_uri = jose.Field('new_authzr_uri') + new_authzr_uri = jose.Field('new_authzr_uri', omitempty=True) terms_of_service = jose.Field('terms_of_service', omitempty=True) @@ -425,11 +425,11 @@ class AuthorizationResource(ResourceWithURI): """Authorization Resource. :ivar acme.messages.Authorization body: - :ivar unicode new_cert_uri: URI found in the 'next' ``Link`` header + :ivar unicode new_cert_uri: Deprecated. Do not use. """ body = jose.Field('body', decoder=Authorization.from_json) - new_cert_uri = jose.Field('new_cert_uri') + new_cert_uri = jose.Field('new_cert_uri', omitempty=True) @Directory.register diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index b3454f25b..e84c3e992 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -225,14 +225,12 @@ class RegistrationResourceTest(unittest.TestCase): from acme.messages import RegistrationResource self.regr = RegistrationResource( body=mock.sentinel.body, uri=mock.sentinel.uri, - new_authzr_uri=mock.sentinel.new_authzr_uri, terms_of_service=mock.sentinel.terms_of_service) def test_to_partial_json(self): self.assertEqual(self.regr.to_json(), { 'body': mock.sentinel.body, 'uri': mock.sentinel.uri, - 'new_authzr_uri': mock.sentinel.new_authzr_uri, 'terms_of_service': mock.sentinel.terms_of_service, }) @@ -346,9 +344,7 @@ class AuthorizationResourceTest(unittest.TestCase): from acme.messages import AuthorizationResource authzr = AuthorizationResource( uri=mock.sentinel.uri, - body=mock.sentinel.body, - new_cert_uri=mock.sentinel.new_cert_uri, - ) + body=mock.sentinel.body) self.assertTrue(isinstance(authzr, jose.JSONDeSerializable)) diff --git a/acme/examples/example_client.py b/acme/examples/example_client.py index 261b37603..1386491b1 100644 --- a/acme/examples/example_client.py +++ b/acme/examples/example_client.py @@ -32,8 +32,7 @@ acme.agree_to_tos(regr) logging.debug(regr) authzr = acme.request_challenges( - identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN), - new_authzr_uri=regr.new_authzr_uri) + identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN)) logging.debug(authzr) authzr, authzr_response = acme.poll(authzr) diff --git a/certbot/account.py b/certbot/account.py index 4b5f48411..1928b90d8 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -98,7 +98,7 @@ def report_new_account(config): class AccountMemoryStorage(interfaces.AccountStorage): - """In-memory account strage.""" + """In-memory account storage.""" def __init__(self, initial_accounts=None): self.accounts = initial_accounts if initial_accounts is not None else {} @@ -106,7 +106,8 @@ class AccountMemoryStorage(interfaces.AccountStorage): def find_all(self): return list(six.itervalues(self.accounts)) - def save(self, account): + def save(self, account, acme): + # pylint: disable=unused-argument if account.id in self.accounts: logger.debug("Overwriting account: %s", account.id) self.accounts[account.id] = account @@ -117,6 +118,16 @@ class AccountMemoryStorage(interfaces.AccountStorage): except KeyError: raise errors.AccountNotFound(account_id) +class RegistrationResourceWithNewAuthzrURI(messages.RegistrationResource): + """A backwards-compatible RegistrationResource with a new-authz URI. + + Hack: Certbot versions pre-0.11.1 expect to load + new_authzr_uri as part of the account. Because people + sometimes switch between old and new versions, we will + continue to write out this field for some time so older + clients don't crash in that scenario. + """ + new_authzr_uri = jose.Field('new_authzr_uri') class AccountFileStorage(interfaces.AccountStorage): """Accounts file storage. @@ -181,16 +192,16 @@ class AccountFileStorage(interfaces.AccountStorage): account_id, acc.id)) return acc - def save(self, account): - self._save(account, regr_only=False) + def save(self, account, acme): + self._save(account, acme, regr_only=False) - def save_regr(self, account): + def save_regr(self, account, acme): """Save the registration resource. :param Account account: account whose regr should be saved """ - self._save(account, regr_only=True) + self._save(account, acme, regr_only=True) def delete(self, account_id): """Delete registration info from disk @@ -204,13 +215,19 @@ class AccountFileStorage(interfaces.AccountStorage): "Account at %s does not exist" % account_dir_path) shutil.rmtree(account_dir_path) - def _save(self, account, regr_only): + def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: - regr_file.write(account.regr.json_dumps()) + regr = account.regr + with_uri = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(with_uri.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 6e9ab25a7..53346a77c 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -63,8 +63,7 @@ class AuthHandler(object): """ for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges( - domain, self.account.regr.new_authzr_uri) + self.authzr[domain] = self.acme.request_domain_challenges(domain) self._choose_challenges(domains) diff --git a/certbot/client.py b/certbot/client.py index a342c1bf3..0f6b35f09 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -138,7 +138,7 @@ def register(config, account_storage, tos_cb=None): acc = account.Account(regr, key) account.report_new_account(config) - account_storage.save(acc) + account_storage.save(acc, acme) eff.handle_subscription(config) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 2df2abfe8..a2767121b 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -32,7 +32,7 @@ class AccountStorage(object): raise NotImplementedError() @abc.abstractmethod - def save(self, account): # pragma: no cover + def save(self, account, client): # pragma: no cover """Save account. :raises .AccountStorageError: if account could not be saved diff --git a/certbot/main.py b/certbot/main.py index 118c0f958..1f247a7d6 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -466,7 +466,7 @@ def register(config, unused_plugins): # We rely on an exception to interrupt this process if it didn't work. acc.regr = acme_client.acme.update_registration(acc.regr.update( body=acc.regr.body.update(contact=('mailto:' + config.email,)))) - account_storage.save_regr(acc) + account_storage.save_regr(acc, acme_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ed591c98..7d335b09b 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -1,5 +1,6 @@ """Tests for certbot.account.""" import datetime +import json import os import shutil import stat @@ -90,10 +91,10 @@ class AccountMemoryStorageTest(unittest.TestCase): account = mock.Mock(id="x") self.assertEqual([], self.storage.find_all()) self.assertRaises(errors.AccountNotFound, self.storage.load, "x") - self.storage.save(account) + self.storage.save(account, None) self.assertEqual([account], self.storage.find_all()) self.assertEqual(account, self.storage.load("x")) - self.storage.save(account) + self.storage.save(account, None) self.assertEqual([account], self.storage.find_all()) @@ -108,10 +109,14 @@ class AccountFileStorageTest(unittest.TestCase): self.storage = AccountFileStorage(self.config) from certbot.account import Account + new_authzr_uri = "hi" self.acc = Account( regr=messages.RegistrationResource( - uri=None, new_authzr_uri=None, body=messages.Registration()), + uri=None, body=messages.Registration(), + new_authzr_uri=new_authzr_uri), key=KEY) + self.mock_client = mock.MagicMock() + self.mock_client.directory.new_authz = new_authzr_uri def tearDown(self): shutil.rmtree(self.tmp) @@ -120,7 +125,7 @@ class AccountFileStorageTest(unittest.TestCase): self.assertTrue(os.path.isdir(self.config.accounts_dir)) def test_save_and_restore(self): - self.storage.save(self.acc) + self.storage.save(self.acc, self.mock_client) account_path = os.path.join(self.config.accounts_dir, self.acc.id) self.assertTrue(os.path.exists(account_path)) for file_name in "regr.json", "meta.json", "private_key.json": @@ -130,10 +135,19 @@ class AccountFileStorageTest(unittest.TestCase): account_path, "private_key.json"))[stat.ST_MODE] & 0o777) in ("0400", "0o400")) # restore - self.assertEqual(self.acc, self.storage.load(self.acc.id)) + loaded = self.storage.load(self.acc.id) + self.assertEqual(self.acc, loaded) + + def test_save_and_restore_old_version(self): + """Saved regr should include a new_authzr_uri for older Certbots""" + self.storage.save(self.acc, self.mock_client) + path = os.path.join(self.config.accounts_dir, self.acc.id, "regr.json") + with open(path, "r") as f: + regr = json.load(f) + self.assertTrue("new_authzr_uri" in regr) def test_save_regr(self): - self.storage.save_regr(self.acc) + self.storage.save_regr(self.acc, self.mock_client) account_path = os.path.join(self.config.accounts_dir, self.acc.id) self.assertTrue(os.path.exists(account_path)) self.assertTrue(os.path.exists(os.path.join( @@ -143,7 +157,7 @@ class AccountFileStorageTest(unittest.TestCase): os.path.join(account_path, file_name))) def test_find_all(self): - self.storage.save(self.acc) + self.storage.save(self.acc, self.mock_client) self.assertEqual([self.acc], self.storage.find_all()) def test_find_all_none_empty_list(self): @@ -164,14 +178,14 @@ class AccountFileStorageTest(unittest.TestCase): self.assertRaises(errors.AccountNotFound, self.storage.load, "missing") def test_load_id_mismatch_raises_error(self): - self.storage.save(self.acc) + self.storage.save(self.acc, self.mock_client) shutil.move(os.path.join(self.config.accounts_dir, self.acc.id), os.path.join(self.config.accounts_dir, "x" + self.acc.id)) self.assertRaises(errors.AccountStorageError, self.storage.load, "x" + self.acc.id) def test_load_ioerror(self): - self.storage.save(self.acc) + self.storage.save(self.acc, self.mock_client) mock_open = mock.mock_open() mock_open.side_effect = IOError with mock.patch("six.moves.builtins.open", mock_open): @@ -183,10 +197,11 @@ class AccountFileStorageTest(unittest.TestCase): mock_open.side_effect = IOError # TODO: [None, None, IOError] with mock.patch("six.moves.builtins.open", mock_open): self.assertRaises( - errors.AccountStorageError, self.storage.save, self.acc) + errors.AccountStorageError, self.storage.save, + self.acc, self.mock_client) def test_delete(self): - self.storage.save(self.acc) + self.storage.save(self.acc, self.mock_client) self.storage.delete(self.acc.id) self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id) diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 5e6b190a7..f0549666a 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -96,6 +96,5 @@ def gen_authzr(authz_status, domain, challs, statuses, combos=True): # pylint: disable=star-args return messages.AuthorizationResource( uri="https://trusted.ca/new-authz-resource", - new_cert_uri="https://trusted.ca/new-cert", body=messages.Authorization(**authz_kwargs) ) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 046eb5ef1..9d22843db 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -309,7 +309,6 @@ class PollChallengesTest(unittest.TestCase): new_authzr = messages.AuthorizationResource( uri=authzr.uri, - new_cert_uri=authzr.new_cert_uri, body=messages.Authorization( identifier=authzr.body.identifier, challenges=new_challbs, @@ -437,7 +436,7 @@ def gen_auth_resp(chall_list): for chall in chall_list] -def gen_dom_authzr(domain, unused_new_authzr_uri, challs, combos=True): +def gen_dom_authzr(domain, challs, combos=True): """Generates new authzr for domains.""" return acme_util.gen_authzr( messages.STATUS_PENDING, domain, challs, diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index f6de33a92..f2a9b3d07 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -104,10 +104,10 @@ class ChooseAccountTest(unittest.TestCase): self.key = KEY self.acc1 = account.Account(messages.RegistrationResource( - uri=None, new_authzr_uri=None, body=messages.Registration.from_data( + uri=None, body=messages.Registration.from_data( email="email1@g.com")), self.key) self.acc2 = account.Account(messages.RegistrationResource( - uri=None, new_authzr_uri=None, body=messages.Registration.from_data( + uri=None, body=messages.Registration.from_data( email="email2@g.com", phone="phone")), self.key) @classmethod diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 3520eb063..170eceb48 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -382,6 +382,9 @@ class DetermineAccountTest(unittest.TestCase): self.config = configuration.NamespaceConfig(self.args) self.accs = [mock.MagicMock(id='x'), mock.MagicMock(id='y')] self.account_storage = account.AccountMemoryStorage() + # For use in saving accounts: fake out the new_authz URL. + self.mock_client = mock.MagicMock() + self.mock_client.directory.new_authz = "hi" def _call(self): # pylint: disable=protected-access @@ -391,14 +394,14 @@ class DetermineAccountTest(unittest.TestCase): return _determine_account(self.config) def test_args_account_set(self): - self.account_storage.save(self.accs[1]) + self.account_storage.save(self.accs[1], self.mock_client) self.config.account = self.accs[1].id self.assertEqual((self.accs[1], None), self._call()) self.assertEqual(self.accs[1].id, self.config.account) self.assertTrue(self.config.email is None) def test_single_account(self): - self.account_storage.save(self.accs[0]) + self.account_storage.save(self.accs[0], self.mock_client) self.assertEqual((self.accs[0], None), self._call()) self.assertEqual(self.accs[0].id, self.config.account) self.assertTrue(self.config.email is None) @@ -406,7 +409,7 @@ class DetermineAccountTest(unittest.TestCase): @mock.patch('certbot.client.display_ops.choose_account') def test_multiple_accounts(self, mock_choose_accounts): for acc in self.accs: - self.account_storage.save(acc) + self.account_storage.save(acc, self.mock_client) mock_choose_accounts.return_value = self.accs[1] self.assertEqual((self.accs[1], None), self._call()) self.assertEqual( From 5758b1687d5cb79beef8b51d90c655657985f6a4 Mon Sep 17 00:00:00 2001 From: kernelpanek Date: Wed, 15 Mar 2017 00:25:26 -0600 Subject: [PATCH 046/128] Fixes issue when parsing an Nginx configuration file containing multiline quoted strings --- certbot-nginx/certbot_nginx/nginxparser.py | 6 +++--- .../certbot_nginx/tests/nginxparser_test.py | 15 +++++++++++++++ .../testdata/etc_nginx/multiline_quotes.conf | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index f6437c589..908a6d6cf 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -6,7 +6,7 @@ import string from pyparsing import ( Literal, White, Word, alphanums, CharsNotIn, Combine, Forward, Group, - Optional, OneOrMore, Regex, ZeroOrMore) + Optional, OneOrMore, QuotedString, Regex, ZeroOrMore) from pyparsing import stringEnd from pyparsing import restOfLine @@ -29,8 +29,8 @@ class RawNginxParser(object): # any chars in single or double quotes # All of these COULD be upgraded to something like # https://stackoverflow.com/a/16130746 - dquoted = Regex(r'(\".*\")') - squoted = Regex(r"(\'.*\')") + dquoted = QuotedString('"', multiline=True) + squoted = QuotedString("'", multiline=True) nonspecial = Regex(r"[^\{\};,]") varsub = Regex(r"(\$\{\w+\})") # nonspecial nibbles one character at a time, but the other objects take diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index e83b414cf..650090cb2 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -109,6 +109,21 @@ class TestRawNginxParser(unittest.TestCase): ['blah', '"hello;world"'], ['try_files', '$uri @rewrites']]]]]]) + def test_parse_from_file3(self): + with open(util.get_data_filename('multiline_quotes.conf')) as handle: + parsed = util.filter_comments(load(handle)) + self.assertEqual( + parsed, + [[['http'], + [[['server'], + [['listen', '*:443'], + [['location', '/'], + [['body_filter_by_lua', + 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000)\n' + ' if ngx.arg[2] then\n' + ' ngx.var.resp_body = ngx.ctx.buffered\n' + ' end']]]]]]]]) + def test_abort_on_parse_failure(self): with open(util.get_data_filename('broken.conf')) as handle: self.assertRaises(ParseException, load, handle) diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf new file mode 100644 index 000000000..74cd84bcd --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf @@ -0,0 +1,16 @@ +# Test nginx configuration file with multiline quoted strings. +# Good example of usage for multilined quoted values is when +# using Openresty's Lua directives and you wish to keep the +# inline Lua code readable. +http { + server { + listen *:443; # because there should be no other port open. + + location / { + body_filter_by_lua 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000) + if ngx.arg[2] then + ngx.var.resp_body = ngx.ctx.buffered + end'; + } + } +} From e715b49dd2b826460f7871776e85f46bc2d8f1d2 Mon Sep 17 00:00:00 2001 From: kernelpanek Date: Wed, 15 Mar 2017 01:26:16 -0600 Subject: [PATCH 047/128] Don't unquote the results of the parse --- certbot-nginx/certbot_nginx/nginxparser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 908a6d6cf..67ac7c3a1 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -29,8 +29,8 @@ class RawNginxParser(object): # any chars in single or double quotes # All of these COULD be upgraded to something like # https://stackoverflow.com/a/16130746 - dquoted = QuotedString('"', multiline=True) - squoted = QuotedString("'", multiline=True) + dquoted = QuotedString('"', multiline=True, unquoteResults=False) + squoted = QuotedString("'", multiline=True, unquoteResults=False) nonspecial = Regex(r"[^\{\};,]") varsub = Regex(r"(\$\{\w+\})") # nonspecial nibbles one character at a time, but the other objects take From f791af5afea61359da4576d006f3557236e69966 Mon Sep 17 00:00:00 2001 From: Richard Panek Date: Wed, 15 Mar 2017 02:13:09 -0600 Subject: [PATCH 048/128] New switch for QuotedStrings allows retainer of quotes but my test fails --- certbot-nginx/certbot_nginx/tests/nginxparser_test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index 650090cb2..44bebb373 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -119,10 +119,13 @@ class TestRawNginxParser(unittest.TestCase): [['listen', '*:443'], [['location', '/'], [['body_filter_by_lua', - 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000)\n' - ' if ngx.arg[2] then\n' - ' ngx.var.resp_body = ngx.ctx.buffered\n' - ' end']]]]]]]]) + '\'ngx.ctx.buffered = (ngx.ctx.buffered or "")' + ' .. string.sub(ngx.arg[1], 1, 1000)\n' + ' ' + 'if ngx.arg[2] then\n' + ' ' + 'ngx.var.resp_body = ngx.ctx.buffered\n' + ' end\'']]]]]]]]) def test_abort_on_parse_failure(self): with open(util.get_data_filename('broken.conf')) as handle: From 5fa20805586213262d5b5848747bfc5ec8241139 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 15 Mar 2017 17:05:52 -0700 Subject: [PATCH 049/128] If we fail to reload Nginx, write to temporary files instead of piping output (#4333) Due to issues with piping and Nginx on Arch. --- certbot-nginx/certbot_nginx/configurator.py | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 7348def2f..46ce18ab2 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -5,6 +5,7 @@ import re import shutil import socket import subprocess +import tempfile import time import OpenSSL @@ -829,22 +830,22 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): """ try: - proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() + proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"]) + proc.communicate() if proc.returncode != 0: # Maybe Nginx isn't running - nginx_proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = nginx_proc.communicate() - - if nginx_proc.returncode != 0: - # Enter recovery routine... - raise errors.MisconfigurationError( - "nginx restart failed:\n%s\n%s" % (stdout, stderr)) + # Write to temporary files instead of piping because of communication issues on Arch + # https://github.com/certbot/certbot/issues/4324 + with tempfile.TemporaryFile() as out: + with tempfile.TemporaryFile() as err: + nginx_proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf], + stdout=out, stderr=err) + nginx_proc.communicate() + if nginx_proc.returncode != 0: + # Enter recovery routine... + raise errors.MisconfigurationError( + "nginx restart failed:\n%s\n%s" % (out.read(), err.read())) except (OSError, ValueError): raise errors.MisconfigurationError("nginx restart failed") From edcfc49303d4ddd2e69ba6ac6933af01d00e332c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 17 Mar 2017 13:02:41 -0700 Subject: [PATCH 050/128] Use setattr in NamespaceConfig (#4362) * set setattr in NamespaceConfig * remove unnecessary uses of .namespace * add simple test to ensure it works --- certbot/client.py | 2 +- certbot/configuration.py | 5 ++++- certbot/main.py | 8 ++++---- certbot/plugins/selection.py | 5 ++--- certbot/renewal.py | 10 +++++----- certbot/tests/configuration_test.py | 6 ++++++ certbot/tests/renewal_test.py | 8 ++++---- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 0f6b35f09..8933ef27e 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -166,7 +166,7 @@ def perform_registration(acme, config): "registration again." % config.email) raise errors.Error(msg) else: - config.namespace.email = display_ops.get_email(invalid=True) + config.email = display_ops.get_email(invalid=True) return perform_registration(acme, config) else: raise diff --git a/certbot/configuration.py b/certbot/configuration.py index d25378922..30c6f0437 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -42,7 +42,7 @@ class NamespaceConfig(object): """ def __init__(self, namespace): - self.namespace = namespace + object.__setattr__(self, 'namespace', namespace) self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) self.namespace.work_dir = os.path.abspath(self.namespace.work_dir) @@ -54,6 +54,9 @@ class NamespaceConfig(object): def __getattr__(self, name): return getattr(self.namespace, name) + def __setattr__(self, name, value): + setattr(self.namespace, name, value) + @property def server_path(self): """File path based on ``server``.""" diff --git a/certbot/main.py b/certbot/main.py index 1f247a7d6..b0689faa2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -359,7 +359,7 @@ def _determine_account(config): acc = accounts[0] else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: - config.namespace.email = display_ops.get_email() + config.email = display_ops.get_email() def _tos_cb(regr): if config.tos: @@ -382,7 +382,7 @@ def _determine_account(config): raise errors.Error( "Unable to register an account with ACME server") - config.namespace.account = acc.id + config.account = acc.id return acc, acme @@ -459,7 +459,7 @@ def register(config, unused_plugins): return ("--register-unsafely-without-email provided, however, a " "new e-mail address must\ncurrently be provided when " "updating a registration.") - config.namespace.email = display_ops.get_email(optional=False) + config.email = display_ops.get_email(optional=False) acc, acme = _determine_account(config) acme_client = client.Client(config, acc, None, None, acme=acme) @@ -565,7 +565,7 @@ def certificates(config, unused_plugins): def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction - config.namespace.installer = config.namespace.authenticator = "None" + config.installer = config.authenticator = "None" if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 81387c435..d138001e6 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -137,9 +137,8 @@ noninstaller_plugins = ["webroot", "manual", "standalone"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." - cn = config.namespace - cn.authenticator = plugins.find_init(auth).name if auth else "None" - cn.installer = plugins.find_init(inst).name if inst else "None" + config.authenticator = plugins.find_init(auth).name if auth else "None" + config.installer = plugins.find_init(inst).name if inst else "None" def choose_configurator_plugins(config, plugins, verb): diff --git a/certbot/renewal.py b/certbot/renewal.py index a0cc872a0..6eb171763 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -103,13 +103,13 @@ def _restore_webroot_config(config, renewalparams): """ if "webroot_map" in renewalparams: if not cli.set_by_cli("webroot_map"): - config.namespace.webroot_map = renewalparams["webroot_map"] + config.webroot_map = renewalparams["webroot_map"] elif "webroot_path" in renewalparams: logger.debug("Ancient renewal conf file without webroot-map, restoring webroot-path") wp = renewalparams["webroot_path"] if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] - config.namespace.webroot_path = wp + config.webroot_path = wp def _restore_plugin_configs(config, renewalparams): @@ -148,10 +148,10 @@ def _restore_plugin_configs(config, renewalparams): if config_value in ("None", "True", "False"): # bool("False") == True # pylint: disable=eval-used - setattr(config.namespace, config_item, eval(config_value)) + setattr(config, config_item, eval(config_value)) else: cast = cli.argparse_type(config_item) - setattr(config.namespace, config_item, cast(config_value)) + setattr(config, config_item, cast(config_value)) def restore_required_config_elements(config, renewalparams): @@ -172,7 +172,7 @@ def restore_required_config_elements(config, renewalparams): for item_name, restore_func in required_items: if item_name in renewalparams and not cli.set_by_cli(item_name): value = restore_func(item_name, renewalparams[item_name]) - setattr(config.namespace, item_name, value) + setattr(config, item_name, value) def _restore_pref_challs(unused_name, value): diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 3a2f7d291..66a07dddd 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -120,6 +120,12 @@ class NamespaceConfigTest(unittest.TestCase): self.assertTrue(os.path.isabs(config.live_dir)) self.assertTrue(os.path.isabs(config.renewal_configs_dir)) + def test_get_and_set_attr(self): + self.config.foo = 42 + self.assertEqual(self.config.namespace.foo, 42) + self.config.namespace.bar = 1337 + self.assertEqual(self.config.bar, 1337) + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index cd53aa91c..d97e05783 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -52,7 +52,7 @@ class RestoreRequiredConfigElementsTest(unittest.TestCase): def test_allow_subset_of_names_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'allow_subset_of_names': 'True'}) - self.assertTrue(self.config.namespace.allow_subset_of_names is True) + self.assertTrue(self.config.allow_subset_of_names is True) @mock.patch('certbot.renewal.cli.set_by_cli') def test_allow_subset_of_names_failure(self, mock_set_by_cli): @@ -68,7 +68,7 @@ class RestoreRequiredConfigElementsTest(unittest.TestCase): self._call(self.config, renewalparams) expected = [challenges.TLSSNI01.typ, challenges.HTTP01.typ, challenges.DNS01.typ] - self.assertEqual(self.config.namespace.pref_challs, expected) + self.assertEqual(self.config.pref_challs, expected) @mock.patch('certbot.renewal.cli.set_by_cli') def test_pref_challs_str(self, mock_set_by_cli): @@ -76,7 +76,7 @@ class RestoreRequiredConfigElementsTest(unittest.TestCase): renewalparams = {'pref_challs': 'dns'} self._call(self.config, renewalparams) expected = [challenges.DNS01.typ] - self.assertEqual(self.config.namespace.pref_challs, expected) + self.assertEqual(self.config.pref_challs, expected) @mock.patch('certbot.renewal.cli.set_by_cli') def test_pref_challs_failure(self, mock_set_by_cli): @@ -88,7 +88,7 @@ class RestoreRequiredConfigElementsTest(unittest.TestCase): def test_must_staple_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'must_staple': 'True'}) - self.assertTrue(self.config.namespace.must_staple is True) + self.assertTrue(self.config.must_staple is True) @mock.patch('certbot.renewal.cli.set_by_cli') def test_must_staple_failure(self, mock_set_by_cli): From 4cad594b4ba81734c3db3343cc418440de99f06e Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 18 Mar 2017 04:10:02 +0800 Subject: [PATCH 051/128] Python 3 compatibility for all tests (#4358) --- certbot-nginx/certbot_nginx/configurator.py | 3 +- certbot-nginx/certbot_nginx/obj.py | 17 ++++++--- certbot-nginx/certbot_nginx/parser.py | 2 +- .../certbot_nginx/tests/configurator_test.py | 7 ++-- .../certbot_nginx/tests/nginxparser_test.py | 4 +-- certbot-nginx/certbot_nginx/tests/obj_test.py | 2 +- .../certbot_nginx/tests/tls_sni_01_test.py | 19 +++++----- certbot-nginx/certbot_nginx/tests/util.py | 7 ++-- certbot-nginx/certbot_nginx/tls_sni_01.py | 7 ++-- letshelp-certbot/letshelp_certbot/apache.py | 6 ++-- .../letshelp_certbot/apache_test.py | 4 ++- tox.ini | 36 ------------------- 12 files changed, 49 insertions(+), 65 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 46ce18ab2..e62194d4f 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -9,6 +9,7 @@ import tempfile import time import OpenSSL +import six import zope.interface from acme import challenges @@ -263,7 +264,7 @@ class NginxConfigurator(common.Plugin): """ if not matches: return None - elif matches[0]['rank'] in xrange(2, 6): + elif matches[0]['rank'] in six.moves.range(2, 6): # Wildcard match - need to find the longest one rank = matches[0]['rank'] wildcards = [x for x in matches if x['rank'] == rank] diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 29fa976f3..849cefe1f 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -1,6 +1,8 @@ """Module contains classes used by the Nginx Configurator.""" import re +import six + from certbot.plugins import common REDIRECT_DIRECTIVES = ['return', 'rewrite'] @@ -97,6 +99,11 @@ class Addr(common.Addr): def __repr__(self): return "Addr(" + self.__str__() + ")" + def __hash__(self): + # Python 3 requires explicit overridden for __hash__ + # See certbot-apache/certbot_apache/obj.py for more information + return super(Addr, self).__hash__() + def super_eq(self, other): """Check ip/port equality, with IPv6 support. """ @@ -147,13 +154,15 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods self.path = path def __str__(self): - addr_str = ", ".join(str(addr) for addr in self.addrs) + addr_str = ", ".join(str(addr) for addr in sorted(self.addrs, key=str)) + # names might be a set, and it has different representations in Python + # 2 and 3. Force it to be a list here for consistent outputs return ("file: %s\n" "addrs: %s\n" "names: %s\n" "ssl: %s\n" "enabled: %s" % (self.filep, addr_str, - self.names, self.ssl, self.enabled)) + list(self.names), self.ssl, self.enabled)) def __repr__(self): return "VirtualHost(" + self.__str__().replace("\n", ", ") + ")\n" @@ -161,7 +170,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and - list(self.addrs) == list(other.addrs) and + sorted(self.addrs, key=str) == sorted(other.addrs, key=str) and self.names == other.names and self.ssl == other.ssl and self.enabled == other.enabled and @@ -181,7 +190,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in xrange(0, len(self.raw) - len(test)): + for i in six.moves.range(0, len(self.raw) - len(test)): if self.raw[i:i + len(test)] == test: return True return False diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index eddc7b9b0..fd4ea4f11 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -342,7 +342,7 @@ class NginxParser(object): vhost.names = parsed_server['names'] vhost.raw = new_server except errors.MisconfigurationError as err: - raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message)) + raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) def _do_for_subarray(entry, condition, func, path=None): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index cc36aa0de..d491d2a15 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -27,12 +27,13 @@ class NginxConfiguratorTest(util.NginxTest): super(NginxConfiguratorTest, self).setUp() self.config = util.get_nginx_configurator( - self.config_path, self.config_dir, self.work_dir) + self.config_path, self.config_dir, self.work_dir, self.logs_dir) def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + shutil.rmtree(self.logs_dir) @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): @@ -261,13 +262,13 @@ class NginxConfiguratorTest(util.NginxTest): # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"), + chall=challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), uri="https://ca.org/chall0_uri", status=messages.Status("pending"), ), domain="localhost", account_key=self.rsa512jwk) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token="m8TdO1qik4JVFtgPPurJmg"), + chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", status=messages.Status("pending"), ), domain="example.com", account_key=self.rsa512jwk) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index e83b414cf..cf4bbde39 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -128,7 +128,7 @@ class TestRawNginxParser(unittest.TestCase): [['root', ' ', 'html'], ['index', ' ', 'index.html index.htm']]]]])) - with tempfile.TemporaryFile() as f: + with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) f.seek(0) parsed_new = load(f) @@ -138,7 +138,7 @@ class TestRawNginxParser(unittest.TestCase): with open(util.get_data_filename('minimalistic_comments.conf')) as handle: parsed = load(handle) - with tempfile.TemporaryFile() as f: + with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) f.seek(0) parsed_new = load(f) diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index b0a2d5ad8..069f860d6 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -158,7 +158,7 @@ class VirtualHostTest(unittest.TestCase): def test_str(self): stringified = '\n'.join(['file: filep', 'addrs: localhost', - "names: set(['localhost'])", 'ssl: False', + "names: ['localhost']", 'ssl: False', 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index e7dacb400..7a2de44a2 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -3,6 +3,7 @@ import unittest import shutil import mock +import six from acme import challenges @@ -23,25 +24,25 @@ class TlsSniPerformTest(util.NginxTest): achalls = [ achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"), "pending"), + challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), domain="www.example.com", account_key=account_key), achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="\xba\xa9\xda? Date: Fri, 17 Mar 2017 13:13:45 -0700 Subject: [PATCH 052/128] Improve plugin-writing docs. (#4329) Move "Writing your own plugin" under Code components and layout, with the other plugin docs. Include instructions on how to install a plugin into a virtualenv and how to check for its presence. Document that users can install third-party plugins systemwide, but not with certbot-auto. Remove obsolete information from Authenticators section and make the section more informative. Remove IDisplay sub-section since it repeats information in the main "Plugin architecture" section. --- docs/contributing.rst | 71 ++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 5cdf86147..9c7f0636f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -145,13 +145,15 @@ different webservers, other TLS servers, and operating systems. The interfaces available for plugins to implement are defined in `interfaces.py`_ and `plugins/common.py`_. -The most common kind of plugin is a "Configurator", which is likely to -implement the `~certbot.interfaces.IAuthenticator` and -`~certbot.interfaces.IInstaller` interfaces (though some -Configurators may implement just one of those). +The main two plugin interfaces are `~certbot.interfaces.IAuthenticator`, which +implements various ways of proving domain control to a certificate authority, +and `~certbot.interfaces.IInstaller`, which configures a server to use a +certificate once it is issued. Some plugins, like the built-in Apache and Nginx +plugins, implement both interfaces and perform both tasks. Others, like the +built-in Standalone authenticator, implement just one interface. There are also `~certbot.interfaces.IDisplay` plugins, -which implement bindings to alternative UI libraries. +which can change how prompts are displayed to a user. .. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py .. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 @@ -160,27 +162,20 @@ which implement bindings to alternative UI libraries. Authenticators -------------- -Authenticators are plugins designed to prove that this client deserves a -certificate for some domain name by solving challenges received from -the ACME server. From the protocol, there are essentially two -different types of challenges. Challenges that must be solved by -individual plugins in order to satisfy domain validation (subclasses -of `~.DVChallenge`, i.e. `~.challenges.TLSSNI01`, -`~.challenges.HTTP01`, `~.challenges.DNS`) and continuity specific -challenges (subclasses of `~.ContinuityChallenge`, -i.e. `~.challenges.RecoveryToken`, `~.challenges.RecoveryContact`, -`~.challenges.ProofOfPossession`). Continuity challenges are -always handled by the `~.ContinuityAuthenticator`, while plugins are -expected to handle `~.DVChallenge` types. -Right now, we have two authenticator plugins, the `~.ApacheConfigurator` -and the `~.StandaloneAuthenticator`. The Standalone and Apache -authenticators only solve the `~.challenges.TLSSNI01` challenge currently. -(You can set which challenges your authenticator can handle through the -:meth:`~.IAuthenticator.get_chall_pref`. +Authenticators are plugins that prove control of a domain name by solving a +challenge provided by the ACME server. ACME currently defines three types of +challenges: HTTP, TLS-SNI, and DNS, represented by classes in `acme.challenges`. +An authenticator plugin should implement support for at least one challenge type. -(FYI: We also have a partial implementation for a `~.DNSAuthenticator` -in a separate branch). +An Authenticator indicates which challenges it supports by implementing +get_chall_pref(domain) to return a sorted list of challenge types in preference +order. +An Authenticator must also implement `perform(achalls)`, which "performs" a list +of challenges by, for instance, provisioning a file on an HTTP server, or +setting a TXT record in DNS. Once all challenges have succeeded or failed, +Certbot will call the plugin's `cleanup(achalls)` method to remove any files or +DNS records that were needed only during authentication. Installer --------- @@ -218,16 +213,10 @@ Augeas may still find the `~.Reverter` class helpful in handling configuration checkpoints and rollback. -Display -~~~~~~~ - -We currently only offer a "text" mode for displays. Display plugins -implement the `~certbot.interfaces.IDisplay` interface. - .. _dev-plugin: Writing your own plugin -======================= +~~~~~~~~~~~~~~~~~~~~~~~ Certbot client supports dynamic discovery of plugins through the `setuptools entry points`_. This way you can, for example, create a @@ -236,6 +225,26 @@ the `~certbot.interfaces.IInstaller` without having to merge it with the core upstream source code. An example is provided in ``examples/plugins/`` directory. +While developing, you can install your plugin into a Certbot development +virtualenv like this: + +.. code-block:: shell + . venv/bin/activate + . tests/integration/_common.sh + pip install -e examples/plugins/ + certbot_test plugins + +Your plugin should show up in the output of the last command. If not, +it was not installed properly. + +Once you've finished your plugin and published it, you can have your +users install it system-wide with `pip install`. Note that this will +only work for users who have Certbot installed from OS packages or via +pip. Users who run `certbot-auto` are currently unable to use third-party +plugins. It's technically possible to install third-party plugins into +the virtualenv used by `certbot-auto`, but they will be wiped away when +`certbot-auto` upgrades. + .. warning:: Please be aware though that as this client is still in a developer-preview stage, the API may undergo a few changes. If you believe the plugin will be beneficial to the community, please From b23a1377e04a7f53269bb36e008a3cf495d2853f Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Mar 2017 13:17:08 -0700 Subject: [PATCH 053/128] Clarify documentation for low-memory machines. (#4305) * Clarify documentation for low-memory machines. * Restore py26/py27 requirement. --- README.rst | 14 +------------- docs/install.rst | 14 +++++++++----- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index ab12562df..e44a61021 100644 --- a/README.rst +++ b/README.rst @@ -129,19 +129,7 @@ email to client-dev+subscribe@letsencrypt.org) System Requirements =================== -The Let's Encrypt Client presently only runs on Unix-ish OSes that include -Python 2.6 or 2.7; Python 3.x support will hopefully be added in the future. The -client requires root access in order to write to ``/etc/letsencrypt``, -``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 -(if you use the ``standalone`` plugin) and to read and modify webserver -configurations (if you use the ``apache`` or ``nginx`` plugins). If none of -these apply to you, it is theoretically possible to run without root privileges, -but for most users who want to avoid running an ACME client as root, either -`letsencrypt-nosudo `_ or -`simp_le `_ are more appropriate choices. - -The Apache plugin currently requires a Debian-based OS with augeas version -1.0; this includes Ubuntu 12.04+ and Debian 7+. +See https://certbot.eff.org/docs/install.html#system-requirements. .. Do not modify this comment unless you know what you're doing. tag:intro-end diff --git a/docs/install.rst b/docs/install.rst index 548c8f2a3..e1ec06f16 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,9 +22,8 @@ your system. System Requirements =================== -The Let's Encrypt Client presently only runs on Unix-ish OSes that include -Python 2.6 or 2.7; Python 3.x support will hopefully be added in the future. The -client requires root access in order to write to ``/etc/letsencrypt``, +Certbot currently requires Python 2.6 or 2.7. By default, it requires root +access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver configurations (if you use the ``apache`` or ``nginx`` plugins). If none of @@ -33,11 +32,16 @@ but for most users who want to avoid running an ACME client as root, either `letsencrypt-nosudo `_ or `simp_le `_ are more appropriate choices. -The Apache plugin currently requires OS with augeas version 1.0; currently `it +The Apache plugin currently requires an OS with augeas version 1.0; currently `it supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. +Installing with ``certbot-auto`` requires 512MB of RAM in order to build some +of the dependencies. Installing from pre-built OS packages avoids this +requirement. You can also temporarily set a swap file. See "Problems with +Python virtual environment" below for details. + Alternate installation methods ================================ @@ -76,7 +80,7 @@ For full command line help, you can type:: Problems with Python virtual environment ---------------------------------------- -On a low memory system such as VPS with only 256MB of RAM, the required dependencies of Certbot will failed to build. +On a low memory system such as VPS with less than 512MB of RAM, the required dependencies of Certbot will failed to build. This can be identified if the pip outputs contains something like ``internal compiler error: Killed (program cc1)``. You can workaround this restriction by creating a temporary swapfile:: From fd789b4e4b9f58970abf10fed399e4e786ba3bb4 Mon Sep 17 00:00:00 2001 From: Piotr Kasprzyk Date: Fri, 17 Mar 2017 22:11:52 +0100 Subject: [PATCH 054/128] Fix choose, remove spaces (#4364) --- docs/using.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 7c17796e7..549a3479c 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -175,15 +175,15 @@ the UI, you can use the plugin to obtain a cert by specifying to copy and paste commands into another terminal session, which may be on a different computer. -The manual plugin can use either the ``http`` or the ``dns`` challenge. You -can use the ``--preferred-challenges`` option to chose the challenge of your +The manual plugin can use either the ``http`` or the ``dns`` challenge. You +can use the ``--preferred-challenges`` option to choose the challenge of your preference. -The ``http`` challenge will ask you to place a file with a specific name and -specific content in the ``/.well-known/acme-challenge/`` directory directly -in the top-level directory (“web root”) containing the files served by your +The ``http`` challenge will ask you to place a file with a specific name and +specific content in the ``/.well-known/acme-challenge/`` directory directly +in the top-level directory (“web root”) containing the files served by your webserver. In essence it's the same as the webroot_ plugin, but not automated. -When using the ``dns`` plugin, ``certbot`` will ask you to place a TXT DNS -record with specific contents under the domain name consisting of the hostname +When using the ``dns`` plugin, ``certbot`` will ask you to place a TXT DNS +record with specific contents under the domain name consisting of the hostname for which you want a certificate issued, prepended by ``_acme-challenge``. For example, for the domain ``example.com``, a zone file entry would look like: From b81f0296140b4b04bd5e0cda640fff2768596d74 Mon Sep 17 00:00:00 2001 From: Osiris Inferi Date: Sat, 18 Mar 2017 00:51:59 +0100 Subject: [PATCH 055/128] Add Gentoo to list of official packages --- docs/packaging.rst | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index 1a1b83f50..7e4781f65 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -44,6 +44,7 @@ Arch ---- From our official releases: + - https://www.archlinux.org/packages/community/any/python2-acme - https://www.archlinux.org/packages/community/any/certbot - https://www.archlinux.org/packages/community/any/certbot-apache @@ -55,9 +56,9 @@ From ``master``: https://aur.archlinux.org/packages/certbot-git Debian (and its derivatives, including Ubuntu) ------ -https://packages.debian.org/sid/certbot -https://packages.debian.org/sid/python-certbot -https://packages.debian.org/sid/python-certbot-apache +- https://packages.debian.org/sid/certbot +- https://packages.debian.org/sid/python-certbot +- https://packages.debian.org/sid/python-certbot-apache Fedora ------ @@ -71,7 +72,17 @@ In Fedora 23+. FreeBSD ------- -https://svnweb.freebsd.org/ports/head/security/py-certbot/ +- https://svnweb.freebsd.org/ports/head/security/py-certbot/ + +Gentoo +------ + +Currently, all ``certbot`` related packages are in the testing branch: + +- https://packages.gentoo.org/packages/app-crypt/certbot +- https://packages.gentoo.org/packages/app-crypt/certbot-apache +- https://packages.gentoo.org/packages/app-crypt/certbot-nginx +- https://packages.gentoo.org/packages/app-crypt/acme GNU Guix -------- From c439057efab57d4b02856839e90d1404eefcb91c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sat, 18 Mar 2017 13:25:02 -0700 Subject: [PATCH 056/128] install python3-dev for python3 tests in docker (#4381) --- Dockerfile-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 2a89b2ff5..c09e04ac3 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -20,10 +20,10 @@ WORKDIR /opt/certbot/src # If doesn't exist, it is created along with all missing # directories in its path. -# TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ + apt-get install python3-dev -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ From 6f979a48084674dc773098709f6f43230277372a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sat, 18 Mar 2017 13:40:01 -0700 Subject: [PATCH 057/128] upgrade pip and setuptools before installing packages (#4378) --- Dockerfile-dev | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index c09e04ac3..607aa3441 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -51,6 +51,8 @@ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tests /opt/certbot/src/tests/ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ + /opt/certbot/venv/bin/pip install -U pip && \ + /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install \ -e /opt/certbot/src/acme \ -e /opt/certbot/src \ @@ -58,8 +60,7 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ -e /opt/certbot/src/certbot-nginx \ -e /opt/certbot/src/letshelp-certbot \ -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] && \ - /opt/certbot/venv/bin/pip install -U setuptools + -e /opt/certbot/src[dev,docs] # install in editable mode (-e) to save space: it's not possible to # "rm -rf /opt/certbot/src" (it's stays in the underlaying image); From e034b50363a75d61d1a7e7add2b8aca52684acd1 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Sat, 18 Mar 2017 13:42:54 -0700 Subject: [PATCH 058/128] Don't save keys/csr on dry run (#4380) * Don't save keys/csr on dry run (#2495) * Replace assertIsNone for py26 * Fix config defaults for compat tests --- certbot-nginx/certbot_nginx/tests/util.py | 1 + certbot/constants.py | 1 + certbot/crypto_util.py | 30 ++++++++++-------- certbot/tests/crypto_util_test.py | 37 +++++++++++++++++++++-- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 259dc1f10..4ab95374e 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -70,6 +70,7 @@ def get_nginx_configurator( in_progress_dir=os.path.join(backups, "IN_PROGRESS"), server="https://acme-server.org:443/new", tls_sni_01_port=5001, + dry_run=False, ), name="nginx", version=version) diff --git a/certbot/constants.py b/certbot/constants.py index b286ca26a..c85992fc1 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -18,6 +18,7 @@ CLI_DEFAULTS = dict( os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), ], + dry_run=False, verbose_count=-int(logging.INFO / 10), server="https://acme-v01.api.letsencrypt.org/directory", rsa_key_size=2048, diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 65e3de345..1ad76d503 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -53,12 +53,15 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): # Save file util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), config.strict_permissions) - key_f, key_path = util.unique_file( - os.path.join(key_dir, keyname), 0o600, "wb") - with key_f: - key_f.write(key_pem) - - logger.info("Generating key (%d bits): %s", key_size, key_path) + if config.dry_run: + key_path = None + logger.info("Generating key (%d bits), not saving to file", key_size) + else: + key_f, key_path = util.unique_file( + os.path.join(key_dir, keyname), 0o600, "wb") + with key_f: + key_f.write(key_pem) + logger.info("Generating key (%d bits): %s", key_size, key_path) return util.Key(key_path, key_pem) @@ -85,12 +88,15 @@ def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): # Save CSR util.make_or_verify_dir(path, 0o755, os.geteuid(), config.strict_permissions) - csr_f, csr_filename = util.unique_file( - os.path.join(path, csrname), 0o644, "wb") - csr_f.write(csr_pem) - csr_f.close() - - logger.info("Creating CSR: %s", csr_filename) + if config.dry_run: + csr_filename = None + logger.info("Creating CSR: not saving to file") + else: + csr_f, csr_filename = util.unique_file( + os.path.join(path, csrname), 0o644, "wb") + with csr_f: + csr_f.write(csr_pem) + logger.info("Creating CSR: %s", csr_filename) return util.CSR(csr_filename, csr_der, "der") diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 946e772c1..a832d0494 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -1,5 +1,6 @@ """Tests for certbot.crypto_util.""" import logging +import os import shutil import tempfile import unittest @@ -26,7 +27,8 @@ class InitSaveKeyTest(unittest.TestCase): def setUp(self): logging.disable(logging.CRITICAL) zope.component.provideUtility( - mock.Mock(strict_permissions=True), interfaces.IConfig) + mock.Mock(strict_permissions=True, dry_run=False), + interfaces.IConfig) self.key_dir = tempfile.mkdtemp('key_dir') def tearDown(self): @@ -44,6 +46,17 @@ class InitSaveKeyTest(unittest.TestCase): key = self._call(1024, self.key_dir) self.assertEqual(key.pem, b'key_pem') self.assertTrue('key-certbot.pem' in key.file) + self.assertTrue(os.path.exists(os.path.join(self.key_dir, key.file))) + + @mock.patch('certbot.crypto_util.make_key') + def test_success_dry_run(self, mock_make): + zope.component.provideUtility( + mock.Mock(strict_permissions=True, dry_run=True), + interfaces.IConfig) + mock_make.return_value = b'key_pem' + key = self._call(1024, self.key_dir) + self.assertEqual(key.pem, b'key_pem') + self.assertTrue(key.file is None) @mock.patch('certbot.crypto_util.make_key') def test_key_failure(self, mock_make): @@ -56,7 +69,8 @@ class InitSaveCSRTest(unittest.TestCase): def setUp(self): zope.component.provideUtility( - mock.Mock(strict_permissions=True), interfaces.IConfig) + mock.Mock(strict_permissions=True, dry_run=False), + interfaces.IConfig) self.csr_dir = tempfile.mkdtemp('csr_dir') def tearDown(self): @@ -64,7 +78,7 @@ class InitSaveCSRTest(unittest.TestCase): @mock.patch('certbot.crypto_util.make_csr') @mock.patch('certbot.crypto_util.util.make_or_verify_dir') - def test_it(self, unused_mock_verify, mock_csr): + def test_success(self, unused_mock_verify, mock_csr): from certbot.crypto_util import init_save_csr mock_csr.return_value = (b'csr_pem', b'csr_der') @@ -76,6 +90,23 @@ class InitSaveCSRTest(unittest.TestCase): self.assertEqual(csr.data, b'csr_der') self.assertTrue('csr-certbot.pem' in csr.file) + @mock.patch('certbot.crypto_util.make_csr') + @mock.patch('certbot.crypto_util.util.make_or_verify_dir') + def test_success_dry_run(self, unused_mock_verify, mock_csr): + from certbot.crypto_util import init_save_csr + + zope.component.provideUtility( + mock.Mock(strict_permissions=True, dry_run=True), + interfaces.IConfig) + mock_csr.return_value = (b'csr_pem', b'csr_der') + + csr = init_save_csr( + mock.Mock(pem='dummy_key'), 'example.com', self.csr_dir, + 'csr-certbot.pem') + + self.assertEqual(csr.data, b'csr_der') + self.assertTrue(csr.file is None) + class MakeCSRTest(unittest.TestCase): """Tests for certbot.crypto_util.make_csr.""" From d54d3eba78cc13bc491507e48388cfeddd14d584 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Sat, 18 Mar 2017 17:04:16 -0700 Subject: [PATCH 059/128] Retry fetch chain errors (#4196) (#4383) * Retry fetch chain errors (#4196) * Trying to avoid confusing pylint * Pylint disable * Typo certz->certr * Move pylint disable, log when fetch chain fails --- certbot/client.py | 30 ++++++++++++++++++++++--- certbot/main.py | 2 +- certbot/tests/client_test.py | 43 ++++++++++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 8933ef27e..bd1971371 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -8,6 +8,7 @@ import OpenSSL import zope.component from acme import client as acme_client +from acme import errors as acme_errors from acme import jose from acme import messages @@ -242,7 +243,28 @@ class Client(object): jose.ComparableX509( OpenSSL.crypto.load_certificate_request(typ, csr.data)), authzr) - return certr, self.acme.fetch_chain(certr) + + notify = zope.component.getUtility(interfaces.IDisplay).notification + retries = 0 + chain = None + + while retries <= 1: + if retries: + notify('Failed to fetch chain, please check your network ' + 'and continue', pause=True) + try: + chain = self.acme.fetch_chain(certr) + break + except acme_errors.Error: + logger.debug('Failed to fetch chain', exc_info=True) + retries += 1 + + if chain is None: + raise acme_errors.Error( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + return certr, chain def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -269,10 +291,12 @@ class Client(object): key = crypto_util.init_save_key( self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + certr, chain = self.obtain_certificate_from_csr( + domains, csr, authzr=authzr) - return (self.obtain_certificate_from_csr(domains, csr, authzr=authzr) - + (key, csr)) + return certr, chain, key, csr + # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): """Obtain and enroll certificate. diff --git a/certbot/main.py b/certbot/main.py index b0689faa2..ab2204428 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -487,7 +487,7 @@ def install(config, plugins): try: installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "install") except errors.PluginSelectionError as e: - return e.message + return str(e) domains, _ = _find_domains_or_certname(config, installer) le_client = _init_le_client(config, authenticator=None, installer=installer) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index cc3bb098d..c61f1fa4e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -7,6 +7,7 @@ import unittest import OpenSSL import mock +from acme import errors as acme_errors from acme import jose from certbot import account @@ -170,7 +171,9 @@ class ClientTest(ClientTestCommon): self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) @mock.patch("certbot.client.logger") - def test_obtain_certificate_from_csr(self, mock_logger): + @test_util.patch_get_utility() + def test_obtain_certificate_from_csr(self, unused_mock_get_utility, + mock_logger): self._mock_obtain_certificate() test_csr = util.CSR(form="der", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler @@ -203,8 +206,44 @@ class ClientTest(ClientTestCommon): test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) + @test_util.patch_get_utility() + def test_obtain_certificate_from_csr_retry_succeeded( + self, mock_get_utility): + self._mock_obtain_certificate() + self.acme.fetch_chain.side_effect = [acme_errors.Error, + mock.sentinel.chain] + test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + + authzr = auth_handler.get_authorizations(self.eg_domains, False) + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + self.eg_domains, + test_csr, + authzr=authzr)) + self.assertEqual(1, mock_get_utility().notification.call_count) + + @test_util.patch_get_utility() + def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): + self._mock_obtain_certificate() + self.acme.fetch_chain.side_effect = acme_errors.Error + test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler + + authzr = auth_handler.get_authorizations(self.eg_domains, False) + self.assertRaises( + acme_errors.Error, + self.client.obtain_certificate_from_csr, + self.eg_domains, + test_csr, + authzr=authzr) + self.assertEqual(1, mock_get_utility().notification.call_count) + @mock.patch("certbot.client.crypto_util") - def test_obtain_certificate(self, mock_crypto_util): + @test_util.patch_get_utility() + def test_obtain_certificate(self, unused_mock_get_utility, + mock_crypto_util): self._mock_obtain_certificate() csr = util.CSR(form="der", file=None, data=CSR_SAN) From 97db9e646afd17e5e7d15b9afdf895d65510f00c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 19 Mar 2017 09:06:32 +0800 Subject: [PATCH 060/128] Fix _get_runtime_cfg on Python 3 (#4262) --- certbot-apache/certbot_apache/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 275a01e7f..67984a26c 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -136,7 +136,8 @@ class ApacheParser(object): proc = subprocess.Popen( constants.os_constant("define_cmd"), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) stdout, stderr = proc.communicate() except (OSError, ValueError): From b9121a8a37456f4504b7bf05a69180973487a885 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Sat, 18 Mar 2017 21:14:53 -0400 Subject: [PATCH 061/128] Do not output apache version when deploying cert (#4023) --- certbot-apache/certbot_apache/configurator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index cdfc01626..9f28eaad0 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -254,9 +254,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) - logger.debug("Apache version is %s", - ".".join(str(i) for i in self.version)) + logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, From 679887f6913072ce3280bdf5862d5dabc5c039f8 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Sat, 18 Mar 2017 18:33:29 -0700 Subject: [PATCH 062/128] Add --debug-challenges flag (#1684) (#4385) * Add --debug-challenges flag (#1684) * Specify None as topic for --debug-challenges --- certbot/auth_handler.py | 5 +++++ certbot/cli.py | 5 +++++ certbot/constants.py | 1 + certbot/tests/auth_handler_test.py | 22 ++++++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 53346a77c..a1f23a895 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -66,11 +66,16 @@ class AuthHandler(object): self.authzr[domain] = self.acme.request_domain_challenges(domain) self._choose_challenges(domains) + config = zope.component.getUtility(interfaces.IConfig) + notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... while self.achalls: resp = self._solve_challenges() logger.info("Waiting for verification...") + if config.debug_challenges: + notify('Challenges loaded. Press continue to submit to CA. ' + 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls self._respond(resp, best_effort) diff --git a/certbot/cli.py b/certbot/cli.py index c0af490d2..4fabd9a50 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -955,6 +955,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "testing", "--debug", action="store_true", help="Show tracebacks in case of errors, and allow certbot-auto " "execution on experimental platforms") + helpful.add( + [None, "certonly", "renew", "run"], "--debug-challenges", action="store_true", + default=flag_default("debug_challenges"), + help="After setting up challenges, wait for user input before " + "submitting to CA") helpful.add( "testing", "--no-verify-ssl", action="store_true", help=config_help("no_verify_ssl"), diff --git a/certbot/constants.py b/certbot/constants.py index c85992fc1..382b0afb3 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -33,6 +33,7 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", strict_permissions=False, + debug_challenges=False, ) STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 9d22843db..32c4c0d3b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -5,6 +5,7 @@ import unittest import mock import six +import zope.component from acme import challenges from acme import client as acme_client @@ -12,6 +13,7 @@ from acme import messages from certbot import achallenges from certbot import errors +from certbot import interfaces from certbot import util from certbot.tests import acme_util @@ -65,6 +67,12 @@ class GetAuthorizationsTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import AuthHandler + self.mock_display = mock.Mock() + zope.component.provideUtility( + self.mock_display, interfaces.IDisplay) + zope.component.provideUtility( + mock.Mock(debug_challenges=False), interfaces.IConfig) + self.mock_auth = mock.MagicMock(name="ApacheConfigurator") self.mock_auth.get_chall_pref.return_value = [challenges.TLSSNI01] @@ -157,6 +165,20 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 3) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") + def test_debug_challenges(self, mock_poll): + zope.component.provideUtility( + mock.Mock(debug_challenges=True), interfaces.IConfig) + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES) + + mock_poll.side_effect = self._validate_all + + self.handler.get_authorizations(["0"]) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + self.assertEqual(self.mock_display.notification.call_count, 1) + def test_perform_failure(self): self.mock_net.request_domain_challenges.side_effect = functools.partial( gen_dom_authzr, challs=acme_util.CHALLENGES) From 1e3678398611e7b8c103bbbd96f526007fc684a3 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Sat, 18 Mar 2017 21:37:37 -0400 Subject: [PATCH 063/128] Still include apache version in debug logging --- certbot-apache/certbot_apache/configurator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 9f28eaad0..39d25619d 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -174,6 +174,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: self.version = self.get_version() + logger.debug('Apache version is %s', + '.'.join(str(i) for i in self.version)) if self.version < (2, 2): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) From 8011fb2879659008528e8b6ba5c04ac3de1e63a0 Mon Sep 17 00:00:00 2001 From: dokazaki Date: Sat, 18 Mar 2017 19:10:10 -0700 Subject: [PATCH 064/128] Add mypy (#4386) * Initial configuration of mypy in box, correction of base mypy errors. * Move mypy install to toe * Add pylint comments for typing imports. * Remove typing module for Python 2.6 compatibility. --- acme/acme/challenges.py | 6 +++--- acme/acme/client.py | 2 +- acme/acme/crypto_util.py | 2 +- acme/acme/crypto_util_test.py | 2 +- acme/acme/jose/jwa.py | 10 +++++----- acme/acme/jose/jwk.py | 8 ++++---- acme/acme/jose/jws.py | 8 ++++---- acme/acme/jose/util.py | 4 ++-- acme/acme/messages.py | 8 ++++---- acme/acme/standalone.py | 4 ++-- acme/acme/standalone_test.py | 2 +- .../certbot_compatibility_test/interfaces.py | 10 +++++----- certbot/cli.py | 2 +- certbot/display/completer.py | 2 +- certbot/hooks.py | 8 ++++++-- certbot/interfaces.py | 16 ++++++++-------- certbot/plugins/disco.py | 2 +- certbot/reporter.py | 2 +- certbot/tests/util_test.py | 2 +- tox.ini | 7 +++++++ 20 files changed, 59 insertions(+), 48 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index ac4e3d60a..14641af10 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -5,7 +5,7 @@ import hashlib import logging import socket -from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import hashes # type: ignore import OpenSSL import requests @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json | pylint: disable=abstract-method """ACME challenge.""" - TYPES = {} + TYPES = {} # type: dict @classmethod def from_json(cls, jobj): @@ -37,7 +37,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(jose.TypedJSONObjectWithFields): # _fields_to_partial_json | pylint: disable=abstract-method """ACME challenge response.""" - TYPES = {} + TYPES = {} # type: dict resource_type = 'challenge' resource = fields.Resource(resource_type) diff --git a/acme/acme/client.py b/acme/acme/client.py index d6166960d..40291e115 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning if sys.version_info < (2, 7, 9): # pragma: no cover try: - requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() + requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore except AttributeError: import urllib3.contrib.pyopenssl # pylint: disable=import-error urllib3.contrib.pyopenssl.inject_into_urllib3() diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 266f2c0c7..6a33b3e52 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni # should be changed to use "set_options" to disable SSLv2 and SSLv3, # in case it's used for things other than probing/serving! -_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD +_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD # type: ignore class SSLSocket(object): # pylint: disable=too-few-public-methods diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index ebb4010a6..9cf1f7deb 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -6,7 +6,7 @@ import time import unittest import six -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver #type: ignore # pylint: disable=import-error import OpenSSL diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py index 1853e0107..9b682ecab 100644 --- a/acme/acme/jose/jwa.py +++ b/acme/acme/jose/jwa.py @@ -9,9 +9,9 @@ import logging import cryptography.exceptions from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import hmac -from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import hmac # type: ignore +from cryptography.hazmat.primitives.asymmetric import padding # type: ignore from acme.jose import errors from acme.jose import interfaces @@ -28,9 +28,9 @@ class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method """JSON Web Algorithm.""" -class JWASignature(JWA, collections.Hashable): +class JWASignature(JWA, collections.Hashable): # type: ignore """JSON Web Signature Algorithm.""" - SIGNATURES = {} + SIGNATURES = {} # type: dict def __init__(self, name): self.name = name diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py index 5b6965c4d..54423f670 100644 --- a/acme/acme/jose/jwk.py +++ b/acme/acme/jose/jwk.py @@ -6,9 +6,9 @@ import logging import cryptography.exceptions from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import hashes # type: ignore from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import ec # type: ignore from cryptography.hazmat.primitives.asymmetric import rsa import six @@ -25,8 +25,8 @@ class JWK(json_util.TypedJSONObjectWithFields): # pylint: disable=too-few-public-methods """JSON Web Key.""" type_field_name = 'kty' - TYPES = {} - cryptography_key_types = () + TYPES = {} # type: dict + cryptography_key_types = () # type: tuple """Subclasses should override.""" required = NotImplemented diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py index 9c14cf729..8fa8d7670 100644 --- a/acme/acme/jose/jws.py +++ b/acme/acme/jose/jws.py @@ -121,12 +121,12 @@ class Header(json_util.JSONObjectWithFields): # x5c does NOT use JOSE Base64 (4.1.6) - @x5c.encoder + @x5c.encoder # type: ignore def x5c(value): # pylint: disable=missing-docstring,no-self-argument return [base64.b64encode(OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] - @x5c.decoder + @x5c.decoder # type: ignore def x5c(value): # pylint: disable=missing-docstring,no-self-argument try: return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( @@ -157,12 +157,12 @@ class Signature(json_util.JSONObjectWithFields): 'signature', decoder=json_util.decode_b64jose, encoder=json_util.encode_b64jose) - @protected.encoder + @protected.encoder # type: ignore def protected(value): # pylint: disable=missing-docstring,no-self-argument # wrong type guess (Signature, not bytes) | pylint: disable=no-member return json_util.encode_b64jose(value.encode('utf-8')) - @protected.decoder + @protected.decoder # type: ignore def protected(value): # pylint: disable=missing-docstring,no-self-argument return json_util.decode_b64jose(value).decode('utf-8') diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index 6be9a6602..26b7e0c5a 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -134,7 +134,7 @@ class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods return hash((self.__class__, pub.n, pub.e)) -class ImmutableMap(collections.Mapping, collections.Hashable): +class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore # pylint: disable=too-few-public-methods """Immutable key to value mapping with attribute access.""" @@ -180,7 +180,7 @@ class ImmutableMap(collections.Mapping, collections.Hashable): for key, value in six.iteritems(self))) -class frozendict(collections.Mapping, collections.Hashable): +class frozendict(collections.Mapping, collections.Hashable): # type: ignore # pylint: disable=invalid-name,too-few-public-methods """Frozen dictionary.""" __slots__ = ('_items', '_keys') diff --git a/acme/acme/messages.py b/acme/acme/messages.py index f7670dd72..4070290ad 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -98,7 +98,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None) -class _Constant(jose.JSONDeSerializable, collections.Hashable): +class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented @@ -132,7 +132,7 @@ class _Constant(jose.JSONDeSerializable, collections.Hashable): class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: dict STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -143,7 +143,7 @@ STATUS_REVOKED = Status('revoked') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: dict IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -161,7 +161,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} + _REGISTERED_TYPES = {} # type: dict class Meta(jose.JSONObjectWithFields): """Directory Meta.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 02cc2daf5..087240c15 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -6,9 +6,9 @@ import logging import os import sys -from six.moves import BaseHTTPServer # pylint: disable=import-error +from six.moves import BaseHTTPServer # type: ignore # pylint: disable=import-error from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver # type: ignore # pylint: disable=import-error import OpenSSL diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 58469d470..613258c97 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -7,7 +7,7 @@ import time import unittest from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver # type: ignore # pylint: disable=import-error import requests diff --git a/certbot-compatibility-test/certbot_compatibility_test/interfaces.py b/certbot-compatibility-test/certbot_compatibility_test/interfaces.py index cd367d9af..7d3daee09 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/interfaces.py +++ b/certbot-compatibility-test/certbot_compatibility_test/interfaces.py @@ -20,20 +20,20 @@ class IPluginProxy(zope.interface.Interface): def __init__(args): """Initializes the plugin with the given command line args""" - def cleanup_from_tests(): + def cleanup_from_tests(): # type: ignore """Performs any necessary cleanup from running plugin tests. This is guaranteed to be called before the program exits. """ - def has_more_configs(): + def has_more_configs(): # type: ignore """Returns True if there are more configs to test""" - def load_config(): + def load_config(): # type: ignore """Loads the next config and returns its name""" - def get_testable_domain_names(): + def get_testable_domain_names(): # type: ignore """Returns the domain names that can be used in testing""" @@ -44,7 +44,7 @@ class IAuthenticatorProxy(IPluginProxy, certbot.interfaces.IAuthenticator): class IInstallerProxy(IPluginProxy, certbot.interfaces.IInstaller): """Wraps a Certbot installer""" - def get_all_names_answer(): + def get_all_names_answer(): # type: ignore """Returns all names that should be found by the installer""" diff --git a/certbot/cli.py b/certbot/cli.py index 4fabd9a50..1d8952d20 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,7 +207,7 @@ def set_by_cli(var): return False # static housekeeping var -set_by_cli.detector = None +set_by_cli.detector = None # type: ignore def has_default_value(option, value): diff --git a/certbot/display/completer.py b/certbot/display/completer.py index 37564954a..08b55fdea 100644 --- a/certbot/display/completer.py +++ b/certbot/display/completer.py @@ -4,7 +4,7 @@ import glob try: import readline except ImportError: - import certbot.display.dummy_readline as readline + import certbot.display.dummy_readline as readline # type: ignore class Completer(object): diff --git a/certbot/hooks.py b/certbot/hooks.py index ada3d3aaa..75d7a3b20 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -13,12 +13,14 @@ from certbot.plugins import util as plug_util logger = logging.getLogger(__name__) + def validate_hooks(config): """Check hook commands are executable.""" validate_hook(config.pre_hook, "pre") validate_hook(config.post_hook, "post") validate_hook(config.renew_hook, "renew") + def _prog(shell_cmd): """Extract the program run by a shell command. @@ -52,6 +54,7 @@ def validate_hook(shell_cmd, hook_name): raise errors.HookCommandNotFound(msg) + def pre_hook(config): "Run pre-hook if it's defined and hasn't been run." cmd = config.pre_hook @@ -62,7 +65,7 @@ def pre_hook(config): elif cmd: logger.info("Pre-hook command already run, skipping: %s", cmd) -pre_hook.already = set() +pre_hook.already = set() # type: ignore def post_hook(config): @@ -82,7 +85,8 @@ def post_hook(config): logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) -post_hook.eventually = [] +post_hook.eventually = [] # type: ignore + def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" diff --git a/certbot/interfaces.py b/certbot/interfaces.py index a2767121b..213992993 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -99,7 +99,7 @@ class IPluginFactory(zope.interface.Interface): class IPlugin(zope.interface.Interface): """Certbot plugin.""" - def prepare(): + def prepare(): # type: ignore """Prepare the plugin. Finish up any additional initialization. @@ -118,7 +118,7 @@ class IPlugin(zope.interface.Interface): """ - def more_info(): + def more_info(): # type: ignore """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user @@ -251,7 +251,7 @@ class IInstaller(IPlugin): """ - def get_all_names(): + def get_all_names(): # type: ignore """Returns all names that may be authenticated. :rtype: `collections.Iterable` of `str` @@ -288,7 +288,7 @@ class IInstaller(IPlugin): """ - def supported_enhancements(): + def supported_enhancements(): # type: ignore """Returns a `collections.Iterable` of supported enhancements. :returns: supported enhancements which should be a subset of @@ -326,7 +326,7 @@ class IInstaller(IPlugin): """ - def recovery_routine(): + def recovery_routine(): # type: ignore """Revert configuration to most recent finalized checkpoint. Remove all changes (temporary and permanent) that have not been @@ -337,21 +337,21 @@ class IInstaller(IPlugin): """ - def view_config_changes(): + def view_config_changes(): # type: ignore """Display all of the LE config changes. :raises .PluginError: when config changes cannot be parsed """ - def config_test(): + def config_test(): # type: ignore """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ - def restart(): + def restart(): # type: ignore """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index e567422e2..a17f8d7b3 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -27,7 +27,7 @@ class PluginEntryPoint(object): """Distributions for which prefix will be omitted.""" # this object is mutable, don't allow it to be hashed! - __hash__ = None + __hash__ = None # type: ignore def __init__(self, entry_point): self.name = self.entry_point_to_plugin_name(entry_point) diff --git a/certbot/reporter.py b/certbot/reporter.py index 118b13166..f836dbc8c 100644 --- a/certbot/reporter.py +++ b/certbot/reporter.py @@ -7,7 +7,7 @@ import os import sys import textwrap -from six.moves import queue # pylint: disable=import-error +from six.moves import queue # type: ignore # pylint: disable=import-error import zope.interface from certbot import interfaces diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6dc839025..13a91dfb8 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -193,7 +193,7 @@ try: file_type = file except NameError: import io - file_type = io.TextIOWrapper + file_type = io.TextIOWrapper # type: ignore class UniqueLineageNameTest(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 3e284bf29..d393bb610 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,13 @@ commands = pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot pylint --reports=n --rcfile=.pylintrc acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot +[testenv:mypy] +basepython = python3.4 +commands = + pip install mypy + pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot + mypy --py2 --ignore-missing-imports acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot + [testenv:apacheconftest] #basepython = python2.7 commands = From 32122cfa21fe7ba43189658158949cfe16dc6717 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 20 Mar 2017 15:48:39 -0700 Subject: [PATCH 065/128] Add a global lock file to Certbot (#4369) * add fasteners as a dependency * add LOCK_FILE constant * Add lock file to Certbot * Move code to _run_subcommand * move lock file path into CLI_CONSTANTS * add --lock-path flag * move locking code to separate function * Add TestAcquireFileLock * assert we log * test lock contention * add fasteners to certbot-auto * Use a different lock file for each test in MainTest --- certbot/cli.py | 2 + certbot/constants.py | 1 + certbot/interfaces.py | 3 ++ certbot/main.py | 53 ++++++++++++++++++- certbot/tests/main_test.py | 53 ++++++++++++++++++- letsencrypt-auto-source/letsencrypt-auto | 6 +++ .../pieces/letsencrypt-auto-requirements.txt | 6 +++ setup.py | 1 + 8 files changed, 123 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 1d8952d20..5b8711da6 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1167,6 +1167,8 @@ def _paths_parser(helpful): help="Logs directory.") add("paths", "--server", default=flag_default("server"), help=config_help("server")) + add("paths", "--lock-path", default=flag_default("lock_path"), + help=config_help('lock_path')) def _plugins_parsing(helpful, plugins): diff --git a/certbot/constants.py b/certbot/constants.py index 382b0afb3..24cf89199 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -33,6 +33,7 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", strict_permissions=False, + lock_path="/tmp/.certbot.lock", debug_challenges=False, ) STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 213992993..33bde9430 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -222,6 +222,9 @@ class IConfig(zope.interface.Interface): key_dir = zope.interface.Attribute("Keys storage.") temp_checkpoint_dir = zope.interface.Attribute( "Temporary checkpoint directory.") + lock_path = zope.interface.Attribute( + "Path to the lock file used to prevent multiple instances of " + "Certbot from modifying your server's configuration at once.") no_verify_ssl = zope.interface.Attribute( "Disable verification of the ACME server's certificate.") diff --git a/certbot/main.py b/certbot/main.py index ab2204428..c6e61fbf9 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -8,6 +8,7 @@ import sys import time import traceback +import fasteners import zope.component from acme import jose @@ -866,6 +867,56 @@ def _post_logging_setup(config, plugins, cli_args): logger.debug("Discovered plugins: %r", plugins) +def acquire_file_lock(lock_path): + """Obtain a lock on the file at the specified path. + + :param str lock_path: path to the file to be locked + + :returns: lock file object representing the acquired lock + :rtype: fasteners.InterProcessLock + + :raises .Error: if the lock is held by another process + + """ + lock = fasteners.InterProcessLock(lock_path) + logger.debug("Attempting to acquire lock file %s", lock_path) + + try: + lock.acquire(blocking=False) + except IOError as err: + logger.debug(err) + logger.warning( + "Unable to access lock file %s. You should set --lock-file " + "to a writeable path to ensure multiple instances of " + "Certbot don't attempt modify your configuration " + "simultaneously.", lock_path) + else: + if not lock.acquired: + raise errors.Error( + "Another instance of Certbot is already running.") + + return lock + + +def _run_subcommand(config, plugins): + """Executes the Certbot subcommand specified in the configuration. + + :param .IConfig config: parsed configuration object + :param .PluginsRegistry plugins: available plugins + + :returns: return value from the specified subcommand + :rtype: str or int + + """ + lock = acquire_file_lock(config.lock_path) + + try: + return config.func(config, plugins) + finally: + if lock.acquired: + lock.release() + + def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" sys.excepthook = functools.partial(_handle_exception, config=None) @@ -893,7 +944,7 @@ def main(cli_args=sys.argv[1:]): zope.component.provideUtility(report) atexit.register(report.atexit_print_messages) - return config.func(config, plugins) + return _run_subcommand(config, plugins) if __name__ == "__main__": diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 170eceb48..5707231b7 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -4,6 +4,7 @@ from __future__ import print_function import itertools import mock +import multiprocessing import os import shutil import tempfile @@ -451,7 +452,8 @@ class MainTest(unittest.TestCase): # pylint: disable=too-many-public-methods os.mkdir(self.logs_dir) self.standard_args = ['--config-dir', self.config_dir, '--work-dir', self.work_dir, - '--logs-dir', self.logs_dir, '--text'] + '--logs-dir', self.logs_dir, '--text', + '--lock-path', os.path.join(self.tmp_dir, 'certbot.lock')] def tearDown(self): # Reset globals in cli @@ -1308,5 +1310,54 @@ class TestHandleException(unittest.TestCase): traceback.format_exception_only(KeyboardInterrupt, interrupt))) +class TestAcquireFileLock(unittest.TestCase): + """Test main.acquire_file_lock.""" + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.lock_path = os.path.join(self.tempdir, 'certbot.lock') + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @mock.patch('certbot.main.logger') + def test_bad_path(self, mock_logger): + lock = main.acquire_file_lock(os.getcwd()) + self.assertTrue(mock_logger.warning.called) + self.assertFalse(lock.acquired) + + def test_held_lock(self): + # start child and wait for it to grab the lock + cv = multiprocessing.Condition() + cv.acquire() + child_args = (cv, self.lock_path,) + child = multiprocessing.Process(target=_hold_lock, args=child_args) + child.start() + cv.wait() + + # assert we can't grab lock and terminate the child + self.assertRaises(errors.Error, main.acquire_file_lock, self.lock_path) + cv.notify() + cv.release() + child.join() + self.assertEqual(child.exitcode, 0) + + +def _hold_lock(cv, lock_path): + """Acquire a file lock at lock_path and wait to release it. + + :param multiprocessing.Condition cv: condition for syncronization + :param str lock_path: path to the file lock + + """ + import fasteners + lock = fasteners.InterProcessLock(lock_path) + lock.acquire() + cv.acquire() + cv.notify() + cv.wait() + lock.release() + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 35c7a530c..475d57fee 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -727,6 +727,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 +fasteners==0.14.1 \ + --hash=sha256:564a115ff9698767df401efca29620cbb1a1c2146b7095ebd304b79cc5807a7c \ + --hash=sha256:427c76773fe036ddfa41e57d89086ea03111bbac57c55fc55f3006d027107e18 funcsigs==0.4 \ --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 @@ -739,6 +742,9 @@ ipaddress==1.0.16 \ linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +monotonic==1.3 \ + --hash=sha256:a8c7690953546c6bc8a4f05d347718db50de1225b29f4b9f346c0c6f19bdc286 \ + --hash=sha256:2b469e2d7dd403f7f7f79227fe5ad551ee1e76f8bb300ae935209884b93c7c1b ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f parsedatetime==2.1 \ diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index d70d24e2a..fbf416d66 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -65,6 +65,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 +fasteners==0.14.1 \ + --hash=sha256:564a115ff9698767df401efca29620cbb1a1c2146b7095ebd304b79cc5807a7c \ + --hash=sha256:427c76773fe036ddfa41e57d89086ea03111bbac57c55fc55f3006d027107e18 funcsigs==0.4 \ --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 @@ -77,6 +80,9 @@ ipaddress==1.0.16 \ linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +monotonic==1.3 \ + --hash=sha256:a8c7690953546c6bc8a4f05d347718db50de1225b29f4b9f346c0c6f19bdc286 \ + --hash=sha256:2b469e2d7dd403f7f7f79227fe5ad551ee1e76f8bb300ae935209884b93c7c1b ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f parsedatetime==2.1 \ diff --git a/setup.py b/setup.py index 0e8d19a22..59f6dbd9e 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate + 'fasteners', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'PyOpenSSL', From b7d282309d9e28221ab557264966d42f4b383529 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 20 Mar 2017 17:57:09 -0700 Subject: [PATCH 066/128] Save hyphenated plugin params for renewal (#4281) * fix plugin namespace check * Add test to prevent regressions --- certbot/storage.py | 10 ++++++---- certbot/tests/storage_test.py | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index dacc73c4c..a1462b72d 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -19,6 +19,9 @@ from certbot import errors from certbot import error_handler from certbot import util +from certbot.plugins import common as plugins_common +from certbot.plugins import disco as plugins_disco + logger = logging.getLogger(__name__) ALL_FOUR = ("cert", "privkey", "chain", "fullchain") @@ -179,13 +182,12 @@ def _relevant(option): :rtype: bool """ - # The list() here produces a list of the plugin names as strings. from certbot import renewal - from certbot.plugins import disco as plugins_disco - plugins = list(plugins_disco.PluginsRegistry.find_all()) + plugins = plugins_disco.PluginsRegistry.find_all() + namespaces = [plugins_common.dest_namespace(plugin) for plugin in plugins] return (option in renewal.CONFIG_ITEMS or - any(option.startswith(x + "_") for x in plugins)) + any(option.startswith(namespace) for namespace in namespaces)) def relevant_values(all_values): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index f52f31d3d..f7bde012d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -581,6 +581,15 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual( self._test_relevant_values_common(values), values) + @mock.patch("certbot.cli.set_by_cli") + @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") + def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): + mock_set_by_cli.return_value = True + mock_find_all.return_value = ["certbot-foo:bar"] + values = {"certbot_foo:bar_baz": 42} + self.assertEqual( + self._test_relevant_values_common(values), values) + @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" From bf45cea7cddd47e47194904d5f5b47757c622306 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 20 Mar 2017 18:00:50 -0700 Subject: [PATCH 067/128] Ensure a SHA2 hash algorithm is used when signing releases (#4384) * use gpg2 * explictly use sha256 --- tools/release.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/release.sh b/tools/release.sh index 81582cef0..1da11fe2c 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -109,7 +109,7 @@ do echo "Signing ($pkg_dir)" for x in dist/*.tar.gz dist/*.whl do - gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign $x + gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x done cd - @@ -194,7 +194,7 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ done # This signature is not quite as strong, but easier for people to verify out of band -gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign letsencrypt-auto-source/letsencrypt-auto +gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto # We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, # but we can use the right name for certbot-auto.asc from day one mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc @@ -214,7 +214,7 @@ name=${root_without_le%.*} ext="${root_without_le##*.}" rev="$(git rev-parse --short HEAD)" echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz +echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz cd ~- echo "New root: $root" From 7be2e790251a9f5b7c8ab48560bfe4e9737c11af Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Fri, 24 Mar 2017 19:45:53 -0700 Subject: [PATCH 068/128] Fix nginx parser (#4296) * rewrite nginx parser to allow everything that nginx does * also make changes in tls_sni_01.py * add test case with * allow embedded variables * allow empty ${} variable * fix quotes * un-special case if * update all tests to reflect current parsing * escape in QuotedString after merge * add test cases for variable weirdness that are almost certainly nginx bugs * update regex for correct variable rules * close paren doesn't invoke last_space * Make test file valid Nginx syntax --- .../simplepythonfcgi/weird-spacing.conf | 6 +- certbot-nginx/certbot_nginx/configurator.py | 8 +- certbot-nginx/certbot_nginx/nginxparser.py | 112 ++++------ certbot-nginx/certbot_nginx/parser.py | 30 ++- .../certbot_nginx/tests/configurator_test.py | 18 +- .../certbot_nginx/tests/nginxparser_test.py | 203 ++++++++++++++++-- certbot-nginx/certbot_nginx/tests/obj_test.py | 18 +- .../certbot_nginx/tests/parser_test.py | 14 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 9 +- 9 files changed, 273 insertions(+), 145 deletions(-) diff --git a/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf b/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf index a201fe659..5fbc76676 100644 --- a/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf +++ b/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf @@ -1,16 +1,16 @@ # static files location ~ ^/(images|javascript|js|css|flash|media|static)/ { - root ${PROJECTBASE}/${PROJECTNAME}/static; + root "${PROJECTBASE}/${PROJECTNAME}/static"; } location = /favicon.ico { - root ${PROJECTBASE}/${PROJECTNAME}/static/images; + root "${PROJECTBASE}/${PROJECTNAME}/static/images"; } # pass all requests to FastCGI TG server listening on ${HOST}:${PORT} # location / { - fastcgi_pass ${HOST}:${PORT}; + fastcgi_pass "${HOST}:${PORT}"; fastcgi_index index; fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; include conf/fastcgi_params; diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e62194d4f..225702251 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -32,16 +32,16 @@ from certbot_nginx import parser logger = logging.getLogger(__name__) REDIRECT_BLOCK = [[ - ['\n ', 'if', ' ', '($scheme != "https") '], - [['\n ', 'return', ' ', '301 https://$host$request_uri'], + ['\n ', 'if', ' ', '($scheme', ' ', '!=', ' ', '"https") '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], '\n '] ], ['\n']] TEST_REDIRECT_BLOCK = [ [ - ['if', '($scheme != "https")'], + ['if', '($scheme', '!=', '"https")'], [ - ['return', '301 https://$host$request_uri'] + ['return', '301', 'https://$host$request_uri'] ] ], ['#', ' managed by Certbot'] diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index 67ac7c3a1..20aeeb554 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -2,11 +2,9 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging -import string from pyparsing import ( - Literal, White, Word, alphanums, CharsNotIn, Combine, Forward, Group, - Optional, OneOrMore, QuotedString, Regex, ZeroOrMore) + Literal, White, Forward, Group, Optional, OneOrMore, QuotedString, Regex, ZeroOrMore, Combine) from pyparsing import stringEnd from pyparsing import restOfLine @@ -14,73 +12,42 @@ logger = logging.getLogger(__name__) class RawNginxParser(object): # pylint: disable=expression-not-assigned + # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" # constants - space = Optional(White()) - nonspace = Regex(r"\S+") + space = Optional(White()).leaveWhitespace() + required_space = White().leaveWhitespace() + left_bracket = Literal("{").suppress() - right_bracket = space.leaveWhitespace() + Literal("}").suppress() + right_bracket = space + Literal("}").suppress() semicolon = Literal(";").suppress() - key = Word(alphanums + "_/+-.") - dollar_var = Combine(Literal('$') + Regex(r"[^\{\};,\s]+")) - condition = Regex(r"\(.+\)") - # Matches anything that is not a special character, and ${SHELL_VARS}, AND - # any chars in single or double quotes - # All of these COULD be upgraded to something like - # https://stackoverflow.com/a/16130746 - dquoted = QuotedString('"', multiline=True, unquoteResults=False) - squoted = QuotedString("'", multiline=True, unquoteResults=False) - nonspecial = Regex(r"[^\{\};,]") - varsub = Regex(r"(\$\{\w+\})") - # nonspecial nibbles one character at a time, but the other objects take - # precedence. We use ZeroOrMore to allow entries like "break ;" to be - # parsed as assignments - value = Combine(ZeroOrMore(dquoted | squoted | varsub | nonspecial)) + dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\') + squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\') + quoted = dquoted | squoted + head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space) + tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else + tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars)) + paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars)) + # note: ')' allows extension, but then we fall into else, not last_space. - location = CharsNotIn("{};," + string.whitespace) - # modifier for location uri [ = | ~ | ~* | ^~ ] - modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~") + token = paren_quote_extend | tokenchars | quoted + + whitespace_token_group = space + token + ZeroOrMore(required_space + token) + space + assignment = whitespace_token_group + semicolon - # rules comment = space + Literal('#') + restOfLine - assignment = space + key + Optional(space + value, default=None) + semicolon - location_statement = space + Optional(modifier) + Optional(space + location + space) - if_statement = space + Literal("if") + space + condition + space - charset_map_statement = space + Literal("charset_map") + space + value + space + value - - map_statement = space + Literal("map") + space + nonspace + space + dollar_var + space - # This is NOT an accurate way to parse nginx map entries; it's almost - # certainly too permissive and may be wrong in other ways, but it should - # preserve things correctly in mmmmost or all cases. - # - # - I can neither prove nor disprove that it is correct wrt all escaped - # semicolon situations - # Addresses https://github.com/fatiherikli/nginxparser/issues/19 - map_pattern = Regex(r'".*"') | Regex(r"'.*'") | nonspace - map_entry = space + map_pattern + space + value + space + semicolon - map_block = Group( - Group(map_statement).leaveWhitespace() + - left_bracket + - Group(ZeroOrMore(Group(comment | map_entry)) + space).leaveWhitespace() + - right_bracket) - block = Forward() - # key could for instance be "server" or "http", or "location" (in which case - # location_statement needs to have a non-empty location) + # order matters! see issue 518, and also http { # server { \n} + contents = Group(comment) | Group(block) | Group(assignment) - block_begin = (Group(space + key + location_statement) ^ - Group(if_statement) ^ - Group(charset_map_statement)).leaveWhitespace() + block_begin = Group(whitespace_token_group) + block_innards = Group(ZeroOrMore(contents) + space).leaveWhitespace() + block << block_begin + left_bracket + block_innards + right_bracket - block_innards = Group(ZeroOrMore(Group(comment | assignment) | block | map_block) - + space).leaveWhitespace() - - block << Group(block_begin + left_bracket + block_innards + right_bracket) - - script = OneOrMore(Group(comment | assignment) ^ block ^ map_block) + space + stringEnd + script = OneOrMore(contents) + space + stringEnd script.parseWithTabs().leaveWhitespace() def __init__(self, source): @@ -107,30 +74,23 @@ class RawNginxDumper(object): if isinstance(b0, str): yield b0 continue - b = copy.deepcopy(b0) - if spacey(b[0]): - yield b.pop(0) # indentation - if not b: + item = copy.deepcopy(b0) + if spacey(item[0]): + yield item.pop(0) # indentation + if not item: continue - key, values = b.pop(0), b.pop(0) - if isinstance(key, list): - yield "".join(key) + '{' - for parameter in values: + if isinstance(item[0], list): # block + yield "".join(item.pop(0)) + '{' + for parameter in item.pop(0): for line in self.__iter__([parameter]): # negate "for b0 in blocks" yield line yield '}' - else: - if isinstance(key, str) and key.strip() == '#': # comment - yield key + values - else: # assignment - gap = "" - # Sometimes the parser has stuck some gap whitespace in here; - # if so rotate it into gap - if values and spacey(values): - gap = values - values = b.pop(0) - yield key + gap + values + ';' + else: # not a block - list of strings + semicolon = ";" + if isinstance(item[0], str) and item[0].strip() == '#': # comment + semicolon = "" + yield "".join(item) + semicolon def __str__(self): """Return the parsed block as a string.""" diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index fd4ea4f11..6f3f344db 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -298,9 +298,9 @@ class NginxParser(object): """ server = vhost.raw for directive in server: - if not directive or len(directive) < 2: + if not directive: continue - elif directive[0] == 'ssl' and directive[1] == 'on': + elif _is_ssl_on_directive(directive): return True return False @@ -468,17 +468,17 @@ def _is_include_directive(entry): len(entry) == 2 and entry[0] == 'include' and isinstance(entry[1], str)) +def _is_ssl_on_directive(entry): + """Checks if an nginx parsed entry is an 'ssl on' directive. -def _get_servernames(names): - """Turns a server_name string into a list of server names - - :param str names: server names - :rtype: list + :param list entry: the parsed entry + :returns: Whether it's an 'ssl on' directive + :rtype: bool """ - whitespace_re = re.compile(r'\s+') - names = re.sub(whitespace_re, ' ', names) - return names.split(' ') + return (isinstance(entry, list) and + len(entry) == 2 and entry[0] == 'ssl' and + entry[1] == 'on') def _add_directives(block, directives, replace): """Adds or replaces directives in a config block. @@ -550,12 +550,11 @@ def _add_directive(block, directive, replace): # and there is already a copy of that directive with a different value # in the config file. directive_name = directive[0] - directive_value = directive[1] if location is None or (isinstance(directive_name, str) and directive_name in REPEATABLE_DIRECTIVES): block.append(directive) _comment_directive(block, len(block) - 1) - elif block[location][1] != directive_value: + elif block[location] != directive: raise errors.MisconfigurationError( 'tried to insert directive "{0}" but found ' 'conflicting "{1}".'.format(directive, block[location])) @@ -585,15 +584,14 @@ def _parse_server_raw(server): if not directive: continue if directive[0] == 'listen': - addr = obj.Addr.fromstring(directive[1]) + addr = obj.Addr.fromstring(" ".join(directive[1:])) if addr: parsed_server['addrs'].add(addr) if addr.ssl: parsed_server['ssl'] = True elif directive[0] == 'server_name': - parsed_server['names'].update( - _get_servernames(directive[1])) - elif directive[0] == 'ssl' and directive[1] == 'on': + parsed_server['names'].update(directive[1:]) + elif _is_ssl_on_directive(directive): parsed_server['ssl'] = True apply_ssl_to_all_addrs = True diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index d491d2a15..b9e70cd59 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -94,7 +94,7 @@ class NginxConfiguratorTest(util.NginxTest): None, [0]) self.config.parser.add_server_directives( mock_vhost, - [['listen', ' ', '5001 ssl']], + [['listen', ' ', '5001', ' ', 'ssl']], replace=False) self.config.save() @@ -105,7 +105,7 @@ class NginxConfiguratorTest(util.NginxTest): ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['#', parser.COMMENT]]]], parsed[0]) @@ -205,13 +205,13 @@ class NginxConfiguratorTest(util.NginxTest): ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['ssl_certificate', 'example/fullchain.pem'], ['ssl_certificate_key', 'example/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) ]], parsed_example_conf) - self.assertEqual([['server_name', 'somename alias another.alias']], + self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], parsed_server_conf) self.assertTrue(util.contains_at_depth( parsed_nginx_conf, @@ -222,8 +222,8 @@ class NginxConfiguratorTest(util.NginxTest): ['include', 'server.conf'], [['location', '/'], [['root', 'html'], - ['index', 'index.html index.htm']]], - ['listen', '5001 ssl'], + ['index', 'index.html', 'index.htm']]], + ['listen', '5001', 'ssl'], ['ssl_certificate', '/etc/nginx/fullchain.pem'], ['ssl_certificate_key', '/etc/nginx/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) @@ -247,7 +247,7 @@ class NginxConfiguratorTest(util.NginxTest): ['server_name', 'summer.com'], ['listen', '80'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['ssl_certificate', 'summer/fullchain.pem'], ['ssl_certificate_key', 'summer/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) @@ -408,8 +408,8 @@ class NginxConfiguratorTest(util.NginxTest): # Test that we successfully add a redirect when there is # a listen directive expected = [ - ['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + ['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ] example_conf = self.config.parser.abs_path('sites-enabled/example.com') diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index 653fb5069..0a8dbd100 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -25,15 +25,15 @@ class TestRawNginxParser(unittest.TestCase): def test_blocks(self): parsed = RawNginxParser.block.parseString('foo {}').asList() - self.assertEqual(parsed, [[['foo', ' '], []]]) + self.assertEqual(parsed, [['foo', ' '], []]) parsed = RawNginxParser.block.parseString('location /foo{}').asList() - self.assertEqual(parsed, [[['location', ' ', '/foo'], []]]) + self.assertEqual(parsed, [['location', ' ', '/foo'], []]) parsed = RawNginxParser.block.parseString('foo { bar foo ; }').asList() - self.assertEqual(parsed, [[['foo', ' '], [[' ', 'bar', ' ', 'foo '], ' ']]]) + self.assertEqual(parsed, [['foo', ' '], [[' ', 'bar', ' ', 'foo', ' '], ' ']]) def test_nested_blocks(self): parsed = RawNginxParser.block.parseString('foo { bar {} }').asList() - block, content = FIRST(parsed) + block, content = parsed self.assertEqual(FIRST(content), [[' ', 'bar', ' '], []]) self.assertEqual(FIRST(block), 'foo') @@ -72,8 +72,8 @@ class TestRawNginxParser(unittest.TestCase): [['user', 'www-data'], [['http'], [[['server'], [ - ['listen', '*:80 default_server ssl'], - ['server_name', '*.www.foo.com *.www.example.com'], + ['listen', '*:80', 'default_server', 'ssl'], + ['server_name', '*.www.foo.com', '*.www.example.com'], ['root', '/home/ubuntu/sites/foo/'], [['location', '/status'], [ [['types'], [['image/jpeg', 'jpg']]], @@ -97,17 +97,17 @@ class TestRawNginxParser(unittest.TestCase): [['server'], [['server_name', 'with.if'], [['location', '~', '^/services/.+$'], - [[['if', '($request_filename ~* \\.(ttf|woff)$)'], - [['add_header', 'Access-Control-Allow-Origin "*"']]]]]]], + [[['if', '($request_filename', '~*', '\\.(ttf|woff)$)'], + [['add_header', 'Access-Control-Allow-Origin', '"*"']]]]]]], [['server'], [['server_name', 'with.complicated.headers'], [['location', '~*', '\\.(?:gif|jpe?g|png)$'], - [['add_header', 'Pragma public'], + [['add_header', 'Pragma', 'public'], ['add_header', - 'Cache-Control \'public, must-revalidate, proxy-revalidate\'' - ' "test,;{}" foo'], + 'Cache-Control', '\'public, must-revalidate, proxy-revalidate\'', + '"test,;{}"', 'foo'], ['blah', '"hello;world"'], - ['try_files', '$uri @rewrites']]]]]]) + ['try_files', '$uri', '@rewrites']]]]]]) def test_parse_from_file3(self): with open(util.get_data_filename('multiline_quotes.conf')) as handle: @@ -135,7 +135,7 @@ class TestRawNginxParser(unittest.TestCase): with open(util.get_data_filename('nginx.conf')) as handle: parsed = load(handle) parsed[-1][-1].append(UnspacedList([['server'], - [['listen', ' ', '443 ssl'], + [['listen', ' ', '443', ' ', 'ssl'], ['server_name', ' ', 'localhost'], ['ssl_certificate', ' ', 'cert.pem'], ['ssl_certificate_key', ' ', 'cert.key'], @@ -144,7 +144,7 @@ class TestRawNginxParser(unittest.TestCase): ['ssl_ciphers', ' ', 'HIGH:!aNULL:!MD5'], [['location', ' ', '/'], [['root', ' ', 'html'], - ['index', ' ', 'index.html index.htm']]]]])) + ['index', ' ', 'index.html', ' ', 'index.htm']]]]])) with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) @@ -179,10 +179,175 @@ class TestRawNginxParser(unittest.TestCase): parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }') self.assertEqual(parsed, [ - [['if', '($http_accept ~* "webp")'], - [['set', '$webp "true"']]] + [['if', '($http_accept', '~*', '"webp")'], + [['set', '$webp', '"true"']]] ]) + def test_comment_in_block(self): + parsed = loads("""http { + # server{ + }""") + + self.assertEqual(parsed, [ + [['http'], + [['#', ' server{']]] + ]) + + def test_access_log(self): + # see issue #3798 + parsed = loads('access_log syslog:server=unix:/dev/log,facility=auth,' + 'tag=nginx_post,severity=info custom;') + + self.assertEqual(parsed, [ + ['access_log', + 'syslog:server=unix:/dev/log,facility=auth,tag=nginx_post,severity=info', + 'custom'] + ]) + + def test_add_header(self): + # see issue #3798 + parsed = loads('add_header Cache-Control no-cache,no-store,must-revalidate,max-age=0;') + + self.assertEqual(parsed, [ + ['add_header', 'Cache-Control', 'no-cache,no-store,must-revalidate,max-age=0'] + ]) + + def test_map_then_assignment_in_block(self): + # see issue #3798 + test_str = """http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + "~Opera Mini" 1; + *.example.com 1; + } + one; + }""" + parsed = loads(test_str) + self.assertEqual(parsed, [ + [['http'], [ + [['map', '$http_upgrade', '$connection_upgrade'], [ + ['default', 'upgrade'], + ["''", 'close'], + ['"~Opera Mini"', '1'], + ['*.example.com', '1'] + ]], + ['one'] + ]] + ]) + + def test_variable_name(self): + parsed = loads('try_files /typo3temp/tx_ncstaticfilecache/' + '$host${request_uri}index.html @nocache;') + + self.assertEqual(parsed, [ + ['try_files', + '/typo3temp/tx_ncstaticfilecache/$host${request_uri}index.html', + '@nocache'] + ]) + + def test_weird_blocks(self): + test = r""" + if ($http_user_agent ~ MSIE) { + rewrite ^(.*)$ /msie/$1 break; + } + + if ($http_cookie ~* "id=([^;]+)(?:;|$)") { + set $id $1; + } + + if ($request_method = POST) { + return 405; + } + + if ($request_method) { + return 403; + } + + if ($args ~ post=140){ + rewrite ^ http://example.com/; + } + + location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ { + alias /data/w3/images/$1; + } + """ + parsed = loads(test) + self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'], + [['rewrite', '^(.*)$', '/msie/$1', 'break']]], + [['if', '($http_cookie', '~*', '"id=([^;]+)(?:;|$)")'], [['set', '$id', '$1']]], + [['if', '($request_method', '=', 'POST)'], [['return', '405']]], + [['if', '($request_method)'], + [['return', '403']]], [['if', '($args', '~', 'post=140)'], + [['rewrite', '^', 'http://example.com/']]], + [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'], + [['alias', '/data/w3/images/$1']]]] + ) + + def test_edge_cases(self): + # quotes + parsed = loads(r'"hello\""; # blah "heh heh"') + self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']]) + + # empty var as block + parsed = loads(r"${}") + self.assertEqual(parsed, [[['$'], []]]) + + # if with comment + parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah ) + }""") + self.assertEqual(parsed, [[['if', '($http_cookie', '~*', '"id=([^;]+)(?:;|$)")'], + [['#', ' blah )']]]]) + + # end paren + test = """ + one"test"; + ("two"); + "test")red; + "test")"blue"; + "test")"three; + (one"test")one; + one"; + one"test; + one"test"one; + """ + parsed = loads(test) + self.assertEqual(parsed, [ + ['one"test"'], + ['("two")'], + ['"test")red'], + ['"test")"blue"'], + ['"test")"three'], + ['(one"test")one'], + ['one"'], + ['one"test'], + ['one"test"one'] + ]) + self.assertRaises(ParseException, loads, r'"test"one;') # fails + self.assertRaises(ParseException, loads, r'"test;') # fails + + # newlines + test = """ + server_name foo.example.com bar.example.com \ + baz.example.com qux.example.com; + server_name foo.example.com bar.example.com + baz.example.com qux.example.com; + """ + parsed = loads(test) + self.assertEqual(parsed, [ + ['server_name', 'foo.example.com', 'bar.example.com', + 'baz.example.com', 'qux.example.com'], + ['server_name', 'foo.example.com', 'bar.example.com', + 'baz.example.com', 'qux.example.com'] + ]) + + # variable weirdness + parsed = loads("directive $var;") + self.assertEqual(parsed, [['directive', '$var']]) + self.assertRaises(ParseException, loads, "server {server_name test.com};") + self.assertRaises(ParseException, loads, "directive ${var};") + + class TestUnspacedList(unittest.TestCase): """Test the UnspacedList data structure""" def setUp(self): @@ -237,18 +402,18 @@ class TestUnspacedList(unittest.TestCase): ['\n ', 'listen', ' ', '127.0.0.1'], ['\n ', 'server_name', ' ', '.example.com'], ['\n ', 'server_name', ' ', 'example.*'], '\n', - ['listen', ' ', '5001 ssl']]) + ['listen', ' ', '5001', ' ', 'ssl']]) x.insert(5, "FROGZ") self.assertEqual(x, [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], 'FROGZ']) + ['listen', '5001', 'ssl'], 'FROGZ']) self.assertEqual(x.spaced, [['\n ', 'listen', ' ', '69.50.225.155:9000'], ['\n ', 'listen', ' ', '127.0.0.1'], ['\n ', 'server_name', ' ', '.example.com'], ['\n ', 'server_name', ' ', 'example.*'], '\n', - ['listen', ' ', '5001 ssl'], + ['listen', ' ', '5001', ' ', 'ssl'], 'FROGZ']) def test_rawlists(self): diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 069f860d6..ba136bb78 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -108,8 +108,8 @@ class VirtualHostTest(unittest.TestCase): from certbot_nginx.obj import Addr raw1 = [ ['listen', '69.50.225.155:9000'], - [['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + [['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ], ['#', ' managed by Certbot'] ] @@ -119,8 +119,8 @@ class VirtualHostTest(unittest.TestCase): set(['localhost']), raw1, []) raw2 = [ ['listen', '69.50.225.155:9000'], - [['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + [['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ] ] self.vhost2 = VirtualHost( @@ -129,7 +129,7 @@ class VirtualHostTest(unittest.TestCase): set(['localhost']), raw2, []) raw3 = [ ['listen', '69.50.225.155:9000'], - ['rewrite', '^(.*)$ $scheme://www.domain.com$1 permanent;'] + ['rewrite', '^(.*)$', '$scheme://www.domain.com$1', 'permanent'] ] self.vhost3 = VirtualHost( "filep", @@ -181,7 +181,9 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme != "https")'], [['return', '301 https://$host$request_uri']]], + [['if', '($scheme', '!=', '"https")'], + [['return', '301', 'https://$host$request_uri']] + ], ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", @@ -195,7 +197,9 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme != "https")'], [['return', '302 https://$host$request_uri']]], + [['if', '($scheme', '!=', '"https")'], + [['return', '302', 'https://$host$request_uri']] + ], ['#', ' managed by Certbot'], []] vhost_bad_haystack = VirtualHost( "filp", diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 6a3f2f1de..8d20faa38 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -52,7 +52,7 @@ class NginxParserTest(util.NginxTest): 'sites-enabled/sslon.com', 'sites-enabled/globalssl.com']]), set(nparser.parsed.keys())) - self.assertEqual([['server_name', 'somename alias another.alias']], + self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], nparser.parsed[nparser.abs_path('server.conf')]) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -168,16 +168,16 @@ class NginxParserTest(util.NginxTest): [['location', '/'], [['root', 'html'], ['index', 'index.html index.htm']]] ], None) self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) - mock_vhost.raw = [['listen', '*:80 default_server ssl'], - ['server_name', '*.www.foo.com *.www.example.com'], + mock_vhost.raw = [['listen', '*:80', 'default_server', 'ssl'], + ['server_name', '*.www.foo.com', '*.www.example.com'], ['root', '/home/ubuntu/sites/foo/']] self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) mock_vhost.raw = [['listen', '80 ssl'], - ['server_name', '*.www.foo.com *.www.example.com']] + ['server_name', '*.www.foo.com', '*.www.example.com']] self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) mock_vhost.raw = [['listen', '80'], ['ssl', 'on'], - ['server_name', '*.www.foo.com *.www.example.com']] + ['server_name', '*.www.foo.com', '*.www.example.com']] self.assertTrue(nparser.has_ssl_on_directive(mock_vhost)) def test_add_server_directives(self): @@ -309,7 +309,7 @@ class NginxParserTest(util.NginxTest): self.assertFalse(server['ssl']) server = parser._parse_server_raw([ #pylint: disable=protected-access - ['listen', '443 ssl'] + ['listen', '443', 'ssl'] ]) self.assertTrue(server['ssl']) @@ -341,7 +341,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(nginxparser.UnspacedList(nparser.loc["ssl_options"]), [['ssl_session_cache', 'shared:le_nginx_SSL:1m'], ['ssl_session_timeout', '1440m'], - ['ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2'], + ['ssl_protocols', 'TLSv1', 'TLSv1.1', 'TLSv1.2'], ['ssl_prefer_server_ciphers', 'on'], ['ssl_ciphers', '"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-'+ 'AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256'+ diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index eedd97c03..2e8125911 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -96,11 +96,12 @@ class NginxTlsSni01(common.TLSSNI01): bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] main = self.configurator.parser.parsed[root] - for key, body in main: - if key == ['http']: + for line in main: + if line[0] == ['http']: + body = line[1] found_bucket = False - for k, _ in body: - if k == bucket_directive[1]: + for inner_line in body: + if inner_line[0] == bucket_directive[1]: found_bucket = True if not found_bucket: body.insert(0, bucket_directive) From 2e102ec9f7523457da30813062f2827608001eec Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sat, 25 Mar 2017 11:39:19 -0700 Subject: [PATCH 069/128] Review feedback. --- docs/packaging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/packaging.rst b/docs/packaging.rst index c3cdfb579..780eb313c 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -28,7 +28,7 @@ Notes for package maintainers 1. Do not package ``certbot-compatibility-test`` or ``letshelp-certbot`` - it's only used internally. -2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. Add a random per-system offset to avoid having a large number of clients hit Let's Encrypt's servers simultaneously. +2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. Additionally you should include a random per-machine time offset to avoid having a large number of your clients hit Let's Encrypt's servers simultaneously. 3. ``jws`` is an internal script for ``acme`` module and it doesn't have to be packaged - it's mostly for debugging: you can use it as ``echo foo | jws sign | jws verify``. From 5c93ceb67550ca4dd6e34d3edcda580cd4ab04c3 Mon Sep 17 00:00:00 2001 From: Damien Tournoud Date: Mon, 27 Mar 2017 18:24:05 +0200 Subject: [PATCH 070/128] acme: Make the network timeout configurable (#4237) This follows up on https://github.com/certbot/certbot/pull/4217, but allows users to override the default setting. --- acme/acme/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 40291e115..0c8886fc6 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -33,6 +33,8 @@ if sys.version_info < (2, 7, 9): # pragma: no cover import urllib3.contrib.pyopenssl # pylint: disable=import-error urllib3.contrib.pyopenssl.inject_into_urllib3() +DEFAULT_NETWORK_TIMEOUT = 45 + DER_CONTENT_TYPE = 'application/pkix-cert' @@ -503,13 +505,14 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes REPLAY_NONCE_HEADER = 'Replay-Nonce' def __init__(self, key, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python'): + user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): self.key = key self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() self.user_agent = user_agent self.session = requests.Session() + self._default_timeout = timeout def __del__(self): self.session.close() @@ -608,7 +611,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes kwargs['verify'] = self.verify_ssl kwargs.setdefault('headers', {}) kwargs['headers'].setdefault('User-Agent', self.user_agent) - kwargs.setdefault('timeout', 45) # timeout after 45 seconds + kwargs.setdefault('timeout', self._default_timeout) response = self.session.request(method, url, *args, **kwargs) # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. From 7d57e3104a2d7e514ae19c99d2a41094f3bfc5eb Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 27 Mar 2017 12:20:51 -0700 Subject: [PATCH 071/128] Ensure --fulchain-path gets put under paths in --help all --- certbot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 5b8711da6..0faf4a7c6 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1155,7 +1155,7 @@ def _paths_parser(helpful): default_cp = None if verb == "certonly": default_cp = flag_default("auth_chain_path") - add(["install", "paths"], "--fullchain-path", default=default_cp, type=os.path.abspath, + add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a full certificate chain (cert plus chain).") add("paths", "--chain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a certificate chain.") From 1c51ae25887f2dc3168a38d1f0042363cd7ac1e3 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 27 Mar 2017 13:58:17 -0700 Subject: [PATCH 072/128] Pin python-augeas version to avoid error with 1.0.0 (#4422) When running ./tools/venv.sh with 1.0.0 (now the latest version), I encountered: build/temp.linux-x86_64-2.7/augeas.c:434:35: fatal error: augeas.h: No such file or directory --- certbot-apache/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index db8cb11db..cb35d2686 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -11,7 +11,7 @@ install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), 'mock', - 'python-augeas', + 'python-augeas<=0.5.0', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', From e9608945c3622540ea0e0dbfb79fd62b7e5d4b47 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 27 Mar 2017 14:47:14 -0700 Subject: [PATCH 073/128] Change registering unsafely without email logging level to info (#4425) * Change registering unsafely without email logging level to info * update test with new behavior --- certbot/client.py | 2 +- certbot/tests/client_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index bd1971371..f6de26e3d 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -118,7 +118,7 @@ def register(config, account_storage, tos_cb=None): logger.warning(msg) raise errors.Error(msg) if not config.dry_run: - logger.warning("Registering without email!") + logger.info("Registering without email!") # Each new registration shall use a fresh new key key = jose.JWKRSA(key=jose.ComparableRSAKey( diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index c61f1fa4e..8b72e1df7 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -102,7 +102,7 @@ class RegisterTest(unittest.TestCase): self.config.register_unsafely_without_email = True self.config.dry_run = False self._call() - mock_logger.warning.assert_called_once_with(mock.ANY) + mock_logger.info.assert_called_once_with(mock.ANY) self.assertTrue(mock_handle.called) def test_unsupported_error(self): From 07f95e619711af5d6af6d4771e8ba445538ddb0f Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Mon, 27 Mar 2017 15:14:07 -0700 Subject: [PATCH 074/128] Improvements to example cli.ini --- examples/cli.ini | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/cli.ini b/examples/cli.ini index 63af3cc49..dbaa9c599 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -1,6 +1,11 @@ # This is an example of the kind of things you can do in a configuration file. # All flags used by the client can be configured here. Run Certbot with # "--help" to learn more about the available options. +# +# Note that these options apply automatically to all use of Certbot for +# obtaining or renewing certificates, so options specific to a single +# certificate on a system with several certificates should not be placed +# here. # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 @@ -8,13 +13,6 @@ rsa-key-size = 4096 # Uncomment and update to register with the specified e-mail address # email = foo@example.com -# Uncomment and update to generate certificates for the specified -# domains. -# domains = example.com, www.example.com - -# Uncomment to use a text interface instead of ncurses -# text = True - # Uncomment to use the standalone authenticator on port 443 # authenticator = standalone # standalone-supported-challenges = tls-sni-01 From ece68a1864b72616565711dcab8f3cfd4f388718 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 27 Mar 2017 15:19:03 -0700 Subject: [PATCH 075/128] Update Nginx ciphersuites to use Mozilla Intermediate (#4426) * Update Nginx ciphersuites to use Mozilla intermediate * update tests to match new behavior --- .../certbot_nginx/options-ssl-nginx.conf | 2 +- .../certbot_nginx/tests/parser_test.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf index 89c920b3e..e1839909d 100644 --- a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf +++ b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf @@ -4,4 +4,4 @@ ssl_session_timeout 1440m; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; -ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; +ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS"; diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 8d20faa38..9c2b8656e 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -343,14 +343,19 @@ class NginxParserTest(util.NginxTest): ['ssl_session_timeout', '1440m'], ['ssl_protocols', 'TLSv1', 'TLSv1.1', 'TLSv1.2'], ['ssl_prefer_server_ciphers', 'on'], - ['ssl_ciphers', '"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-'+ - 'AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256'+ - '-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384'+ - ' ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384'+ - ' ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-'+ - 'AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM'+ - '-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-'+ - 'AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"'] + ['ssl_ciphers', '"ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-'+ + 'RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:'+ + 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-'+ + 'SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-'+ + 'SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-'+ + 'SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:'+ + 'ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-'+ + 'AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:'+ + 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-'+ + 'SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-'+ + 'RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:'+ + 'AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:'+ + 'AES256-SHA:DES-CBC3-SHA:!DSS"'] ]) if __name__ == "__main__": From db0a4f903976c3e0a4f4d0c5fa7f7a543be0e90e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 28 Mar 2017 16:15:09 -0700 Subject: [PATCH 076/128] Document tests/integration/_common.sh. This makes it much easier to run certbot during development. --- docs/contributing.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 5cdf86147..cc24ad898 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -39,17 +39,17 @@ Then in each shell where you're working on the client, do: .. code-block:: shell source ./venv/bin/activate + . tests/integration/_common.sh After that, your shell will be using the virtual environment, and you run the -client by typing: +client by typing `certbot` or `certbot_test`. The latter is an alias that +includes several flags useful for testing. For instance, it sets various output +directories to point to /tmp/, and uses non-privileged ports for challenges, so +root privileges are not required. -.. code-block:: shell - - certbot - -Activating a shell in this way makes it easier to run unit tests -with ``tox`` and integration tests, as described below. To reverse this, you -can type ``deactivate``. More information can be found in the `virtualenv docs`_. +Activating a shell with `venv/bin/activate` sets environment variables so that +Python pulls in the correct versions of various packages needed by Certbot. +More information can be found in the `virtualenv docs`_. .. _`virtualenv docs`: https://virtualenv.pypa.io From 67e11ae1d895c5b0488872b091d7ef29527f74e3 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Wed, 29 Mar 2017 10:01:16 -0700 Subject: [PATCH 077/128] tests: deduplicate temporary directory code (#4078) (#4297) Introduce a test class to deduplicate temporary directory setup and teardown in testing code and update existing test code to use this new class. --- certbot/tests/account_test.py | 13 +++-- certbot/tests/cert_manager_test.py | 13 ++--- certbot/tests/cli_test.py | 7 +-- certbot/tests/crypto_util_test.py | 30 ++++++----- certbot/tests/display/completer_test.py | 20 +++----- certbot/tests/display/ops_test.py | 10 ++-- certbot/tests/main_test.py | 67 +++++++++++-------------- certbot/tests/renewal_test.py | 10 ++-- certbot/tests/storage_test.py | 10 ++-- certbot/tests/util.py | 11 ++++ certbot/tests/util_test.py | 62 +++++++++-------------- 11 files changed, 113 insertions(+), 140 deletions(-) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7d335b09b..11d14132d 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,6 @@ import json import os import shutil import stat -import tempfile import unittest import mock @@ -17,6 +16,8 @@ from certbot import errors from certbot.tests import util +from certbot.tests.util import TempDirTestCase + KEY = jose.JWKRSA.load(util.load_vector("rsa512_key_2.pem")) @@ -98,13 +99,14 @@ class AccountMemoryStorageTest(unittest.TestCase): self.assertEqual([account], self.storage.find_all()) -class AccountFileStorageTest(unittest.TestCase): +class AccountFileStorageTest(TempDirTestCase): """Tests for certbot.account.AccountFileStorage.""" def setUp(self): - self.tmp = tempfile.mkdtemp() + super(AccountFileStorageTest, self).setUp() + self.config = mock.MagicMock( - accounts_dir=os.path.join(self.tmp, "accounts")) + accounts_dir=os.path.join(self.tempdir, "accounts")) from certbot.account import AccountFileStorage self.storage = AccountFileStorage(self.config) @@ -118,9 +120,6 @@ class AccountFileStorageTest(unittest.TestCase): self.mock_client = mock.MagicMock() self.mock_client.directory.new_authz = new_authzr_uri - def tearDown(self): - shutil.rmtree(self.tmp) - def test_init_creates_dir(self): self.assertTrue(os.path.isdir(self.config.accounts_dir)) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 473970870..b5731fbd6 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -18,11 +18,14 @@ from certbot.storage import ALL_FOUR from certbot.tests import storage_test from certbot.tests import util as test_util -class BaseCertManagerTest(unittest.TestCase): +from certbot.tests.util import TempDirTestCase + + +class BaseCertManagerTest(TempDirTestCase): """Base class for setting up Cert Manager tests. """ def setUp(self): - self.tempdir = tempfile.mkdtemp() + super(BaseCertManagerTest, self).setUp() os.makedirs(os.path.join(self.tempdir, "renewal")) @@ -68,9 +71,6 @@ class BaseCertManagerTest(unittest.TestCase): config.write() return config - def tearDown(self): - shutil.rmtree(self.tempdir) - class UpdateLiveSymlinksTest(BaseCertManagerTest): """Tests for certbot.cert_manager.update_live_symlinks @@ -437,9 +437,6 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): self.config.write() self._write_out_ex_kinds() - def tearDown(self): - shutil.rmtree(self.tempdir) - @mock.patch('certbot.util.make_or_verify_dir') def test_find_duplicative_names(self, unused_makedir): from certbot.cert_manager import find_duplicative_certs diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 5f4a4e2c7..498bd309d 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -15,17 +15,18 @@ from certbot import constants from certbot import errors from certbot.plugins import disco +from certbot.tests.util import TempDirTestCase + PLUGINS = disco.PluginsRegistry.find_all() -class TestReadFile(unittest.TestCase): +class TestReadFile(TempDirTestCase): '''Test cli.read_file''' _multiprocess_can_split_ = True def test_read_file(self): - tmp_dir = tempfile.mkdtemp() - rel_test_path = os.path.relpath(os.path.join(tmp_dir, 'foo')) + rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo')) self.assertRaises( argparse.ArgumentTypeError, cli.read_file, rel_test_path) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index a832d0494..df5d9f0f6 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -1,8 +1,6 @@ """Tests for certbot.crypto_util.""" import logging import os -import shutil -import tempfile import unittest import OpenSSL @@ -22,18 +20,20 @@ CERT = test_util.load_vector('cert.pem') SAN_CERT = test_util.load_vector('cert-san.pem') -class InitSaveKeyTest(unittest.TestCase): +class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" def setUp(self): + super(InitSaveKeyTest, self).setUp() + logging.disable(logging.CRITICAL) zope.component.provideUtility( mock.Mock(strict_permissions=True, dry_run=False), interfaces.IConfig) - self.key_dir = tempfile.mkdtemp('key_dir') def tearDown(self): + super(InitSaveKeyTest, self).tearDown() + logging.disable(logging.NOTSET) - shutil.rmtree(self.key_dir) @classmethod def _call(cls, key_size, key_dir): @@ -43,10 +43,10 @@ class InitSaveKeyTest(unittest.TestCase): @mock.patch('certbot.crypto_util.make_key') def test_success(self, mock_make): mock_make.return_value = b'key_pem' - key = self._call(1024, self.key_dir) + key = self._call(1024, self.tempdir) self.assertEqual(key.pem, b'key_pem') self.assertTrue('key-certbot.pem' in key.file) - self.assertTrue(os.path.exists(os.path.join(self.key_dir, key.file))) + self.assertTrue(os.path.exists(os.path.join(self.tempdir, key.file))) @mock.patch('certbot.crypto_util.make_key') def test_success_dry_run(self, mock_make): @@ -54,27 +54,25 @@ class InitSaveKeyTest(unittest.TestCase): mock.Mock(strict_permissions=True, dry_run=True), interfaces.IConfig) mock_make.return_value = b'key_pem' - key = self._call(1024, self.key_dir) + key = self._call(1024, self.tempdir) self.assertEqual(key.pem, b'key_pem') self.assertTrue(key.file is None) @mock.patch('certbot.crypto_util.make_key') def test_key_failure(self, mock_make): mock_make.side_effect = ValueError - self.assertRaises(ValueError, self._call, 431, self.key_dir) + self.assertRaises(ValueError, self._call, 431, self.tempdir) -class InitSaveCSRTest(unittest.TestCase): +class InitSaveCSRTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_csr.""" def setUp(self): + super(InitSaveCSRTest, self).setUp() + zope.component.provideUtility( mock.Mock(strict_permissions=True, dry_run=False), interfaces.IConfig) - self.csr_dir = tempfile.mkdtemp('csr_dir') - - def tearDown(self): - shutil.rmtree(self.csr_dir) @mock.patch('certbot.crypto_util.make_csr') @mock.patch('certbot.crypto_util.util.make_or_verify_dir') @@ -84,7 +82,7 @@ class InitSaveCSRTest(unittest.TestCase): mock_csr.return_value = (b'csr_pem', b'csr_der') csr = init_save_csr( - mock.Mock(pem='dummy_key'), 'example.com', self.csr_dir, + mock.Mock(pem='dummy_key'), 'example.com', self.tempdir, 'csr-certbot.pem') self.assertEqual(csr.data, b'csr_der') @@ -101,7 +99,7 @@ class InitSaveCSRTest(unittest.TestCase): mock_csr.return_value = (b'csr_pem', b'csr_der') csr = init_save_csr( - mock.Mock(pem='dummy_key'), 'example.com', self.csr_dir, + mock.Mock(pem='dummy_key'), 'example.com', self.tempdir, 'csr-certbot.pem') self.assertEqual(csr.data, b'csr_der') diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 16805314c..333acf2b3 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,31 +1,30 @@ """Test certbot.display.completer.""" import os import readline -import shutil import string import sys -import tempfile import unittest import mock from six.moves import reload_module # pylint: disable=import-error +from certbot.tests.util import TempDirTestCase -class CompleterTest(unittest.TestCase): +class CompleterTest(TempDirTestCase): """Test certbot.display.completer.Completer.""" def setUp(self): - self.temp_dir = tempfile.mkdtemp() + super(CompleterTest, self).setUp() # directories must end with os.sep for completer to # search inside the directory for possible completions - if self.temp_dir[-1] != os.sep: - self.temp_dir += os.sep + if self.tempdir[-1] != os.sep: + self.tempdir += os.sep self.paths = [] # create some files and directories in temp_dir for c in string.ascii_lowercase: - path = os.path.join(self.temp_dir, c) + path = os.path.join(self.tempdir, c) self.paths.append(path) if ord(c) % 2: os.mkdir(path) @@ -33,21 +32,18 @@ class CompleterTest(unittest.TestCase): with open(path, 'w'): pass - def tearDown(self): - shutil.rmtree(self.temp_dir) - def test_complete(self): from certbot.display import completer my_completer = completer.Completer() num_paths = len(self.paths) for i in range(num_paths): - completion = my_completer.complete(self.temp_dir, i) + completion = my_completer.complete(self.tempdir, i) self.assertTrue(completion in self.paths) self.paths.remove(completion) self.assertFalse(self.paths) - completion = my_completer.complete(self.temp_dir, num_paths) + completion = my_completer.complete(self.tempdir, num_paths) self.assertEqual(completion, None) def test_import_error(self): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index f2a9b3d07..89cd9e43d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -2,7 +2,6 @@ """Test certbot.display.ops.""" import os import sys -import tempfile import unittest import mock @@ -87,18 +86,19 @@ class GetEmailTest(unittest.TestCase): self.assertTrue(invalid_txt in mock_input.call_args[0][0]) -class ChooseAccountTest(unittest.TestCase): +class ChooseAccountTest(test_util.TempDirTestCase): """Tests for certbot.display.ops.choose_account.""" def setUp(self): + super(ChooseAccountTest, self).setUp() + zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) - self.accounts_dir = tempfile.mkdtemp("accounts") - self.account_keys_dir = os.path.join(self.accounts_dir, "keys") + self.account_keys_dir = os.path.join(self.tempdir, "keys") os.makedirs(self.account_keys_dir, 0o700) self.config = mock.MagicMock( - accounts_dir=self.accounts_dir, + accounts_dir=self.tempdir, account_keys_dir=self.account_keys_dir, server="certbot-demo.org") self.key = KEY diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 5707231b7..02f241255 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -7,7 +7,6 @@ import mock import multiprocessing import os import shutil -import tempfile import traceback import unittest import datetime @@ -220,13 +219,14 @@ class FindDomainsOrCertnameTest(unittest.TestCase): (["one.com", "two.com"], "one.com")) -class RevokeTest(unittest.TestCase): +class RevokeTest(test_util.TempDirTestCase): """Tests for certbot.main.revoke.""" def setUp(self): - self.tempdir_path = tempfile.mkdtemp() - shutil.copy(CERT_PATH, self.tempdir_path) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir_path, + super(RevokeTest, self).setUp() + + shutil.copy(CERT_PATH, self.tempdir) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert.pem')) self.patches = [ @@ -252,7 +252,8 @@ class RevokeTest(unittest.TestCase): self.mock_determine_account.return_value = (self.acc, None) def tearDown(self): - shutil.rmtree(self.tempdir_path) + super(RevokeTest, self).tearDown() + for patch in self.patches: patch.stop() @@ -288,15 +289,14 @@ class RevokeTest(unittest.TestCase): self.mock_success_revoke.assert_not_called() -class SetupLogFileHandlerTest(unittest.TestCase): +class SetupLogFileHandlerTest(test_util.TempDirTestCase): """Tests for certbot.main.setup_log_file_handler.""" def setUp(self): - self.config = mock.Mock(spec_set=['logs_dir'], - logs_dir=tempfile.mkdtemp()) + super(SetupLogFileHandlerTest, self).setUp() - def tearDown(self): - shutil.rmtree(self.config.logs_dir) + self.config = mock.Mock(spec_set=['logs_dir'], + logs_dir=self.tempdir) def _call(self, *args, **kwargs): from certbot.main import setup_log_file_handler @@ -309,18 +309,17 @@ class SetupLogFileHandlerTest(unittest.TestCase): self.config, "test.log", "%s") -class SetupLoggingTest(unittest.TestCase): +class SetupLoggingTest(test_util.TempDirTestCase): """Tests for certbot.main.setup_logging.""" def setUp(self): + super(SetupLoggingTest, self).setUp() + self.config = mock.Mock( - logs_dir=tempfile.mkdtemp(), + logs_dir=self.tempdir, noninteractive_mode=False, quiet=False, verbose_count=constants.CLI_DEFAULTS['verbose_count']) - def tearDown(self): - shutil.rmtree(self.config.logs_dir) - @classmethod def _call(cls, *args, **kwargs): from certbot.main import setup_logging @@ -346,21 +345,15 @@ class SetupLoggingTest(unittest.TestCase): isinstance(cli_handler, colored_logging.StreamHandler)) -class MakeOrVerifyCoreDirTest(unittest.TestCase): +class MakeOrVerifyCoreDirTest(test_util.TempDirTestCase): """Tests for certbot.main.make_or_verify_core_dir.""" - def setUp(self): - self.dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.dir) - def _call(self, *args, **kwargs): from certbot.main import make_or_verify_core_dir return make_or_verify_core_dir(*args, **kwargs) def test_success(self): - new_dir = os.path.join(self.dir, 'new') + new_dir = os.path.join(self.tempdir, 'new') self._call(new_dir, 0o700, os.geteuid(), False) self.assertTrue(os.path.exists(new_dir)) @@ -368,7 +361,7 @@ class MakeOrVerifyCoreDirTest(unittest.TestCase): def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, - self.dir, 0o700, os.geteuid(), False) + self.tempdir, 0o700, os.geteuid(), False) class DetermineAccountTest(unittest.TestCase): @@ -441,24 +434,26 @@ class DetermineAccountTest(unittest.TestCase): self.assertEqual('other email', self.config.email) -class MainTest(unittest.TestCase): # pylint: disable=too-many-public-methods +class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-methods """Tests for different commands.""" def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - self.config_dir = os.path.join(self.tmp_dir, 'config') - self.work_dir = os.path.join(self.tmp_dir, 'work') - self.logs_dir = os.path.join(self.tmp_dir, 'logs') + super(MainTest, self).setUp() + + self.config_dir = os.path.join(self.tempdir, 'config') + self.work_dir = os.path.join(self.tempdir, 'work') + self.logs_dir = os.path.join(self.tempdir, 'logs') os.mkdir(self.logs_dir) self.standard_args = ['--config-dir', self.config_dir, '--work-dir', self.work_dir, '--logs-dir', self.logs_dir, '--text', - '--lock-path', os.path.join(self.tmp_dir, 'certbot.lock')] + '--lock-path', os.path.join(self.tempdir, 'certbot.lock')] def tearDown(self): # Reset globals in cli reload_module(cli) - shutil.rmtree(self.tmp_dir) + + super(MainTest, self).tearDown() def _call(self, args, stdout=None): "Run the cli with output streams and actual client mocked out" @@ -1310,15 +1305,13 @@ class TestHandleException(unittest.TestCase): traceback.format_exception_only(KeyboardInterrupt, interrupt))) -class TestAcquireFileLock(unittest.TestCase): +class TestAcquireFileLock(test_util.TempDirTestCase): """Test main.acquire_file_lock.""" def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.lock_path = os.path.join(self.tempdir, 'certbot.lock') + super(TestAcquireFileLock, self).setUp() - def tearDown(self): - shutil.rmtree(self.tempdir) + self.lock_path = os.path.join(self.tempdir, 'certbot.lock') @mock.patch('certbot.main.logger') def test_bad_path(self, mock_logger): diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index d97e05783..de3efe39c 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -2,8 +2,6 @@ import os import mock import unittest -import shutil -import tempfile from acme import challenges @@ -14,13 +12,11 @@ from certbot import storage from certbot.tests import util -class RenewalTest(unittest.TestCase): +class RenewalTest(util.TempDirTestCase): def setUp(self): - self.tmp_dir = tempfile.mkdtemp() - self.config_dir = os.path.join(self.tmp_dir, 'config') + super(RenewalTest, self).setUp() - def tearDown(self): - shutil.rmtree(self.tmp_dir) + self.config_dir = os.path.join(self.tempdir, 'config') @mock.patch('certbot.cli.set_by_cli') def test_ancient_webroot_renewal_conf(self, mock_set_by_cli): diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index f7bde012d..6635461aa 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -3,7 +3,6 @@ import datetime import os import shutil -import tempfile import unittest import configobj @@ -36,7 +35,7 @@ def fill_with_sample_data(rc_object): f.write(kind) -class BaseRenewableCertTest(unittest.TestCase): +class BaseRenewableCertTest(util.TempDirTestCase): """Base class for setting up Renewable Cert tests. .. note:: It may be required to write out self.config for @@ -47,7 +46,8 @@ class BaseRenewableCertTest(unittest.TestCase): def setUp(self): from certbot import storage - self.tempdir = tempfile.mkdtemp() + + super(BaseRenewableCertTest, self).setUp() self.cli_config = configuration.NamespaceConfig( namespace=mock.MagicMock( @@ -91,9 +91,6 @@ class BaseRenewableCertTest(unittest.TestCase): check.return_value = True self.test_rc = storage.RenewableCert(config.filename, self.cli_config) - def tearDown(self): - shutil.rmtree(self.tempdir) - def _write_out_kind(self, kind, ver, value=None): link = getattr(self.test_rc, kind) if os.path.lexists(link): @@ -798,6 +795,7 @@ class DeleteFilesTest(BaseRenewableCertTest): """Tests for certbot.storage.delete_files""" def setUp(self): super(DeleteFilesTest, self).setUp() + for kind in ALL_FOUR: kind_path = os.path.join(self.tempdir, "live", "example.org", kind + ".pem") diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 092807b56..d58834335 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -6,6 +6,7 @@ import os import pkg_resources import shutil +import tempfile import unittest from cryptography.hazmat.backends import default_backend @@ -230,3 +231,13 @@ def _assert_valid_call(*args, **kwargs): # pylint: disable=star-args display_util.assert_valid_call(*assert_args, **assert_kwargs) + + +class TempDirTestCase(unittest.TestCase): + """Base test class which sets up and tears down a temporary directory""" + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tempdir) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 13a91dfb8..480120772 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -2,9 +2,7 @@ import argparse import errno import os -import shutil import stat -import tempfile import unittest import mock @@ -75,7 +73,7 @@ class ExeExistsTest(unittest.TestCase): self.assertFalse(self._call("exe")) -class MakeOrVerifyDirTest(unittest.TestCase): +class MakeOrVerifyDirTest(test_util.TempDirTestCase): """Tests for certbot.util.make_or_verify_dir. Note that it is not possible to test for a wrong directory owner, @@ -84,21 +82,19 @@ class MakeOrVerifyDirTest(unittest.TestCase): """ def setUp(self): - self.root_path = tempfile.mkdtemp() - self.path = os.path.join(self.root_path, "foo") + super(MakeOrVerifyDirTest, self).setUp() + + self.path = os.path.join(self.tempdir, "foo") os.mkdir(self.path, 0o400) self.uid = os.getuid() - def tearDown(self): - shutil.rmtree(self.root_path, ignore_errors=True) - def _call(self, directory, mode): from certbot.util import make_or_verify_dir return make_or_verify_dir(directory, mode, self.uid, strict=True) def test_creates_dir_when_missing(self): - path = os.path.join(self.root_path, "bar") + path = os.path.join(self.tempdir, "bar") self._call(path, 0o650) self.assertTrue(os.path.isdir(path)) self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o650) @@ -116,7 +112,7 @@ class MakeOrVerifyDirTest(unittest.TestCase): self.assertRaises(OSError, self._call, "bar", 12312312) -class CheckPermissionsTest(unittest.TestCase): +class CheckPermissionsTest(test_util.TempDirTestCase): """Tests for certbot.util.check_permissions. Note that it is not possible to test for a wrong file owner, @@ -125,34 +121,30 @@ class CheckPermissionsTest(unittest.TestCase): """ def setUp(self): - _, self.path = tempfile.mkstemp() - self.uid = os.getuid() + super(CheckPermissionsTest, self).setUp() - def tearDown(self): - os.remove(self.path) + self.uid = os.getuid() def _call(self, mode): from certbot.util import check_permissions - return check_permissions(self.path, mode, self.uid) + return check_permissions(self.tempdir, mode, self.uid) def test_ok_mode(self): - os.chmod(self.path, 0o600) + os.chmod(self.tempdir, 0o600) self.assertTrue(self._call(0o600)) def test_wrong_mode(self): - os.chmod(self.path, 0o400) + os.chmod(self.tempdir, 0o400) self.assertFalse(self._call(0o600)) -class UniqueFileTest(unittest.TestCase): +class UniqueFileTest(test_util.TempDirTestCase): """Tests for certbot.util.unique_file.""" def setUp(self): - self.root_path = tempfile.mkdtemp() - self.default_name = os.path.join(self.root_path, "foo.txt") + super(UniqueFileTest, self).setUp() - def tearDown(self): - shutil.rmtree(self.root_path, ignore_errors=True) + self.default_name = os.path.join(self.tempdir, "foo.txt") def _call(self, mode=0o600): from certbot.util import unique_file @@ -177,9 +169,9 @@ class UniqueFileTest(unittest.TestCase): self.assertNotEqual(name1, name3) self.assertNotEqual(name2, name3) - self.assertEqual(os.path.dirname(name1), self.root_path) - self.assertEqual(os.path.dirname(name2), self.root_path) - self.assertEqual(os.path.dirname(name3), self.root_path) + self.assertEqual(os.path.dirname(name1), self.tempdir) + self.assertEqual(os.path.dirname(name2), self.tempdir) + self.assertEqual(os.path.dirname(name3), self.tempdir) basename1 = os.path.basename(name2) self.assertTrue(basename1.endswith("foo.txt")) @@ -196,23 +188,17 @@ except NameError: file_type = io.TextIOWrapper # type: ignore -class UniqueLineageNameTest(unittest.TestCase): +class UniqueLineageNameTest(test_util.TempDirTestCase): """Tests for certbot.util.unique_lineage_name.""" - def setUp(self): - self.root_path = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.root_path, ignore_errors=True) - def _call(self, filename, mode=0o777): from certbot.util import unique_lineage_name - return unique_lineage_name(self.root_path, filename, mode) + return unique_lineage_name(self.tempdir, filename, mode) def test_basic(self): f, path = self._call("wow") self.assertTrue(isinstance(f, file_type)) - self.assertEqual(os.path.join(self.root_path, "wow.conf"), path) + self.assertEqual(os.path.join(self.tempdir, "wow.conf"), path) def test_multiple(self): for _ in six.moves.range(10): @@ -237,15 +223,13 @@ class UniqueLineageNameTest(unittest.TestCase): self.assertRaises(OSError, self._call, "wow") -class SafelyRemoveTest(unittest.TestCase): +class SafelyRemoveTest(test_util.TempDirTestCase): """Tests for certbot.util.safely_remove.""" def setUp(self): - self.tmp = tempfile.mkdtemp() - self.path = os.path.join(self.tmp, "foo") + super(SafelyRemoveTest, self).setUp() - def tearDown(self): - shutil.rmtree(self.tmp) + self.path = os.path.join(self.tempdir, "foo") def _call(self): from certbot.util import safely_remove From 6fb78dab671f723f6b2b4a2d0846fbec0e9634c3 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 30 Mar 2017 04:34:09 +0800 Subject: [PATCH 078/128] Fix Docker IP detection with different ifconfig output formats (#4376) --- tests/boulder-fetch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index ef61fe3f5..d9a979667 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -12,5 +12,7 @@ fi cd ${BOULDERPATH} FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') +[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) +[ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml docker-compose up -d From e73c29374a59d11caaab8d7e7db875964769715a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 29 Mar 2017 14:32:02 -0700 Subject: [PATCH 079/128] Change to source. --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index cc24ad898..d6b77facf 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -39,7 +39,7 @@ Then in each shell where you're working on the client, do: .. code-block:: shell source ./venv/bin/activate - . tests/integration/_common.sh + source tests/integration/_common.sh After that, your shell will be using the virtual environment, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that From d5f1edf2bb85cba4c0de18722b29e080fb43d4d9 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 29 Mar 2017 16:48:08 -0700 Subject: [PATCH 080/128] Dump Boulder logs on integration test failures. (#4442) Might help debug #4363. Also: make "bash" vs "sh" explicit move the paranoia flags (-ex) from the shebang into the body add -u (fail on unset variables) change _common to work with -u remove some env vars that were no longer used remove shebang from _common.sh because it's meant to be sourced, not run --- tests/boulder-integration.sh | 15 +++++++++++---- tests/integration/_common.sh | 13 ++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index ca6f48e60..6612b2e67 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -1,4 +1,4 @@ -#!/bin/sh -xe +#!/bin/bash # Simple integration test. Make sure to activate virtualenv beforehand # (source venv/bin/activate) and that you are running Boulder test # instance (see ./boulder-fetch.sh). @@ -8,12 +8,11 @@ # # Note: this script is called by Boulder integration test suite! +set -eux + . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx -export GOPATH="${GOPATH:-/tmp/go}" -export PATH="$GOPATH/bin:$PATH" - if [ `uname` = "Darwin" ];then readlink="greadlink" else @@ -27,6 +26,14 @@ cleanup_and_exit() { echo Kill server subprocess, left running by abnormal exit kill $SERVER_STILL_RUNNING fi + # Dump boulder logs in case they contain useful debugging information. + : "------------------ ------------------ ------------------" + : "------------------ begin boulder logs ------------------" + : "------------------ ------------------ ------------------" + docker logs boulder_boulder_1 + : "------------------ ------------------ ------------------" + : "------------------ end boulder logs ------------------" + : "------------------ ------------------ ------------------" exit $EXIT_STATUS } diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 9b44631d4..8d4baff95 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -1,12 +1,7 @@ -#!/bin/sh - -if [ "xxx$root" = "xxx" ]; -then - # The -t is required on macOS. It provides a template file path for - # the kernel to use. - root="$(mktemp -d -t leitXXXX)" - echo "Root integration tests directory: $root" -fi +# The -t is required on macOS. It provides a template file path for +# the kernel to use. +root=${root:-$(mktemp -d -t leitXXXX)} +echo "Root integration tests directory: $root" store_flags="--config-dir $root/conf --work-dir $root/work" store_flags="$store_flags --logs-dir $root/logs" tls_sni_01_port=5001 From 52e22b22e5120d43f0a3b220d5ba48fca15cafe2 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 30 Mar 2017 07:47:36 -0700 Subject: [PATCH 081/128] Add additional Nginx parsing test case (#4440) --- certbot-nginx/certbot_nginx/tests/nginxparser_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index 0a8dbd100..dd31ebac8 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -346,6 +346,8 @@ class TestRawNginxParser(unittest.TestCase): self.assertEqual(parsed, [['directive', '$var']]) self.assertRaises(ParseException, loads, "server {server_name test.com};") self.assertRaises(ParseException, loads, "directive ${var};") + self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']]) + self.assertRaises(ParseException, loads, "blag${dfgdf{g};") class TestUnspacedList(unittest.TestCase): From d09bde972ae842c97263414a1c0cf54a721329ce Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 30 Mar 2017 15:28:24 -0700 Subject: [PATCH 082/128] Remove unused default parameter (#4447) * Remove unnecessary, nonexistent default --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 225702251..b36f638ab 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -820,7 +820,7 @@ class NginxConfigurator(common.Plugin): self.restart() -def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): +def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. .. todo:: Nginx restart is fatal if the configuration references From a542fcd019e1e3eb99dd9872aaa00ca99450800c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 30 Mar 2017 15:47:31 -0700 Subject: [PATCH 083/128] Revert "Add a global lock file to Certbot (#4369)" (#4445) This reverts commit 32122cfa21fe7ba43189658158949cfe16dc6717. --- certbot/cli.py | 2 - certbot/constants.py | 1 - certbot/interfaces.py | 3 -- certbot/main.py | 53 +------------------ certbot/tests/main_test.py | 51 +----------------- letsencrypt-auto-source/letsencrypt-auto | 6 --- .../pieces/letsencrypt-auto-requirements.txt | 6 --- setup.py | 1 - 8 files changed, 2 insertions(+), 121 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 0faf4a7c6..6217baa27 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1167,8 +1167,6 @@ def _paths_parser(helpful): help="Logs directory.") add("paths", "--server", default=flag_default("server"), help=config_help("server")) - add("paths", "--lock-path", default=flag_default("lock_path"), - help=config_help('lock_path')) def _plugins_parsing(helpful, plugins): diff --git a/certbot/constants.py b/certbot/constants.py index 24cf89199..382b0afb3 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -33,7 +33,6 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", strict_permissions=False, - lock_path="/tmp/.certbot.lock", debug_challenges=False, ) STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 33bde9430..213992993 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -222,9 +222,6 @@ class IConfig(zope.interface.Interface): key_dir = zope.interface.Attribute("Keys storage.") temp_checkpoint_dir = zope.interface.Attribute( "Temporary checkpoint directory.") - lock_path = zope.interface.Attribute( - "Path to the lock file used to prevent multiple instances of " - "Certbot from modifying your server's configuration at once.") no_verify_ssl = zope.interface.Attribute( "Disable verification of the ACME server's certificate.") diff --git a/certbot/main.py b/certbot/main.py index c6e61fbf9..ab2204428 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -8,7 +8,6 @@ import sys import time import traceback -import fasteners import zope.component from acme import jose @@ -867,56 +866,6 @@ def _post_logging_setup(config, plugins, cli_args): logger.debug("Discovered plugins: %r", plugins) -def acquire_file_lock(lock_path): - """Obtain a lock on the file at the specified path. - - :param str lock_path: path to the file to be locked - - :returns: lock file object representing the acquired lock - :rtype: fasteners.InterProcessLock - - :raises .Error: if the lock is held by another process - - """ - lock = fasteners.InterProcessLock(lock_path) - logger.debug("Attempting to acquire lock file %s", lock_path) - - try: - lock.acquire(blocking=False) - except IOError as err: - logger.debug(err) - logger.warning( - "Unable to access lock file %s. You should set --lock-file " - "to a writeable path to ensure multiple instances of " - "Certbot don't attempt modify your configuration " - "simultaneously.", lock_path) - else: - if not lock.acquired: - raise errors.Error( - "Another instance of Certbot is already running.") - - return lock - - -def _run_subcommand(config, plugins): - """Executes the Certbot subcommand specified in the configuration. - - :param .IConfig config: parsed configuration object - :param .PluginsRegistry plugins: available plugins - - :returns: return value from the specified subcommand - :rtype: str or int - - """ - lock = acquire_file_lock(config.lock_path) - - try: - return config.func(config, plugins) - finally: - if lock.acquired: - lock.release() - - def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" sys.excepthook = functools.partial(_handle_exception, config=None) @@ -944,7 +893,7 @@ def main(cli_args=sys.argv[1:]): zope.component.provideUtility(report) atexit.register(report.atexit_print_messages) - return _run_subcommand(config, plugins) + return config.func(config, plugins) if __name__ == "__main__": diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 02f241255..5d456bc3d 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -4,7 +4,6 @@ from __future__ import print_function import itertools import mock -import multiprocessing import os import shutil import traceback @@ -446,8 +445,7 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me os.mkdir(self.logs_dir) self.standard_args = ['--config-dir', self.config_dir, '--work-dir', self.work_dir, - '--logs-dir', self.logs_dir, '--text', - '--lock-path', os.path.join(self.tempdir, 'certbot.lock')] + '--logs-dir', self.logs_dir, '--text'] def tearDown(self): # Reset globals in cli @@ -1305,52 +1303,5 @@ class TestHandleException(unittest.TestCase): traceback.format_exception_only(KeyboardInterrupt, interrupt))) -class TestAcquireFileLock(test_util.TempDirTestCase): - """Test main.acquire_file_lock.""" - - def setUp(self): - super(TestAcquireFileLock, self).setUp() - - self.lock_path = os.path.join(self.tempdir, 'certbot.lock') - - @mock.patch('certbot.main.logger') - def test_bad_path(self, mock_logger): - lock = main.acquire_file_lock(os.getcwd()) - self.assertTrue(mock_logger.warning.called) - self.assertFalse(lock.acquired) - - def test_held_lock(self): - # start child and wait for it to grab the lock - cv = multiprocessing.Condition() - cv.acquire() - child_args = (cv, self.lock_path,) - child = multiprocessing.Process(target=_hold_lock, args=child_args) - child.start() - cv.wait() - - # assert we can't grab lock and terminate the child - self.assertRaises(errors.Error, main.acquire_file_lock, self.lock_path) - cv.notify() - cv.release() - child.join() - self.assertEqual(child.exitcode, 0) - - -def _hold_lock(cv, lock_path): - """Acquire a file lock at lock_path and wait to release it. - - :param multiprocessing.Condition cv: condition for syncronization - :param str lock_path: path to the file lock - - """ - import fasteners - lock = fasteners.InterProcessLock(lock_path) - lock.acquire() - cv.acquire() - cv.notify() - cv.wait() - lock.release() - - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 475d57fee..35c7a530c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -727,9 +727,6 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -fasteners==0.14.1 \ - --hash=sha256:564a115ff9698767df401efca29620cbb1a1c2146b7095ebd304b79cc5807a7c \ - --hash=sha256:427c76773fe036ddfa41e57d89086ea03111bbac57c55fc55f3006d027107e18 funcsigs==0.4 \ --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 @@ -742,9 +739,6 @@ ipaddress==1.0.16 \ linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -monotonic==1.3 \ - --hash=sha256:a8c7690953546c6bc8a4f05d347718db50de1225b29f4b9f346c0c6f19bdc286 \ - --hash=sha256:2b469e2d7dd403f7f7f79227fe5ad551ee1e76f8bb300ae935209884b93c7c1b ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f parsedatetime==2.1 \ diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index fbf416d66..d70d24e2a 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -65,9 +65,6 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -fasteners==0.14.1 \ - --hash=sha256:564a115ff9698767df401efca29620cbb1a1c2146b7095ebd304b79cc5807a7c \ - --hash=sha256:427c76773fe036ddfa41e57d89086ea03111bbac57c55fc55f3006d027107e18 funcsigs==0.4 \ --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 @@ -80,9 +77,6 @@ ipaddress==1.0.16 \ linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -monotonic==1.3 \ - --hash=sha256:a8c7690953546c6bc8a4f05d347718db50de1225b29f4b9f346c0c6f19bdc286 \ - --hash=sha256:2b469e2d7dd403f7f7f79227fe5ad551ee1e76f8bb300ae935209884b93c7c1b ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f parsedatetime==2.1 \ diff --git a/setup.py b/setup.py index 59f6dbd9e..0e8d19a22 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,6 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate - 'fasteners', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'PyOpenSSL', From e194e0dd5f3fdde78c0342df4b09c7a982249d56 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 30 Mar 2017 16:17:57 -0700 Subject: [PATCH 084/128] Refactoring for better logging (#4444) * Move colored_logging.py to log.py * Add atexit.register code to util * Add tests for atexit_register * Copy except_hook to log * Add pre_arg_setup * move setup_log_file_handler to log.py * Add post_arg_setup * move changes to main * Undo changes to MainTest * s/pre_arg_setup/pre_arg_parse_setup * s/post_arg_setup/post_arg_parse_setup --- certbot/colored_logging.py | 45 ------ certbot/log.py | 180 +++++++++++++++++++++++ certbot/main.py | 163 ++------------------- certbot/reporter.py | 19 --- certbot/tests/colored_logging_test.py | 41 ------ certbot/tests/log_test.py | 199 ++++++++++++++++++++++++++ certbot/tests/main_test.py | 130 ----------------- certbot/tests/reporter_test.py | 12 -- certbot/tests/util_test.py | 51 +++++++ certbot/util.py | 45 ++++++ 10 files changed, 486 insertions(+), 399 deletions(-) delete mode 100644 certbot/colored_logging.py create mode 100644 certbot/log.py delete mode 100644 certbot/tests/colored_logging_test.py create mode 100644 certbot/tests/log_test.py diff --git a/certbot/colored_logging.py b/certbot/colored_logging.py deleted file mode 100644 index 93bf3a55a..000000000 --- a/certbot/colored_logging.py +++ /dev/null @@ -1,45 +0,0 @@ -"""A formatter and StreamHandler for colorizing logging output.""" -import logging -import sys - -from certbot import util - - -class StreamHandler(logging.StreamHandler): - """Sends colored logging output to a stream. - - If the specified stream is not a tty, the class works like the - standard logging.StreamHandler. Default red_level is logging.WARNING. - - :ivar bool colored: True if output should be colored - :ivar bool red_level: The level at which to output - - """ - - def __init__(self, stream=None): - if sys.version_info < (2, 7): - # pragma: no cover - # pylint: disable=non-parent-init-called - logging.StreamHandler.__init__(self, stream) - else: - super(StreamHandler, self).__init__(stream) - self.colored = (sys.stderr.isatty() if stream is None else - stream.isatty()) - self.red_level = logging.WARNING - - def format(self, record): - """Formats the string representation of record. - - :param logging.LogRecord record: Record to be formatted - - :returns: Formatted, string representation of record - :rtype: str - - """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(StreamHandler, self).format(record)) - if self.colored and record.levelno >= self.red_level: - return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) - else: - return out diff --git a/certbot/log.py b/certbot/log.py new file mode 100644 index 000000000..92b35ed51 --- /dev/null +++ b/certbot/log.py @@ -0,0 +1,180 @@ +"""Logging utilities for Certbot.""" +from __future__ import print_function +import functools +import logging +import os +import sys +import time +import traceback + +from acme import messages + +from certbot import cli +from certbot import constants +from certbot import errors +from certbot import util + +# Logging format +CLI_FMT = "%(message)s" +FILE_FMT = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + + +logger = logging.getLogger(__name__) + + +def pre_arg_parse_setup(): + """Ensures fatal exceptions are logged and reported to the user.""" + sys.excepthook = functools.partial(except_hook, config=None) + + +def post_arg_parse_setup(config): + """Setup logging after command line arguments are parsed. + + :param certbot.interface.IConfig config: Configuration object + + """ + file_handler, file_path = setup_log_file_handler( + config, 'letsencrypt.log', FILE_FMT) + + if config.quiet: + level = constants.QUIET_LOGGING_LEVEL + else: + level = -config.verbose_count * 10 + stderr_handler = ColoredStreamHandler() + stderr_handler.setFormatter(logging.Formatter(CLI_FMT)) + stderr_handler.setLevel(level) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG) # send all records to handlers + root_logger.addHandler(stderr_handler) + root_logger.addHandler(file_handler) + + logger.debug('Root logging level set at %d', level) + logger.info('Saving debug log to %s', file_path) + + sys.excepthook = functools.partial(except_hook, config=config) + + +def setup_log_file_handler(config, logfile, fmt): + """Setup file debug logging. + + :param certbot.interface.IConfig config: Configuration object + :param str logfile: basename for the log file + :param str fmt: logging format string + + :returns: file handler and absolute path to the log file + :rtype: tuple + + """ + # TODO: logs might contain sensitive data such as contents of the + # private key! #525 + util.make_or_verify_core_dir( + config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) + log_file_path = os.path.join(config.logs_dir, logfile) + try: + handler = logging.handlers.RotatingFileHandler( + log_file_path, maxBytes=2 ** 20, backupCount=1000) + except IOError as error: + raise errors.Error(util.PERM_ERR_FMT.format(error)) + # rotate on each invocation, rollover only possible when maxBytes + # is nonzero and backupCount is nonzero, so we set maxBytes as big + # as possible not to overrun in single CLI invocation (1MB). + handler.doRollover() # TODO: creates empty letsencrypt.log.1 file + handler.setLevel(logging.DEBUG) + handler_formatter = logging.Formatter(fmt=fmt) + handler_formatter.converter = time.gmtime # don't use localtime + handler.setFormatter(handler_formatter) + return handler, log_file_path + + +class ColoredStreamHandler(logging.StreamHandler): + """Sends colored logging output to a stream. + + If the specified stream is not a tty, the class works like the + standard logging.StreamHandler. Default red_level is logging.WARNING. + + :ivar bool colored: True if output should be colored + :ivar bool red_level: The level at which to output + + """ + + def __init__(self, stream=None): + if sys.version_info < (2, 7): + # pragma: no cover + # pylint: disable=non-parent-init-called + logging.StreamHandler.__init__(self, stream) + else: + super(ColoredStreamHandler, self).__init__(stream) + self.colored = (sys.stderr.isatty() if stream is None else + stream.isatty()) + self.red_level = logging.WARNING + + def format(self, record): + """Formats the string representation of record. + + :param logging.LogRecord record: Record to be formatted + + :returns: Formatted, string representation of record + :rtype: str + + """ + out = (logging.StreamHandler.format(self, record) + if sys.version_info < (2, 7) + else super(ColoredStreamHandler, self).format(record)) + if self.colored and record.levelno >= self.red_level: + return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) + else: + return out + + +def except_hook(exc_type, exc_value, trace, config): + """Logs exceptions and reports them to the user. + + Config is used to determine how to display exceptions to the user. In + general, if config.debug is True, then the full exception and traceback is + shown to the user, otherwise it is suppressed. If config itself is None, + then the traceback and exception is attempted to be written to a logfile. + If this is successful, the traceback is suppressed, otherwise it is shown + to the user. sys.exit is always called with a nonzero status. + + """ + tb_str = "".join(traceback.format_exception(exc_type, exc_value, trace)) + logger.debug("Exiting abnormally:%s%s", os.linesep, tb_str) + + if issubclass(exc_type, Exception) and (config is None or not config.debug): + if config is None: + logfile = "certbot.log" + try: + with open(logfile, "w") as logfd: + traceback.print_exception( + exc_type, exc_value, trace, file=logfd) + assert "--debug" not in sys.argv # config is None if this explodes + except: # pylint: disable=bare-except + sys.exit(tb_str) + if "--debug" in sys.argv: + sys.exit(tb_str) + + if issubclass(exc_type, errors.Error): + sys.exit(exc_value) + else: + # Here we're passing a client or ACME error out to the client at the shell + # Tell the user a bit about what happened, without overwhelming + # them with a full traceback + err = traceback.format_exception_only(exc_type, exc_value)[0] + # Typical error from the ACME module: + # acme.messages.Error: urn:ietf:params:acme:error:malformed :: The + # request message was malformed :: Error creating new registration + # :: Validation of contact mailto:none@longrandomstring.biz failed: + # Server failure at resolver + if (messages.is_acme_error(err) and ":: " in err and + config.verbose_count <= cli.flag_default("verbose_count")): + # prune ACME error code, we have a human description + _code, _sep, err = err.partition(":: ") + msg = "An unexpected error occurred:\n" + err + "Please see the " + if config is None: + msg += "logfile '{0}' for more details.".format(logfile) + else: + msg += "logfiles in {0} for more details.".format(config.logs_dir) + sys.exit(msg) + else: + sys.exit(tb_str) diff --git a/certbot/main.py b/certbot/main.py index ab2204428..090479d20 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,47 +1,37 @@ """Certbot main entry point.""" from __future__ import print_function -import atexit -import functools import logging.handlers import os import sys -import time -import traceback import zope.component from acme import jose -from acme import messages from acme import errors as acme_errors import certbot from certbot import account from certbot import cert_manager -from certbot import client from certbot import cli -from certbot import crypto_util -from certbot import colored_logging +from certbot import client from certbot import configuration from certbot import constants +from certbot import crypto_util from certbot import eff from certbot import errors from certbot import hooks from certbot import interfaces -from certbot import util -from certbot import reporter +from certbot import log from certbot import renewal +from certbot import reporter +from certbot import util from certbot.display import util as display_util, ops as display_ops from certbot.plugins import disco as plugins_disco from certbot.plugins import selection as plug_sel -_PERM_ERR_FMT = os.linesep.join(( - "The following error was encountered:", "{0}", - "If running as non-root, set --config-dir, " - "--work-dir, and --logs-dir to writeable paths.")) - USER_CANCELLED = ("User chose to cancel the operation and may " "reinvoke the client.") @@ -704,138 +694,11 @@ def renew(config, unused_plugins): hooks.run_saved_post_hooks() -def setup_log_file_handler(config, logfile, fmt): - """Setup file debug logging.""" - log_file_path = os.path.join(config.logs_dir, logfile) - try: - handler = logging.handlers.RotatingFileHandler( - log_file_path, maxBytes=2 ** 20, backupCount=1000) - except IOError as error: - raise errors.Error(_PERM_ERR_FMT.format(error)) - # rotate on each invocation, rollover only possible when maxBytes - # is nonzero and backupCount is nonzero, so we set maxBytes as big - # as possible not to overrun in single CLI invocation (1MB). - handler.doRollover() # TODO: creates empty letsencrypt.log.1 file - handler.setLevel(logging.DEBUG) - handler_formatter = logging.Formatter(fmt=fmt) - handler_formatter.converter = time.gmtime # don't use localtime - handler.setFormatter(handler_formatter) - return handler, log_file_path - - -def _cli_log_handler(level, fmt): - handler = colored_logging.StreamHandler() - handler.setFormatter(logging.Formatter(fmt)) - handler.setLevel(level) - return handler - - -def setup_logging(config): - """Sets up logging to logfiles and the terminal. - - :param certbot.interface.IConfig config: Configuration object - - """ - cli_fmt = "%(message)s" - file_fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" - logfile = "letsencrypt.log" - if config.quiet: - level = constants.QUIET_LOGGING_LEVEL - else: - level = -config.verbose_count * 10 - file_handler, log_file_path = setup_log_file_handler( - config, logfile=logfile, fmt=file_fmt) - cli_handler = _cli_log_handler(level, cli_fmt) - - # TODO: use fileConfig? - - root_logger = logging.getLogger() - root_logger.setLevel(logging.DEBUG) # send all records to handlers - root_logger.addHandler(cli_handler) - root_logger.addHandler(file_handler) - - logger.debug("Root logging level set at %d", level) - logger.info("Saving debug log to %s", log_file_path) - - -def _handle_exception(exc_type, exc_value, trace, config): - """Logs exceptions and reports them to the user. - - Config is used to determine how to display exceptions to the user. In - general, if config.debug is True, then the full exception and traceback is - shown to the user, otherwise it is suppressed. If config itself is None, - then the traceback and exception is attempted to be written to a logfile. - If this is successful, the traceback is suppressed, otherwise it is shown - to the user. sys.exit is always called with a nonzero status. - - """ - tb_str = "".join(traceback.format_exception(exc_type, exc_value, trace)) - logger.debug("Exiting abnormally:%s%s", os.linesep, tb_str) - - if issubclass(exc_type, Exception) and (config is None or not config.debug): - if config is None: - logfile = "certbot.log" - try: - with open(logfile, "w") as logfd: - traceback.print_exception( - exc_type, exc_value, trace, file=logfd) - assert "--debug" not in sys.argv # config is None if this explodes - except: # pylint: disable=bare-except - sys.exit(tb_str) - if "--debug" in sys.argv: - sys.exit(tb_str) - - if issubclass(exc_type, errors.Error): - sys.exit(exc_value) - else: - # Here we're passing a client or ACME error out to the client at the shell - # Tell the user a bit about what happened, without overwhelming - # them with a full traceback - err = traceback.format_exception_only(exc_type, exc_value)[0] - # Typical error from the ACME module: - # acme.messages.Error: urn:ietf:params:acme:error:malformed :: The - # request message was malformed :: Error creating new registration - # :: Validation of contact mailto:none@longrandomstring.biz failed: - # Server failure at resolver - if (messages.is_acme_error(err) and ":: " in err and - config.verbose_count <= cli.flag_default("verbose_count")): - # prune ACME error code, we have a human description - _code, _sep, err = err.partition(":: ") - msg = "An unexpected error occurred:\n" + err + "Please see the " - if config is None: - msg += "logfile '{0}' for more details.".format(logfile) - else: - msg += "logfiles in {0} for more details.".format(config.logs_dir) - sys.exit(msg) - else: - sys.exit(tb_str) - - -def make_or_verify_core_dir(directory, mode, uid, strict): - """Make sure directory exists with proper permissions. - - :param str directory: Path to a directory. - :param int mode: Directory mode. - :param int uid: Directory owner. - :param bool strict: require directory to be owned by current user - - :raises .errors.Error: if the directory cannot be made or verified - - """ - try: - util.make_or_verify_dir(directory, mode, uid, strict) - except OSError as error: - raise errors.Error(_PERM_ERR_FMT.format(error)) - def make_or_verify_needed_dirs(config): - """Create or verify existence of config, work, or logs directories""" - make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, + """Create or verify existence of config and work directories""" + util.make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) - make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) - # TODO: logs might contain sensitive data such as contents of the - # private key! #525 - make_or_verify_core_dir(config.logs_dir, 0o700, + util.make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) @@ -868,7 +731,7 @@ def _post_logging_setup(config, plugins, cli_args): def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" - sys.excepthook = functools.partial(_handle_exception, config=None) + log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() # note: arg parser internally handles --help (and exits afterwards) @@ -878,20 +741,16 @@ def main(cli_args=sys.argv[1:]): make_or_verify_needed_dirs(config) - # Setup logging ASAP, otherwise "No handlers could be found for - # logger ..." TODO: this should be done before plugins discovery - setup_logging(config) + log.post_arg_parse_setup(config) _post_logging_setup(config, plugins, cli_args) - sys.excepthook = functools.partial(_handle_exception, config=config) - set_displayer(config) # Reporter report = reporter.Reporter(config) zope.component.provideUtility(report) - atexit.register(report.atexit_print_messages) + util.atexit_register(report.print_messages) return config.func(config, plugins) diff --git a/certbot/reporter.py b/certbot/reporter.py index f836dbc8c..e0063d8e5 100644 --- a/certbot/reporter.py +++ b/certbot/reporter.py @@ -3,7 +3,6 @@ from __future__ import print_function import collections import logging -import os import sys import textwrap @@ -16,11 +15,6 @@ from certbot import util logger = logging.getLogger(__name__) -# Store the pid of the process that first imported this module so that -# atexit_print_messages side-effects such as error reporting can be limited to -# this process and not any fork()'d children. -INITIAL_PID = os.getpid() - @zope.interface.implementer(interfaces.IReporter) class Reporter(object): @@ -60,19 +54,6 @@ class Reporter(object): self.messages.put(self._msg_type(priority, msg, on_crash)) logger.debug("Reporting to user: %s", msg) - def atexit_print_messages(self, pid=None): - """Function to be registered with atexit to print messages. - - :param int pid: Process ID - - """ - if pid is None: - pid = INITIAL_PID - # This ensures that messages are only printed from the process that - # created the Reporter. - if pid == os.getpid(): - self.print_messages() - def print_messages(self): """Prints messages to the user and clears the message queue. diff --git a/certbot/tests/colored_logging_test.py b/certbot/tests/colored_logging_test.py deleted file mode 100644 index 0a7929561..000000000 --- a/certbot/tests/colored_logging_test.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tests for certbot.colored_logging.""" -import logging -import unittest - -import six - -from certbot import util - - -class StreamHandlerTest(unittest.TestCase): - """Tests for certbot.colored_logging.""" - - def setUp(self): - from certbot import colored_logging - - self.stream = six.StringIO() - self.stream.isatty = lambda: True - self.handler = colored_logging.StreamHandler(self.stream) - - self.logger = logging.getLogger() - self.logger.setLevel(logging.DEBUG) - self.logger.addHandler(self.handler) - - def test_format(self): - msg = 'I did a thing' - self.logger.debug(msg) - self.assertEqual(self.stream.getvalue(), '{0}\n'.format(msg)) - - def test_format_and_red_level(self): - msg = 'I did another thing' - self.handler.red_level = logging.DEBUG - self.logger.debug(msg) - - self.assertEqual(self.stream.getvalue(), - '{0}{1}{2}\n'.format(util.ANSI_SGR_RED, - msg, - util.ANSI_SGR_RESET)) - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py new file mode 100644 index 000000000..745147f0d --- /dev/null +++ b/certbot/tests/log_test.py @@ -0,0 +1,199 @@ +"""Tests for certbot.log.""" +import logging +import traceback +import logging.handlers +import os +import sys +import time +import unittest + +import mock +import six + +from acme import messages + +from certbot import constants +from certbot import errors +from certbot import util +from certbot.tests import util as test_util + + +class PreArgParseSetupTest(unittest.TestCase): + """Tests for certbot.log.pre_arg_parse_setup.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.log import pre_arg_parse_setup + return pre_arg_parse_setup(*args, **kwargs) + + def test_it(self): + with mock.patch('certbot.log.except_hook') as mock_except_hook: + with mock.patch('certbot.log.sys') as mock_sys: + self._call() + + mock_sys.excepthook(1, 2, 3) + mock_except_hook.assert_called_once_with(1, 2, 3, config=None) + + +class PostArgParseSetupTest(test_util.TempDirTestCase): + """Tests for certbot.log.post_arg_parse_setup.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.log import post_arg_parse_setup + return post_arg_parse_setup(*args, **kwargs) + + def setUp(self): + super(PostArgParseSetupTest, self).setUp() + self.config = mock.MagicMock( + logs_dir=self.tempdir, quiet=False, + verbose_count=constants.CLI_DEFAULTS['verbose_count']) + self.root_logger = mock.MagicMock() + + def test_common(self): + with mock.patch('certbot.log.logging.getLogger') as mock_get_logger: + mock_get_logger.return_value = self.root_logger + with mock.patch('certbot.log.except_hook') as mock_except_hook: + with mock.patch('certbot.log.sys') as mock_sys: + mock_sys.version_info = sys.version_info + self._call(self.config) + + self.assertEqual(self.root_logger.addHandler.call_count, 2) + self.assertTrue(os.path.exists(os.path.join( + self.config.logs_dir, 'letsencrypt.log'))) + mock_sys.excepthook(1, 2, 3) + mock_except_hook.assert_called_once_with(1, 2, 3, config=self.config) + + stderr_handler = self.root_logger.addHandler.call_args_list[0][0][0] + level = stderr_handler.level + if self.config.quiet: + self.assertEqual(level, constants.QUIET_LOGGING_LEVEL) + else: + self.assertEqual(level, -self.config.verbose_count * 10) + + def test_quiet(self): + self.config.quiet = True + self.test_common() + + +class SetupLogFileHandlerTest(test_util.TempDirTestCase): + """Tests for certbot.log.setup_log_file_handler.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.log import setup_log_file_handler + return setup_log_file_handler(*args, **kwargs) + + def setUp(self): + super(SetupLogFileHandlerTest, self).setUp() + self.config = mock.MagicMock(logs_dir=self.tempdir) + + @mock.patch('certbot.main.logging.handlers.RotatingFileHandler') + def test_failure(self, mock_handler): + mock_handler.side_effect = IOError + + try: + self._call(self.config, 'test.log', '%(message)s') + except errors.Error as err: + self.assertTrue('--logs-dir' in str(err)) + else: # pragma: no cover + self.fail('Error not raised.') + + def test_success(self): + log_file = 'test.log' + handler, log_path = self._call(self.config, log_file, '%(message)s') + self.assertEqual(handler.level, logging.DEBUG) + self.assertEqual(handler.formatter.converter, time.gmtime) + + expected_path = os.path.join(self.config.logs_dir, log_file) + self.assertEqual(log_path, expected_path) + + +class ColoredStreamHandlerTest(unittest.TestCase): + """Tests for certbot.log.""" + + def setUp(self): + from certbot import log + + self.stream = six.StringIO() + self.stream.isatty = lambda: True + self.handler = log.ColoredStreamHandler(self.stream) + + self.logger = logging.getLogger() + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(self.handler) + + def test_format(self): + msg = 'I did a thing' + self.logger.debug(msg) + self.assertEqual(self.stream.getvalue(), '{0}\n'.format(msg)) + + def test_format_and_red_level(self): + msg = 'I did another thing' + self.handler.red_level = logging.DEBUG + self.logger.debug(msg) + + self.assertEqual(self.stream.getvalue(), + '{0}{1}{2}\n'.format(util.ANSI_SGR_RED, + msg, + util.ANSI_SGR_RESET)) + + +class ExceptHookTest(unittest.TestCase): + """Tests for certbot.log.except_hook.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.log import except_hook + return except_hook(*args, **kwargs) + + @mock.patch('certbot.log.sys') + def test_except_hook(self, mock_sys): + config = mock.MagicMock() + mock_open = mock.mock_open() + + with mock.patch('certbot.log.open', mock_open, create=True): + exception = Exception('detail') + config.verbose_count = 1 + self._call( + Exception, exc_value=exception, trace=None, config=None) + mock_open().write.assert_any_call(''.join( + traceback.format_exception_only(Exception, exception))) + error_msg = mock_sys.exit.call_args_list[0][0][0] + self.assertTrue('unexpected error' in error_msg) + + with mock.patch('certbot.log.open', mock_open, create=True): + mock_open.side_effect = [KeyboardInterrupt] + error = errors.Error('detail') + self._call( + errors.Error, exc_value=error, trace=None, config=None) + # assert_any_call used because sys.exit doesn't exit in cli.py + mock_sys.exit.assert_any_call(''.join( + traceback.format_exception_only(errors.Error, error))) + + bad_typ = messages.ERROR_PREFIX + 'triffid' + exception = messages.Error(detail='alpha', typ=bad_typ, title='beta') + config = mock.MagicMock(debug=False, verbose_count=-3) + self._call( + messages.Error, exc_value=exception, trace=None, config=config) + error_msg = mock_sys.exit.call_args_list[-1][0][0] + self.assertTrue('unexpected error' in error_msg) + self.assertTrue('acme:error' not in error_msg) + self.assertTrue('alpha' in error_msg) + self.assertTrue('beta' in error_msg) + config = mock.MagicMock(debug=False, verbose_count=1) + self._call( + messages.Error, exc_value=exception, trace=None, config=config) + error_msg = mock_sys.exit.call_args_list[-1][0][0] + self.assertTrue('unexpected error' in error_msg) + self.assertTrue('acme:error' in error_msg) + self.assertTrue('alpha' in error_msg) + + interrupt = KeyboardInterrupt('detail') + self._call( + KeyboardInterrupt, exc_value=interrupt, trace=None, config=None) + mock_sys.exit.assert_called_with(''.join( + traceback.format_exception_only(KeyboardInterrupt, interrupt))) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 5d456bc3d..1afe95924 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -18,7 +18,6 @@ from acme import jose from certbot import account from certbot import cli -from certbot import colored_logging from certbot import constants from certbot import configuration from certbot import crypto_util @@ -288,81 +287,6 @@ class RevokeTest(test_util.TempDirTestCase): self.mock_success_revoke.assert_not_called() -class SetupLogFileHandlerTest(test_util.TempDirTestCase): - """Tests for certbot.main.setup_log_file_handler.""" - - def setUp(self): - super(SetupLogFileHandlerTest, self).setUp() - - self.config = mock.Mock(spec_set=['logs_dir'], - logs_dir=self.tempdir) - - def _call(self, *args, **kwargs): - from certbot.main import setup_log_file_handler - return setup_log_file_handler(*args, **kwargs) - - @mock.patch('certbot.main.logging.handlers.RotatingFileHandler') - def test_ioerror(self, mock_handler): - mock_handler.side_effect = IOError - self.assertRaises(errors.Error, self._call, - self.config, "test.log", "%s") - - -class SetupLoggingTest(test_util.TempDirTestCase): - """Tests for certbot.main.setup_logging.""" - - def setUp(self): - super(SetupLoggingTest, self).setUp() - - self.config = mock.Mock( - logs_dir=self.tempdir, - noninteractive_mode=False, quiet=False, - verbose_count=constants.CLI_DEFAULTS['verbose_count']) - - @classmethod - def _call(cls, *args, **kwargs): - from certbot.main import setup_logging - return setup_logging(*args, **kwargs) - - @mock.patch('certbot.main.logging.getLogger') - def test_defaults(self, mock_get_logger): - self._call(self.config) - - cli_handler = mock_get_logger().addHandler.call_args_list[0][0][0] - self.assertEqual(cli_handler.level, -self.config.verbose_count * 10) - self.assertTrue( - isinstance(cli_handler, colored_logging.StreamHandler)) - - @mock.patch('certbot.main.logging.getLogger') - def test_quiet_mode(self, mock_get_logger): - self.config.quiet = self.config.noninteractive_mode = True - self._call(self.config) - - cli_handler = mock_get_logger().addHandler.call_args_list[0][0][0] - self.assertEqual(cli_handler.level, constants.QUIET_LOGGING_LEVEL) - self.assertTrue( - isinstance(cli_handler, colored_logging.StreamHandler)) - - -class MakeOrVerifyCoreDirTest(test_util.TempDirTestCase): - """Tests for certbot.main.make_or_verify_core_dir.""" - - def _call(self, *args, **kwargs): - from certbot.main import make_or_verify_core_dir - return make_or_verify_core_dir(*args, **kwargs) - - def test_success(self): - new_dir = os.path.join(self.tempdir, 'new') - self._call(new_dir, 0o700, os.geteuid(), False) - self.assertTrue(os.path.exists(new_dir)) - - @mock.patch('certbot.main.util.make_or_verify_dir') - def test_failure(self, mock_make_or_verify): - mock_make_or_verify.side_effect = OSError - self.assertRaises(errors.Error, self._call, - self.tempdir, 0o700, os.geteuid(), False) - - class DetermineAccountTest(unittest.TestCase): """Tests for certbot.main._determine_account.""" @@ -1249,59 +1173,5 @@ class UnregisterTest(unittest.TestCase): self.assertFalse(acme_client.acme.deactivate_registration.called) -class TestHandleException(unittest.TestCase): - """Test main._handle_exception""" - @mock.patch('certbot.main.sys') - def test_handle_exception(self, mock_sys): - # pylint: disable=protected-access - from acme import messages - - config = mock.MagicMock() - mock_open = mock.mock_open() - - with mock.patch('certbot.main.open', mock_open, create=True): - exception = Exception('detail') - config.verbose_count = 1 - main._handle_exception( - Exception, exc_value=exception, trace=None, config=None) - mock_open().write.assert_any_call(''.join( - traceback.format_exception_only(Exception, exception))) - error_msg = mock_sys.exit.call_args_list[0][0][0] - self.assertTrue('unexpected error' in error_msg) - - with mock.patch('certbot.main.open', mock_open, create=True): - mock_open.side_effect = [KeyboardInterrupt] - error = errors.Error('detail') - main._handle_exception( - errors.Error, exc_value=error, trace=None, config=None) - # assert_any_call used because sys.exit doesn't exit in cli.py - mock_sys.exit.assert_any_call(''.join( - traceback.format_exception_only(errors.Error, error))) - - bad_typ = messages.ERROR_PREFIX + 'triffid' - exception = messages.Error(detail='alpha', typ=bad_typ, title='beta') - config = mock.MagicMock(debug=False, verbose_count=-3) - main._handle_exception( - messages.Error, exc_value=exception, trace=None, config=config) - error_msg = mock_sys.exit.call_args_list[-1][0][0] - self.assertTrue('unexpected error' in error_msg) - self.assertTrue('acme:error' not in error_msg) - self.assertTrue('alpha' in error_msg) - self.assertTrue('beta' in error_msg) - config = mock.MagicMock(debug=False, verbose_count=1) - main._handle_exception( - messages.Error, exc_value=exception, trace=None, config=config) - error_msg = mock_sys.exit.call_args_list[-1][0][0] - self.assertTrue('unexpected error' in error_msg) - self.assertTrue('acme:error' in error_msg) - self.assertTrue('alpha' in error_msg) - - interrupt = KeyboardInterrupt('detail') - main._handle_exception( - KeyboardInterrupt, exc_value=interrupt, trace=None, config=None) - mock_sys.exit.assert_called_with(''.join( - traceback.format_exception_only(KeyboardInterrupt, interrupt))) - - if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 0b06cccd7..9ec8dca28 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -38,18 +38,6 @@ class ReporterTest(unittest.TestCase): self.reporter.print_messages() self.assertEqual(sys.stdout.getvalue(), "") - @mock.patch('certbot.reporter.os.getpid') - def test_atexit_print_messages(self, mock_getpid): - self._add_messages() - mock_getpid.return_value = 42 - with mock.patch('certbot.reporter.INITIAL_PID', 42): - self.reporter.atexit_print_messages() - output = sys.stdout.getvalue() - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" in output) - self.assertTrue("Low" in output) - def test_tty_successful_exit(self): sys.stdout.isatty = lambda: True self._successful_exit_common() diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 480120772..f021c04cf 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -73,6 +73,25 @@ class ExeExistsTest(unittest.TestCase): self.assertFalse(self._call("exe")) +class MakeOrVerifyCoreDirTest(test_util.TempDirTestCase): + """Tests for certbot.util.make_or_verify_core_dir.""" + + def _call(self, *args, **kwargs): + from certbot.util import make_or_verify_core_dir + return make_or_verify_core_dir(*args, **kwargs) + + def test_success(self): + new_dir = os.path.join(self.tempdir, 'new') + self._call(new_dir, 0o700, os.geteuid(), False) + self.assertTrue(os.path.exists(new_dir)) + + @mock.patch('certbot.main.util.make_or_verify_dir') + def test_failure(self, mock_make_or_verify): + mock_make_or_verify.side_effect = OSError + self.assertRaises(errors.Error, self._call, + self.tempdir, 0o700, os.geteuid(), False) + + class MakeOrVerifyDirTest(test_util.TempDirTestCase): """Tests for certbot.util.make_or_verify_dir. @@ -473,5 +492,37 @@ class OsInfoTest(unittest.TestCase): ("windows", "95")) +class AtexitRegisterTest(unittest.TestCase): + """Tests for certbot.util.atexit_register.""" + def setUp(self): + self.func = mock.MagicMock() + self.args = ('hi',) + self.kwargs = {'answer': 42} + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.util import atexit_register + return atexit_register(*args, **kwargs) + + def test_called(self): + self._test_common(os.getpid()) + self.func.assert_called_with(*self.args, **self.kwargs) + + def test_not_called(self): + self._test_common(initial_pid=-1) + self.assertFalse(self.func.called) + + def _test_common(self, initial_pid): + with mock.patch('certbot.util._INITIAL_PID', initial_pid): + with mock.patch('certbot.util.atexit') as mock_atexit: + self._call(self.func, *self.args, **self.kwargs) + + # _INITAL_PID must be mocked when calling atexit_func + self.assertTrue(mock_atexit.register.called) + args, kwargs = mock_atexit.register.call_args + atexit_func = args[0] + atexit_func(*args[1:], **kwargs) # pylint: disable=star-args + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/util.py b/certbot/util.py index 95c669d0d..55a75097f 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -1,5 +1,6 @@ """Utilities for all Certbot.""" import argparse +import atexit import collections # distutils.version under virtualenv confuses pylint # For more info, see: https://github.com/PyCQA/pylint/issues/73 @@ -38,6 +39,16 @@ ANSI_SGR_RED = "\033[31m" ANSI_SGR_RESET = "\033[0m" +PERM_ERR_FMT = os.linesep.join(( + "The following error was encountered:", "{0}", + "If running as non-root, set --config-dir, " + "--work-dir, and --logs-dir to writeable paths.")) + + +# Stores importing process ID to be used by atexit_register() +_INITIAL_PID = os.getpid() + + def run_script(params, log=logger.error): """Run the script with the given params. @@ -92,6 +103,23 @@ def exe_exists(exe): return False +def make_or_verify_core_dir(directory, mode, uid, strict): + """Make sure directory exists with proper permissions. + + :param str directory: Path to a directory. + :param int mode: Directory mode. + :param int uid: Directory owner. + :param bool strict: require directory to be owned by current user + + :raises .errors.Error: if the directory cannot be made or verified + + """ + try: + make_or_verify_dir(directory, mode, uid, strict) + except OSError as error: + raise errors.Error(PERM_ERR_FMT.format(error)) + + def make_or_verify_dir(directory, mode=0o755, uid=0, strict=False): """Make sure directory exists with proper permissions. @@ -533,3 +561,20 @@ def is_staging(srv): :rtype bool: """ return srv == constants.STAGING_URI or "staging" in srv + + +def atexit_register(func, *args, **kwargs): + """Sets func to be called before the program exits. + + Special care is taken to ensure func is only called when the process + that first imports this module exits rather than any child processes. + + :param function func: function to be called in case of an error + + """ + atexit.register(_atexit_call, func, *args, **kwargs) + + +def _atexit_call(func, *args, **kwargs): + if _INITIAL_PID == os.getpid(): + func(*args, **kwargs) From c1cb762b325bf0b85eadc64957d7897ecf56524c Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 3 Apr 2017 14:33:41 -0700 Subject: [PATCH 085/128] docs: clarify when venv needs to be re-setup When dependencies change or a new plugin is introduced, the venv needs to be re-created. Partially addresses issue #4368. --- docs/contributing.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9c7f0636f..f75f2516a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -26,7 +26,8 @@ If you're on macOS, we recommend you skip the rest of this section and instead run Certbot in Docker. You can find instructions for how to do this :ref:`here `. If you're running on Linux, you can run the following commands to install dependencies and set up a virtual environment where you can run -Certbot. You only need to do this once. +Certbot. You will need to repeat this when Certbot's dependencies change or when +a new plugin is introduced. .. code-block:: shell From 3f625d3a0d396326561af02dfeec922783aaa0e1 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 3 Apr 2017 14:51:48 -0700 Subject: [PATCH 086/128] docs: clarify the use of entry points for plugins The documentation did not list the entry point group, certbot.plugins. Partially addresses issue #4368. --- docs/contributing.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f75f2516a..75a65fab7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -220,9 +220,10 @@ Writing your own plugin ~~~~~~~~~~~~~~~~~~~~~~~ Certbot client supports dynamic discovery of plugins through the -`setuptools entry points`_. This way you can, for example, create a -custom implementation of `~certbot.interfaces.IAuthenticator` or -the `~certbot.interfaces.IInstaller` without having to merge it +`setuptools entry points`_ using the `certbot.plugins` group. This +way you can, for example, create a custom implementation of +`~certbot.interfaces.IAuthenticator` or the +`~certbot.interfaces.IInstaller` without having to merge it with the core upstream source code. An example is provided in ``examples/plugins/`` directory. From af5fd4f6bdeb21edb4666323191db7500a377eb9 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 3 Apr 2017 14:55:27 -0700 Subject: [PATCH 087/128] docs: use monospace syntax for method name --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 75a65fab7..98ecfaccc 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -169,8 +169,8 @@ challenges: HTTP, TLS-SNI, and DNS, represented by classes in `acme.challenges`. An authenticator plugin should implement support for at least one challenge type. An Authenticator indicates which challenges it supports by implementing -get_chall_pref(domain) to return a sorted list of challenge types in preference -order. +`get_chall_pref(domain)` to return a sorted list of challenge types in +preference order. An Authenticator must also implement `perform(achalls)`, which "performs" a list of challenges by, for instance, provisioning a file on an HTTP server, or From 22248c1393d22d489edebd5e804806f47d6d0eb7 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 3 Apr 2017 14:58:25 -0700 Subject: [PATCH 088/128] docs: fix syntax error in plugin installation example Partially addresses issue #4368. --- docs/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 98ecfaccc..96bd30e0d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -231,6 +231,7 @@ While developing, you can install your plugin into a Certbot development virtualenv like this: .. code-block:: shell + . venv/bin/activate . tests/integration/_common.sh pip install -e examples/plugins/ From b92b37d9fea96f3d3a085e97332ffd729abfbefb Mon Sep 17 00:00:00 2001 From: Robotic-Brain Date: Tue, 4 Apr 2017 00:35:11 +0200 Subject: [PATCH 089/128] Fixing Typo in Readme.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e44a61021..f4ba65859 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ .. This file contains a series of comments that are used to include sections of this README in other files. Do not modify these comments unless you know what you are doing. tag:intro-begin -Certbot is part of EFF’s effort to encrypt the entire Internet. Secure communication over the Web relies on HTTPS, which requires the use of a digital certificate that lets browsers verify the identify of web servers (e.g., is that really google.com?). Web servers obtain their certificates from trusted third parties called certificate authorities (CAs). Certbot is an easy-to-use client that fetches a certificate from Let’s Encrypt—an open certificate authority launched by the EFF, Mozilla, and others—and deploys it to a web server. +Certbot is part of EFF’s effort to encrypt the entire Internet. Secure communication over the Web relies on HTTPS, which requires the use of a digital certificate that lets browsers verify the identity of web servers (e.g., is that really google.com?). Web servers obtain their certificates from trusted third parties called certificate authorities (CAs). Certbot is an easy-to-use client that fetches a certificate from Let’s Encrypt—an open certificate authority launched by the EFF, Mozilla, and others—and deploys it to a web server. Anyone who has gone through the trouble of setting up a secure website knows what a hassle getting and maintaining a certificate is. Certbot and Let’s Encrypt can automate away the pain and let you turn on and manage HTTPS with simple commands. Using Certbot and Let's Encrypt is free, so there’s no need to arrange payment. From 345e77efcf86ecd1c91561e89dd82332ed7732b8 Mon Sep 17 00:00:00 2001 From: Robotic-Brain Date: Tue, 4 Apr 2017 00:42:33 +0200 Subject: [PATCH 090/128] Fixing Typo in README.rst (issues -> issued) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e44a61021..3e95dfdbd 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. -If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issues by Let’s Encrypt. +If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME From 4c36d8081ef6e752ca1eecbd18fdb39054c054c2 Mon Sep 17 00:00:00 2001 From: St-Ranger Date: Mon, 3 Apr 2017 23:45:48 -0500 Subject: [PATCH 091/128] Correct info fro FreeBSD 1. Replace the outdated reference to "vritualenv", "see below" -> "see above". 2. Replace the awkward sentence (partially incorrect) about installing on FreeBSD. It can be installed via different ways: "pkg" is not exclusive. I am not aware of any reason why it canNOT be installed from ports. Some people prefer build and install everything from ports. (The previous version of the description implies that ports cannot be used.) --- docs/contributing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9c7f0636f..4012895b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -427,8 +427,12 @@ For squeeze you will need to: FreeBSD ------- -Package installation for FreeBSD uses ``pkg``, not ports. +The package can be installed FreeBSD using ``pkg`` (e.g. ``pkg install py27-certbot``), +or any other port-management tool (``portupgrade``, ``portmanager``, etc.) +from the pre-compiled package or can be built and installed from ports. +Either way will ensure proper installation of all the dependencies required +to run ``certbot``. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see -below), you will need a compatible shell, e.g. ``pkg install bash && +above), you will need a compatible shell, e.g. ``pkg install bash && bash``. From 971439a518f6df879c14e610f966bc3442a1d3c1 Mon Sep 17 00:00:00 2001 From: Nathan Arthur Date: Tue, 4 Apr 2017 00:48:33 -0400 Subject: [PATCH 092/128] Sort list of OSes with packages --- docs/install.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index e1ec06f16..2487131b7 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -140,16 +140,6 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. Operating System Packages ------------------------- -**FreeBSD** - - * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py27-certbot`` - -**OpenBSD** - - * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` - * Package: ``pkg_add letsencrypt`` - **Arch Linux** .. code-block:: shell @@ -182,6 +172,11 @@ repo, if you have not already done so. Then run: sudo dnf install certbot python2-certbot-apache +**FreeBSD** + + * Port: ``cd /usr/ports/security/py-certbot && make install clean`` + * Package: ``pkg install py27-certbot`` + **Gentoo** The official Certbot client is available in Gentoo Portage. If you @@ -217,6 +212,11 @@ For the time being, this is the only way for the Apache plugin to recognise the appropriate directives when installing the certificate. Note: this change is not required for the other plugins. +**OpenBSD** + + * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` + * Package: ``pkg_add letsencrypt`` + **Other Operating Systems** OS packaging is an ongoing effort. If you'd like to package From 2b03833752dc33475bdfaf53ff4465b374a7fd44 Mon Sep 17 00:00:00 2001 From: Nathan Arthur Date: Tue, 4 Apr 2017 00:55:25 -0400 Subject: [PATCH 093/128] Add NetBSD to list of OSes with packages --- docs/install.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/install.rst b/docs/install.rst index 2487131b7..6c56584be 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -212,6 +212,11 @@ For the time being, this is the only way for the Apache plugin to recognise the appropriate directives when installing the certificate. Note: this change is not required for the other plugins. +**NetBSD** + + * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` + * Install pre-compiled package: ``pkg_add py27-certbot`` + **OpenBSD** * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` From 035e5b93ea2ef12f1738c172915b8256bfb504e6 Mon Sep 17 00:00:00 2001 From: St-Ranger Date: Tue, 4 Apr 2017 00:41:55 -0500 Subject: [PATCH 094/128] Futher improvement re: FreeBSD packages Further improved the previous change to make it more clear. --- docs/contributing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4012895b8..f3879f53f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -427,11 +427,11 @@ For squeeze you will need to: FreeBSD ------- -The package can be installed FreeBSD using ``pkg`` (e.g. ``pkg install py27-certbot``), +Packages can be installed FreeBSD using ``pkg``, or any other port-management tool (``portupgrade``, ``portmanager``, etc.) -from the pre-compiled package or can be built and installed from ports. +from the pre-built package or can be built and installed from ports. Either way will ensure proper installation of all the dependencies required -to run ``certbot``. +for the package. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see above), you will need a compatible shell, e.g. ``pkg install bash && From 43dccfc67187b0ae3d677fbe0c7fffab20e6179c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 4 Apr 2017 11:20:58 -0700 Subject: [PATCH 095/128] More thoroughly rename during `certbot rename`. (#4320) * rename more files in rename command * Revert "Hide rename command (#4007)" This reverts commit 8c14de13a5d9d5be34aa13ac49a4f47bfb51a8fe. * Rename files in configuration files * Delete new files if we fail during the renaming process * update tests and error catching * More expressive error message --- certbot/cert_manager.py | 37 +++++++++-- certbot/cli.py | 15 ++++- certbot/main.py | 9 +-- certbot/storage.py | 102 ++++++++++++++++++++++++++--- certbot/tests/cert_manager_test.py | 21 ++++-- 5 files changed, 157 insertions(+), 27 deletions(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 35d539a16..7b1811c99 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -13,6 +13,7 @@ from certbot import storage from certbot import util from certbot.display import util as display_util +from certbot.plugins import disco as plugins_disco logger = logging.getLogger(__name__) @@ -46,6 +47,7 @@ def rename_lineage(config): certname = _get_certname(config, "rename") + # what is the new name we want to use? new_certname = config.new_certname if not new_certname: code, new_certname = disp.input( @@ -54,13 +56,34 @@ def rename_lineage(config): if code != display_util.OK or not new_certname: raise errors.Error("User ended interaction.") - lineage = lineage_for_certname(config, certname) - if not lineage: - raise errors.ConfigurationError("No existing certificate with name " - "{0} found.".format(certname)) - storage.rename_renewal_config(certname, new_certname, config) - disp.notification("Successfully renamed {0} to {1}." - .format(certname, new_certname), pause=False) + try: + # copy files to new name + new_lineage = storage.duplicate_lineage(config, certname, new_certname) + + # install the new name's files + config.certname = new_certname + plugins = plugins_disco.PluginsRegistry.find_all() + from certbot.main import install + install(config, plugins, new_lineage, False) + except (errors.CertStorageError, errors.ConfigurationError, IOError, OSError) as e: + # delete the new files + config.certname = new_certname + # we might not have created anything to delete + try: + storage.delete_files(config, new_certname) + except errors.CertStorageError: + pass + reporter = zope.component.getUtility(interfaces.IReporter) + reporter.add_message("Unable to rename certificate", reporter.HIGH_PRIORITY) + raise e + else: + # delete old files + config.certname = certname + storage.delete_files(config, certname) + disp.notification("Renamed files for {0} to {1}." + .format(certname, new_certname), pause=False) + finally: + config.certname = certname def certificates(config): """Display information about certs configured with Certbot diff --git a/certbot/cli.py b/certbot/cli.py index 6217baa27..0442f481d 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -81,6 +81,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certs you have from Certbot revoke Revoke a certificate (supply --cert-path) + rename Rename a certificate delete Delete a certificate manage your account with Let's Encrypt: @@ -363,6 +364,10 @@ VERB_HELP = [ "opts": "Options for revocation of certs", "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem [options]\n\n" }), + ("rename", { + "short": "Change a certificate's name (for management purposes)", + "opts": "Options for changing certificate names" + }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", "opts": "Options for account registration & modification" @@ -420,6 +425,7 @@ class HelpfulArgumentParser(object): "plugins": main.plugins_cmd, "register": main.register, "unregister": main.unregister, + "rename": main.rename, "renew": main.renew, "revoke": main.revoke, "rollback": main.rollback, @@ -790,7 +796,7 @@ def _add_all_groups(helpful): helpful.add_group("paths", description="Arguments changing execution paths & servers") helpful.add_group("manage", description="Various subcommands and flags are available for managing your certificates:", - verbs=["certificates", "delete", "renew", "revoke", "update_symlinks"]) + verbs=["certificates", "delete", "rename", "renew", "revoke", "update_symlinks"]) # VERBS for verb, docs in VERB_HELP: @@ -843,12 +849,17 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "multiple -d flags or enter a comma separated list of domains " "as a parameter. (default: Ask)") helpful.add( - [None, "run", "certonly", "manage", "delete", "certificates"], + [None, "run", "certonly", "manage", "rename", "delete", "certificates"], "--cert-name", dest="certname", metavar="CERTNAME", default=None, help="Certificate name to apply. Only one certificate name can be used " "per Certbot run. To see certificate names, run 'certbot certificates'. " "When creating a new certificate, specifies the new certificate's name.") + helpful.add( + ["rename", "manage"], + "--updated-cert-name", dest="new_certname", + metavar="NEW_CERTNAME", default=None, + help="New name for the certificate. Must be a valid filename.") helpful.add( [None, "testing", "renew", "certonly"], "--dry-run", action="store_true", dest="dry_run", diff --git a/certbot/main.py b/certbot/main.py index 090479d20..256503d4e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -460,15 +460,16 @@ def register(config, unused_plugins): eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) -def _install_cert(config, le_client, domains, lineage=None): +def _install_cert(config, le_client, domains, lineage=None, enhance=True): path_provider = lineage if lineage else config assert path_provider.cert_path is not None le_client.deploy_certificate(domains, path_provider.key_path, path_provider.cert_path, path_provider.chain_path, path_provider.fullchain_path) - le_client.enhance_config(domains, path_provider.chain_path) + if enhance: + le_client.enhance_config(domains, path_provider.chain_path) -def install(config, plugins): +def install(config, plugins, lineage=None, enhance=True): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert # FIXME: be consistent about whether errors are raised or returned from @@ -481,7 +482,7 @@ def install(config, plugins): domains, _ = _find_domains_or_certname(config, installer) le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + _install_cert(config, le_client, domains, lineage, enhance) def plugins_cmd(config, plugins): # TODO: Use IDisplay rather than print diff --git a/certbot/storage.py b/certbot/storage.py index a1462b72d..34dc57884 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -34,7 +34,9 @@ def renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def renewal_file_for_certname(config, certname): - """Return /path/to/certname.conf in the renewal conf directory""" + """Return /path/to/certname.conf in the renewal conf directory + :raises .CertStorageError: if file is missing + """ path = os.path.join(config.renewal_configs_dir, "{0}.conf".format(certname)) if not os.path.exists(path): raise errors.CertStorageError("No certificate found with name {0} (expected " @@ -130,6 +132,8 @@ def rename_renewal_config(prev_name, new_name, cli_config): except OSError: raise errors.ConfigurationError("Please specify a valid filename " "for the new certificate name.") + else: + return new_filename def update_configuration(lineagename, archive_dir, target, cli_config): @@ -146,19 +150,25 @@ def update_configuration(lineagename, archive_dir, target, cli_config): """ config_filename = renewal_filename_for_lineagename(cli_config, lineagename) - temp_filename = config_filename + ".new" + + def _save_renewal_values(unused_config, temp_filename): + # Save only the config items that are relevant to renewal + values = relevant_values(vars(cli_config.namespace)) + write_renewal_config(config_filename, temp_filename, archive_dir, target, values) + _modify_config_with_tempfile(config_filename, _save_renewal_values) + + return configobj.ConfigObj(config_filename) + +def _modify_config_with_tempfile(filename, function): + temp_filename = filename + ".new" # If an existing tempfile exists, delete it if os.path.exists(temp_filename): os.unlink(temp_filename) - # Save only the config items that are relevant to renewal - values = relevant_values(vars(cli_config.namespace)) - write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - os.rename(temp_filename, config_filename) - - return configobj.ConfigObj(config_filename) - + config = configobj.ConfigObj(filename) + function(config, temp_filename) + os.rename(temp_filename, filename) def get_link_target(link): """Get an absolute path to the target of link. @@ -243,6 +253,7 @@ def delete_files(config, certname): """Delete all files related to the certificate. If some files are not found, ignore them and continue. + :raises .CertStorageError: if lineage is missing """ renewal_filename = renewal_file_for_certname(config, certname) # file exists @@ -304,6 +315,79 @@ def delete_files(config, certname): except OSError: logger.debug("Unable to remove %s", archive_path) +def duplicate_lineage(config, certname, new_certname): + """Create a duplicate of certname with name new_certname + + :raises .CertStorageError: for storage errors + :raises .ConfigurationError: for cli and renewal configuration errors + :raises IOError: for filename errors + :raises OSError: for OS errors + """ + + # copy renewal config file + prev_filename = renewal_filename_for_lineagename(config, certname) + new_filename = renewal_filename_for_lineagename(config, new_certname) + if os.path.exists(new_filename): + raise errors.ConfigurationError("The new certificate name " + "is already in use.") + try: + shutil.copy2(prev_filename, new_filename) + except (OSError, IOError): + raise errors.ConfigurationError("Please specify a valid filename " + "for the new certificate name.") + logger.debug("Copied %s to %s", prev_filename, new_filename) + + # load config file + try: + renewal_config = configobj.ConfigObj(new_filename) + except configobj.ConfigObjError: + # config is corrupted + logger.warning("Could not parse %s. Only the certificate has been renamed.", + new_filename) + raise errors.CertStorageError( + "error parsing {0}".format(new_filename)) + + def copy_to_new_dir(prev_dir): + """Replace certname with new_certname in prev_dir""" + new_dir = prev_dir.replace(certname, new_certname) + # make dir iff it doesn't exist + shutil.copytree(prev_dir, new_dir, symlinks=True) + logger.debug("Copied %s to %s", prev_dir, new_dir) + return new_dir + + # archive dir + prev_archive_dir = _full_archive_path(renewal_config, config, certname) + new_archive_dir = prev_archive_dir + if not certname in prev_archive_dir: + raise errors.CertStorageError("Archive directory does not conform to defaults: " + "{0} not in {1}", certname, prev_archive_dir) + else: + new_archive_dir = copy_to_new_dir(prev_archive_dir) + + # live dir + # if things aren't in their default places, don't try to change things. + prev_live_dir = _full_live_path(config, certname) + prev_links = dict((kind, renewal_config.get(kind)) for kind in ALL_FOUR) + if (certname not in prev_live_dir or + len(set(os.path.dirname(renewal_config.get(kind)) for kind in ALL_FOUR)) != 1): + raise errors.CertStorageError("Live directory does not conform to defaults.") + else: + copy_to_new_dir(prev_live_dir) + new_links = dict((k, prev_links[k].replace(certname, new_certname)) for k in prev_links) + + # Update renewal config file + def _update_and_write(renewal_config, temp_filename): + renewal_config["archive_dir"] = new_archive_dir + renewal_config["version"] = certbot.__version__ + for kind in ALL_FOUR: + renewal_config[kind] = new_links[kind] + with open(temp_filename, "wb") as f: + renewal_config.write(outfile=f) + _modify_config_with_tempfile(new_filename, _update_and_write) + + # Update symlinks + return RenewableCert(new_filename, config, update_symlinks=True) + class RenewableCert(object): # pylint: disable=too-many-instance-attributes,too-many-public-methods diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index b5731fbd6..d0e563979 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -384,14 +384,19 @@ class RenameLineageTest(BaseCertManagerTest): @test_util.patch_get_utility() @mock.patch('certbot.cert_manager.lineage_for_certname') def test_no_existing_certname(self, mock_lineage_for_certname, unused_get_utility): - mock_config = mock.Mock(certname="one", new_certname="two") + mock_config = mock.Mock(certname="one", new_certname="two", + renewal_configs_dir="/tmp/etc/letsencrypt/renewal/") mock_lineage_for_certname.return_value = None - self.assertRaises(errors.ConfigurationError, - self._call, mock_config) + self.assertRaises(errors.ConfigurationError, self._call, mock_config) + @mock.patch("certbot.storage.RenewableCert._update_symlinks") @test_util.patch_get_utility() @mock.patch("certbot.storage.RenewableCert._check_symlinks") - def test_rename_cert(self, mock_check, unused_get_utility): + @mock.patch("certbot.storage.relevant_values") + def test_rename_cert(self, mock_rv, mock_check, unused_get_utility, unused_update_symlinks): + # Mock relevant_values() to claim that all values are relevant here + # (to avoid instantiating parser) + mock_rv.side_effect = lambda x: x mock_check.return_value = True mock_config = self.mock_config self._call(mock_config) @@ -400,9 +405,15 @@ class RenameLineageTest(BaseCertManagerTest): self.assertTrue(updated_lineage is not None) self.assertEqual(updated_lineage.lineagename, mock_config.new_certname) + @mock.patch("certbot.storage.RenewableCert._update_symlinks") @test_util.patch_get_utility() @mock.patch("certbot.storage.RenewableCert._check_symlinks") - def test_rename_cert_interactive_certname(self, mock_check, mock_get_utility): + @mock.patch("certbot.storage.relevant_values") + def test_rename_cert_interactive_certname(self, mock_rv, mock_check, mock_get_utility, + unused_update_symlinks): + # python 3.4 and 3.5 order things differently, so remove other.com for this test + os.remove(self.configs["other.com"].filename) + mock_rv.side_effect = lambda x: x mock_check.return_value = True mock_config = self.mock_config mock_config.certname = None From 2887b888f7dfbf0e2052c8a5f88d40412c1a2dae Mon Sep 17 00:00:00 2001 From: Alex Jordan Date: Mon, 27 Mar 2017 14:21:43 -0400 Subject: [PATCH 096/128] Start organizing the change log This style is based on http://keepachangelog.com/en/0.3.0/. --- CHANGELOG.md | 288 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 180 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc1ad82ed..2325019bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,24 @@ -# 0.12.0 -## 03/02/2017 +# Certbot change log + +Certbot adheres to [Semantic Versioning](http://semver.org/). + +## 0.12.0 - 2017-02-03 + +### Added * Allow non-camelcase Apache VirtualHost names * Allow more log messages to be silenced + +### Fixed + * Fix a regression around using `--cert-name` when getting new certificates More information about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.12.0 -# 0.11.1 -## 02/01/2017 +## 0.11.1 - 2017-02-01 + +### Fixed * Resolve a problem where Certbot would crash while parsing command line arguments in some cases. @@ -18,10 +27,10 @@ arguments in some cases. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Aclosed -# 0.11.0 -## 02/01/2017 +## 0.11.0 - 2017-02-01 + +### Added -* Providing `--quiet` to `certbot-auto` now silences package manager output. * The UI has been improved in the standalone plugin. When using the plugin while running Certbot interactively and a required port is bound by another process, Certbot will give you the option to retry to grab @@ -31,6 +40,13 @@ server using the `unregister` subcommand. * When revoking a certificate using the `revoke` subcommand, you now have the option to provide the reason the certificate is being revoked to Let's Encrypt with `--reason`. + +### Changed + +* Providing `--quiet` to `certbot-auto` now silences package manager output. + +### Removed + * Removal of the optional `dnspython` dependency in our `acme` package. Now the library does not support client side verification of the DNS challenge. @@ -38,23 +54,28 @@ challenge. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.11.0+is%3Aclosed -# 0.10.2 -## 01/25/2017 +## 0.10.2 - 2017-01-25 + +### Added -* We now save `--preferred-challenges` values for renewal. Previously -these values were discarded causing a different challenge type to be -used when renewing certs in some cases. * If Certbot receives a request with a `badNonce` error, we automatically retry the request. Since nonces from Let's Encrypt expire, this helps people performing the DNS challenge with the `manual` plugin who may have to wait an extended period of time for their DNS changes to propagate. +### Fixed + +* We now save `--preferred-challenges` values for renewal. Previously +these values were discarded causing a different challenge type to be +used when renewing certs in some cases. + More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed -# 0.10.1 -## 01/13/2017 +## 0.10.1 - 2017-01-13 + +### Fixed * Resolve problems where when asking Certbot to update a certificate at an existing path to include different domain names, the old names would @@ -64,10 +85,11 @@ continue to be used. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.1+is%3Aclosed -# 0.10.0 -## 01/11/2017 +## 0.10.0 - 2017-01-11 -* The ability to customize and automatically complete DNS and HTTP +## Added + +* Added the ability to customize and automatically complete DNS and HTTP domain validation challenges with the manual plugin. The flags `--manual-auth-hook` and `--manual-cleanup-hook` can now be provided when using the manual plugin to execute commands provided by the user to @@ -75,23 +97,18 @@ perform and clean up challenges provided by the CA. This is best used in complicated setups where the DNS challenge must be used or Certbot's existing plugins cannot be used to perform HTTP challenges. For more information on how this works, see `certbot --help manual`. -* A `--cert-name` flag for specifying the name to use for the +* Added a `--cert-name` flag for specifying the name to use for the certificate in Certbot's configuration directory. Using this flag in combination with `-d/--domains`, a user can easily request a new certificate with different domains and save it with the name provided by `--cert-name`. Additionally, `--cert-name` can be used to select a certificate with the `certonly` and `run` subcommands so a full list of domains in the certificate does not have to be provided. -* The subcommand `certificates` for listing the certificates managed by +* Added subcommand `certificates` for listing the certificates managed by Certbot and their properties. -* A `delete` subcommand for removing certificates managed by Certbot +* Added `delete` subcommand for removing certificates managed by Certbot from the configuration directory. -* Support for requesting internationalized domain names (IDNs). -* Removal of the ncurses interface. This change solves problems people -were having on many systems, reduces the number of Certbot dependencies, -and simplifies our code. Certbot's only interface now is the text -interface which was available by providing `-t/--text` to earlier -versions of Certbot. +* Support requesting internationalized domain names (IDNs). * Hooks provided to Certbot are now saved to be reused during renewal. If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` flags when obtaining a certificate, the provided commands will @@ -101,56 +118,80 @@ command either on the command line or in a [configuration file](https://certbot.eff.org/docs/using.html#configuration-file) to run an additional command before/after any certificate is renewed. Hooks will only be run if a certificate is renewed. +* Support Busybox in certbot-auto. + +### Changed + * Recategorized `-h/--help` output to improve documentation and discoverability. -* Busybox support in certbot-auto. + +### Removed + +* Removed the ncurses interface. This change solves problems people +were having on many systems, reduces the number of Certbot +dependencies, and simplifies our code. Certbot's only interface now is +the text interface which was available by providing `-t/--text` to +earlier versions of Certbot. + +### Fixed + * Many small bug fixes. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.0is%3Aclosed -# 0.9.3 -## 10/13/2016 +## 0.9.3 - 2016-10-13 + +### Added -* Adopt more conservative behavior about reporting a needed port as -unavailable when using the standalone plugin. * The Apache plugin uses information about your OS to help determine the layout of your Apache configuration directory. We added a patch to ensure this code behaves the same way when testing on different systems as the tests were failing in some cases. +### Changed + +* Adopt more conservative behavior about reporting a needed port as +unavailable when using the standalone plugin. + More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/milestone/27?closed=1 -# 0.9.2 -## 10/12/2016 +## 0.9.2 - 2016-10-12 + +### Added -* Ensuring we properly copy `ssl on;` directives as necessary when -performing domain validation in the Nginx plugin. -* Verifying that our optional dependencies version matches what is -required by Certbot. -* A fix for problems where symlinks were becoming files when they were -packaged, causing errors during testing and OS packaging. * Stop requiring that all possibly required ports are available when using the standalone plugin. Only verify the ports are available when you know they are necessary. +### Fixed + +* Verify that our optional dependencies version matches what is +required by Certbot. +* Ensure we properly copy `ssl on;` directives as necessary when +performing domain validation in the Nginx plugin. +* Fix problems where symlinks were becoming files when they were +packaged, causing errors during testing and OS packaging. + More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/milestone/26?closed=1 -# 0.9.1 -## 10/06/2016 +## 0.9.1 - 2016-10-06 -* This version of Certbot simply fixes a bug that was introduced in version -0.9.0 where the command line flag -q/--quiet wasn't respected in some cases. +### Fixed + +* Fix a bug that was introduced in version 0.9.0 where the command +line flag -q/--quiet wasn't respected in some cases. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/milestone/25?closed=1 -# 0.9.0 -## 10/05/2016 +## 0.9.0 - 2016-10-05 -* An alpha version of the Nginx plugin. This plugin fully automates the +### Added + +* Add an alpha version of the Nginx plugin. This plugin fully automates the process of obtaining and installing certificates with Nginx. Additionally, it is able to automatically configure security enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use @@ -159,11 +200,11 @@ is installed automatically when using `certbot-auto`) and provide `--nginx` on the command line. This plugin is still in its early stages so we recommend you use it with some caution and make sure you have a backup of your Nginx configuration. -* Support for the `DNS` challenge in the `acme` library as well as `DNS` -support in Certbot's `manual` plugin. This allows you to create DNS -records to prove to Let's Encrypt you control the requested the domain -name. To use this feature, include `--manual --preferred-challenges dns` -on the command line. +* Support the `DNS` challenge in the `acme` library and `DNS` in +Certbot's `manual` plugin. This allows you to create DNS records to +prove to Let's Encrypt you control the requested domain name. To use +this feature, include `--manual --preferred-challenges dns` on the +command line. * Help with enabling Extra Packages for Enterprise Linux (EPEL) on CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, the EPEL repository has to be enabled. `certbot-auto` will now prompt @@ -175,20 +216,25 @@ included on the command line. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Aclosed -# 0.8.1 -## 06/14/2016 +## 0.8.1 - 2016-06-14 -* Preserving a certificate's common name when using `renew` +### Added + +* Preserve a certificate's common name when using `renew` * Save webroot values for renewal when they are entered interactively -* Problems with an invalid user-agent string on OS X -* Gracefully reporting the Apache plugin isn't usable when Augeas is not installed +* Gracefully report the Apache plugin isn't usable when Augeas is not installed * Experimental support for Mageia has been added to `certbot-auto` +### Fixed + +* Fix problems with an invalid user-agent string on OS X + More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ -# 0.8.0 -## 06/02/2016 +## 0.8.0 - 2016-06-02 + +### Added * The main new feature in this release is the `register` subcommand which can be used to register an account with the Let's Encrypt CA. @@ -198,76 +244,94 @@ change the e-mail address associated with your registration. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ -# 0.7.0 -## 05/27/2016 +## 0.7.0 - 2016-05-27 -* `--must-staple` to request certificates from Let's Encrypt with the -OCSP must staple extension -* automatic configuration of OSCP stapling for Apache -* requesting certificates for domains found in the common name of a -custom CSR -* a number of bug fixes +### Added + +* Added `--must-staple` to request certificates from Let's Encrypt +with the OCSP must staple extension +* Automatically configure OSCP stapling for Apache +* Allow requesting certificates for domains found in the common name +of a custom CSR + +### Fixed + +* Miscellaneous bug fixes More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue -# 0.6.0 -## 05/12/2016 +## 0.6.0 - 2016-05-12 + +### Added + +* Versioned the datetime dependency in setup.py + +### Changed * Renamed the client from `letsencrypt` to `certbot` + +### Fixed + * Fixed a small json deserialization error -* Versioned the datetime dependency in setup.py * Preserve domain order in generated CSRs * Some minor bug fixes More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is%3Aclosed%20 -# 0.5.0 -## 04/05/2016 +## 0.5.0 - 2016-04-05 -* The ability to use the webroot plugin interactively. -* The flags --pre-hook, --post-hook, and --renew-hook which can be used -with the renew subcommand to register shell commands to run in -response to renewal events. Pre-hook commands will be run before -any certs are renewed, post-hook commands will be run after any -certs are renewed, and renew-hook commands will be run after each -cert is renewed. If no certs are due for renewal, no command is run. -* Cleaner renewal configuration files. In /etc/letsencrypt/renewal by -default, these files can be used to control what parameters are used -when renewing a specific certificate. +### Added + +* Add the ability to use the webroot plugin interactively. +* The flags --pre-hook, --post-hook, and --renew-hook can be used with +the renew subcommand to register shell commands to run in response to +renewal events. Pre-hook commands will be run before any certs are +renewed, post-hook commands will be run after any certs are renewed, +and renew-hook commands will be run after each cert is renewed. If no +certs are due for renewal, no command is run. * A -q/--quiet flag which silences all output except errors. * An --allow-subset-of-domains flag which can be used with the renew command to prevent renewal failures for a subset of the requested domains from causing the client to exit. +### Changed + +* Use cleaner renewal configuration files. In /etc/letsencrypt/renewal +by default, these files can be used to control what parameters are +used when renewing a specific certificate. + More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue -# 0.4.2 -## 03/03/2016 +## 0.4.2 - 2016-03-03 + +### Fixed * Resolves problems encountered when compiling letsencrypt against the new OpenSSL release. -* A patch fixing problems of using letsencrypt renew with configuration files +* A patch fixing problems of using `letsencrypt renew` with configuration files from private beta has been added. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 -# 0.4.1 -## 02/29/2016 +## 0.4.1 - 2016-02-29 -* Fixes Apache parsing errors with some configurations -* Fixes Werkzeug dependency problems on some Red Hat systems -* Fixes bootstrapping failures when using letsencrypt-auto with --no-self-upgrade -* Fixes problems with parsing renewal config files from private beta +### Fixed + +* Fix Apache parsing errors with some configurations +* Fix Werkzeug dependency problems on some Red Hat systems +* Fix bootstrapping failures when using letsencrypt-auto with --no-self-upgrade +* Fix problems with parsing renewal config files from private beta More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 -# 0.4.0 -## 02/10/2016 +## 0.4.0 - 2016-02-10 + +### Added * The new verb/subcommand `renew` can be used to renew your existing certificates as they approach expiration. Running `letsencrypt renew` @@ -290,13 +354,13 @@ stability, security, and performance of the script. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 -# 0.3.0 -## 01/27/2016 +## 0.3.0 - 2016-01-27 -* A non-interactive mode which can be enabled by including `-n` or -`--non-interactive` on the command line. This can be used to -guarantee the client will not prompt when run automatically using -cron/systemd. +### Added + +* Add a non-interactive mode which can be enabled by including `-n` or +`--non-interactive` on the command line. This can be used to guarantee +the client will not prompt when run automatically using cron/systemd. * Preparation for the new letsencrypt-auto script. Over the past couple months, we've been working on increasing the reliability and security of letsencrypt-auto. A number of changes landed in this @@ -305,8 +369,9 @@ release to prepare for the new version of this script. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 -# 0.2.0 -## 01/14/2016 +## 0.2.0 - 2016-01-14 + +### Added * Apache plugin support for non-Debian based systems. Support has been added for modern Red Hat based systems such as Fedora 23, Red Hat 7, @@ -314,23 +379,30 @@ and CentOS 7 running Apache 2.4. In theory, this plugin should be able to be configured to run on any Unix-like OS running Apache 2.4. * Relaxed PyOpenSSL version requirements. This adds support for systems with PyOpenSSL versions 0.13 or 0.14. +* Improved error messages from the client. + +### Fixed + * Resolves issues with the Apache plugin enabling an HTTP to HTTPS redirect on some systems. -* Improved error messages from the client. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 -# 0.1.1 -## 12/15/2015 +## 0.1.1 - 2015-12-15 + +### Added -* Fix a confusing UI path that caused some users to repeatedly renew -their certs while experimenting with the client, in some cases -hitting issuance rate limits -* Fixes numerous Apache configuration parser fixes * Avoids attempting to issue for unqualified domain names like "localhost" -* Fixes --webroot permission handling for non-root users + +### Fixed + +* Fix a confusing UI path that caused some users to repeatedly renew +their certs while experimenting with the client, in some cases hitting +issuance rate limits +* Fix numerous Apache configuration parser problems +* Fix --webroot permission handling for non-root users More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 From 2bdfffb23a079f1df6bc6cad40de2436e7897adb Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Tue, 4 Apr 2017 16:12:31 -0700 Subject: [PATCH 097/128] Support "certbot-auto --no-bootstrap" (#3955) * Support "certbot-auto --no-bootstrap" * Tell people about --no-bootstrap? * Document new certbot-auto flag in its cli help * Rebuild * Less variables is less variability * Alphabetize help * Make it extra clear we only take one branch * Add --no-bootstrap message to experimentalbootstrap exit --- certbot/cli.py | 5 ++++ letsencrypt-auto-source/letsencrypt-auto | 28 +++++++++++++------ .../letsencrypt-auto.template | 28 +++++++++++++------ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 0442f481d..1c91935d3 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -952,6 +952,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="(certbot-auto only) prevent the certbot-auto script from" " upgrading itself to newer released versions (default: Upgrade" " automatically)") + helpful.add( + "automation", "--no-bootstrap", action="store_true", + help="(certbot-auto only) prevent the certbot-auto script from" + " installing OS-level dependencies (default: Prompt to install " + " OS-wide dependencies, but exit if the user says 'No')") helpful.add( ["automation", "renew", "certonly", "run"], "-q", "--quiet", dest="quiet", action="store_true", diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 35c7a530c..af278b118 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -576,21 +586,23 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -615,7 +627,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 327799210..4e07bec02 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -206,21 +216,23 @@ DeterminePythonVersion() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -245,7 +257,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 From 18486d6ea0a9695efea8fa9e639afae5bb77afc4 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Wed, 5 Apr 2017 12:45:03 -0700 Subject: [PATCH 098/128] Revert "More thoroughly rename during `certbot rename`. (#4320)" (#4467) This reverts commit 43dccfc67187b0ae3d677fbe0c7fffab20e6179c. --- certbot/cert_manager.py | 37 ++--------- certbot/cli.py | 15 +---- certbot/main.py | 9 ++- certbot/storage.py | 102 +++-------------------------- certbot/tests/cert_manager_test.py | 21 ++---- 5 files changed, 27 insertions(+), 157 deletions(-) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 7b1811c99..35d539a16 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -13,7 +13,6 @@ from certbot import storage from certbot import util from certbot.display import util as display_util -from certbot.plugins import disco as plugins_disco logger = logging.getLogger(__name__) @@ -47,7 +46,6 @@ def rename_lineage(config): certname = _get_certname(config, "rename") - # what is the new name we want to use? new_certname = config.new_certname if not new_certname: code, new_certname = disp.input( @@ -56,34 +54,13 @@ def rename_lineage(config): if code != display_util.OK or not new_certname: raise errors.Error("User ended interaction.") - try: - # copy files to new name - new_lineage = storage.duplicate_lineage(config, certname, new_certname) - - # install the new name's files - config.certname = new_certname - plugins = plugins_disco.PluginsRegistry.find_all() - from certbot.main import install - install(config, plugins, new_lineage, False) - except (errors.CertStorageError, errors.ConfigurationError, IOError, OSError) as e: - # delete the new files - config.certname = new_certname - # we might not have created anything to delete - try: - storage.delete_files(config, new_certname) - except errors.CertStorageError: - pass - reporter = zope.component.getUtility(interfaces.IReporter) - reporter.add_message("Unable to rename certificate", reporter.HIGH_PRIORITY) - raise e - else: - # delete old files - config.certname = certname - storage.delete_files(config, certname) - disp.notification("Renamed files for {0} to {1}." - .format(certname, new_certname), pause=False) - finally: - config.certname = certname + lineage = lineage_for_certname(config, certname) + if not lineage: + raise errors.ConfigurationError("No existing certificate with name " + "{0} found.".format(certname)) + storage.rename_renewal_config(certname, new_certname, config) + disp.notification("Successfully renamed {0} to {1}." + .format(certname, new_certname), pause=False) def certificates(config): """Display information about certs configured with Certbot diff --git a/certbot/cli.py b/certbot/cli.py index 1c91935d3..cb769872e 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -81,7 +81,6 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certs you have from Certbot revoke Revoke a certificate (supply --cert-path) - rename Rename a certificate delete Delete a certificate manage your account with Let's Encrypt: @@ -364,10 +363,6 @@ VERB_HELP = [ "opts": "Options for revocation of certs", "usage": "\n\n certbot revoke --cert-path /path/to/fullchain.pem [options]\n\n" }), - ("rename", { - "short": "Change a certificate's name (for management purposes)", - "opts": "Options for changing certificate names" - }), ("register", { "short": "Register for account with Let's Encrypt / other ACME server", "opts": "Options for account registration & modification" @@ -425,7 +420,6 @@ class HelpfulArgumentParser(object): "plugins": main.plugins_cmd, "register": main.register, "unregister": main.unregister, - "rename": main.rename, "renew": main.renew, "revoke": main.revoke, "rollback": main.rollback, @@ -796,7 +790,7 @@ def _add_all_groups(helpful): helpful.add_group("paths", description="Arguments changing execution paths & servers") helpful.add_group("manage", description="Various subcommands and flags are available for managing your certificates:", - verbs=["certificates", "delete", "rename", "renew", "revoke", "update_symlinks"]) + verbs=["certificates", "delete", "renew", "revoke", "update_symlinks"]) # VERBS for verb, docs in VERB_HELP: @@ -849,17 +843,12 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "multiple -d flags or enter a comma separated list of domains " "as a parameter. (default: Ask)") helpful.add( - [None, "run", "certonly", "manage", "rename", "delete", "certificates"], + [None, "run", "certonly", "manage", "delete", "certificates"], "--cert-name", dest="certname", metavar="CERTNAME", default=None, help="Certificate name to apply. Only one certificate name can be used " "per Certbot run. To see certificate names, run 'certbot certificates'. " "When creating a new certificate, specifies the new certificate's name.") - helpful.add( - ["rename", "manage"], - "--updated-cert-name", dest="new_certname", - metavar="NEW_CERTNAME", default=None, - help="New name for the certificate. Must be a valid filename.") helpful.add( [None, "testing", "renew", "certonly"], "--dry-run", action="store_true", dest="dry_run", diff --git a/certbot/main.py b/certbot/main.py index 256503d4e..090479d20 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -460,16 +460,15 @@ def register(config, unused_plugins): eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) -def _install_cert(config, le_client, domains, lineage=None, enhance=True): +def _install_cert(config, le_client, domains, lineage=None): path_provider = lineage if lineage else config assert path_provider.cert_path is not None le_client.deploy_certificate(domains, path_provider.key_path, path_provider.cert_path, path_provider.chain_path, path_provider.fullchain_path) - if enhance: - le_client.enhance_config(domains, path_provider.chain_path) + le_client.enhance_config(domains, path_provider.chain_path) -def install(config, plugins, lineage=None, enhance=True): +def install(config, plugins): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert # FIXME: be consistent about whether errors are raised or returned from @@ -482,7 +481,7 @@ def install(config, plugins, lineage=None, enhance=True): domains, _ = _find_domains_or_certname(config, installer) le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains, lineage, enhance) + _install_cert(config, le_client, domains) def plugins_cmd(config, plugins): # TODO: Use IDisplay rather than print diff --git a/certbot/storage.py b/certbot/storage.py index 34dc57884..a1462b72d 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -34,9 +34,7 @@ def renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def renewal_file_for_certname(config, certname): - """Return /path/to/certname.conf in the renewal conf directory - :raises .CertStorageError: if file is missing - """ + """Return /path/to/certname.conf in the renewal conf directory""" path = os.path.join(config.renewal_configs_dir, "{0}.conf".format(certname)) if not os.path.exists(path): raise errors.CertStorageError("No certificate found with name {0} (expected " @@ -132,8 +130,6 @@ def rename_renewal_config(prev_name, new_name, cli_config): except OSError: raise errors.ConfigurationError("Please specify a valid filename " "for the new certificate name.") - else: - return new_filename def update_configuration(lineagename, archive_dir, target, cli_config): @@ -150,25 +146,19 @@ def update_configuration(lineagename, archive_dir, target, cli_config): """ config_filename = renewal_filename_for_lineagename(cli_config, lineagename) - - def _save_renewal_values(unused_config, temp_filename): - # Save only the config items that are relevant to renewal - values = relevant_values(vars(cli_config.namespace)) - write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - _modify_config_with_tempfile(config_filename, _save_renewal_values) - - return configobj.ConfigObj(config_filename) - -def _modify_config_with_tempfile(filename, function): - temp_filename = filename + ".new" + temp_filename = config_filename + ".new" # If an existing tempfile exists, delete it if os.path.exists(temp_filename): os.unlink(temp_filename) - config = configobj.ConfigObj(filename) - function(config, temp_filename) - os.rename(temp_filename, filename) + # Save only the config items that are relevant to renewal + values = relevant_values(vars(cli_config.namespace)) + write_renewal_config(config_filename, temp_filename, archive_dir, target, values) + os.rename(temp_filename, config_filename) + + return configobj.ConfigObj(config_filename) + def get_link_target(link): """Get an absolute path to the target of link. @@ -253,7 +243,6 @@ def delete_files(config, certname): """Delete all files related to the certificate. If some files are not found, ignore them and continue. - :raises .CertStorageError: if lineage is missing """ renewal_filename = renewal_file_for_certname(config, certname) # file exists @@ -315,79 +304,6 @@ def delete_files(config, certname): except OSError: logger.debug("Unable to remove %s", archive_path) -def duplicate_lineage(config, certname, new_certname): - """Create a duplicate of certname with name new_certname - - :raises .CertStorageError: for storage errors - :raises .ConfigurationError: for cli and renewal configuration errors - :raises IOError: for filename errors - :raises OSError: for OS errors - """ - - # copy renewal config file - prev_filename = renewal_filename_for_lineagename(config, certname) - new_filename = renewal_filename_for_lineagename(config, new_certname) - if os.path.exists(new_filename): - raise errors.ConfigurationError("The new certificate name " - "is already in use.") - try: - shutil.copy2(prev_filename, new_filename) - except (OSError, IOError): - raise errors.ConfigurationError("Please specify a valid filename " - "for the new certificate name.") - logger.debug("Copied %s to %s", prev_filename, new_filename) - - # load config file - try: - renewal_config = configobj.ConfigObj(new_filename) - except configobj.ConfigObjError: - # config is corrupted - logger.warning("Could not parse %s. Only the certificate has been renamed.", - new_filename) - raise errors.CertStorageError( - "error parsing {0}".format(new_filename)) - - def copy_to_new_dir(prev_dir): - """Replace certname with new_certname in prev_dir""" - new_dir = prev_dir.replace(certname, new_certname) - # make dir iff it doesn't exist - shutil.copytree(prev_dir, new_dir, symlinks=True) - logger.debug("Copied %s to %s", prev_dir, new_dir) - return new_dir - - # archive dir - prev_archive_dir = _full_archive_path(renewal_config, config, certname) - new_archive_dir = prev_archive_dir - if not certname in prev_archive_dir: - raise errors.CertStorageError("Archive directory does not conform to defaults: " - "{0} not in {1}", certname, prev_archive_dir) - else: - new_archive_dir = copy_to_new_dir(prev_archive_dir) - - # live dir - # if things aren't in their default places, don't try to change things. - prev_live_dir = _full_live_path(config, certname) - prev_links = dict((kind, renewal_config.get(kind)) for kind in ALL_FOUR) - if (certname not in prev_live_dir or - len(set(os.path.dirname(renewal_config.get(kind)) for kind in ALL_FOUR)) != 1): - raise errors.CertStorageError("Live directory does not conform to defaults.") - else: - copy_to_new_dir(prev_live_dir) - new_links = dict((k, prev_links[k].replace(certname, new_certname)) for k in prev_links) - - # Update renewal config file - def _update_and_write(renewal_config, temp_filename): - renewal_config["archive_dir"] = new_archive_dir - renewal_config["version"] = certbot.__version__ - for kind in ALL_FOUR: - renewal_config[kind] = new_links[kind] - with open(temp_filename, "wb") as f: - renewal_config.write(outfile=f) - _modify_config_with_tempfile(new_filename, _update_and_write) - - # Update symlinks - return RenewableCert(new_filename, config, update_symlinks=True) - class RenewableCert(object): # pylint: disable=too-many-instance-attributes,too-many-public-methods diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index d0e563979..b5731fbd6 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -384,19 +384,14 @@ class RenameLineageTest(BaseCertManagerTest): @test_util.patch_get_utility() @mock.patch('certbot.cert_manager.lineage_for_certname') def test_no_existing_certname(self, mock_lineage_for_certname, unused_get_utility): - mock_config = mock.Mock(certname="one", new_certname="two", - renewal_configs_dir="/tmp/etc/letsencrypt/renewal/") + mock_config = mock.Mock(certname="one", new_certname="two") mock_lineage_for_certname.return_value = None - self.assertRaises(errors.ConfigurationError, self._call, mock_config) + self.assertRaises(errors.ConfigurationError, + self._call, mock_config) - @mock.patch("certbot.storage.RenewableCert._update_symlinks") @test_util.patch_get_utility() @mock.patch("certbot.storage.RenewableCert._check_symlinks") - @mock.patch("certbot.storage.relevant_values") - def test_rename_cert(self, mock_rv, mock_check, unused_get_utility, unused_update_symlinks): - # Mock relevant_values() to claim that all values are relevant here - # (to avoid instantiating parser) - mock_rv.side_effect = lambda x: x + def test_rename_cert(self, mock_check, unused_get_utility): mock_check.return_value = True mock_config = self.mock_config self._call(mock_config) @@ -405,15 +400,9 @@ class RenameLineageTest(BaseCertManagerTest): self.assertTrue(updated_lineage is not None) self.assertEqual(updated_lineage.lineagename, mock_config.new_certname) - @mock.patch("certbot.storage.RenewableCert._update_symlinks") @test_util.patch_get_utility() @mock.patch("certbot.storage.RenewableCert._check_symlinks") - @mock.patch("certbot.storage.relevant_values") - def test_rename_cert_interactive_certname(self, mock_rv, mock_check, mock_get_utility, - unused_update_symlinks): - # python 3.4 and 3.5 order things differently, so remove other.com for this test - os.remove(self.configs["other.com"].filename) - mock_rv.side_effect = lambda x: x + def test_rename_cert_interactive_certname(self, mock_check, mock_get_utility): mock_check.return_value = True mock_config = self.mock_config mock_config.certname = None From 5259901d04ba7cce19380109fff881d138bb432a Mon Sep 17 00:00:00 2001 From: Zero King Date: Thu, 6 Apr 2017 11:02:00 +0000 Subject: [PATCH 099/128] Fix link in README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e44a61021..fcf956a34 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Certbot is part of EFF’s effort to encrypt the entire Internet. Secure communi Anyone who has gone through the trouble of setting up a secure website knows what a hassle getting and maintaining a certificate is. Certbot and Let’s Encrypt can automate away the pain and let you turn on and manage HTTPS with simple commands. Using Certbot and Let's Encrypt is free, so there’s no need to arrange payment. -How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. +How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issues by Let’s Encrypt. From 82f3f7523e60236d46c057ea9efb7aafb4a6fe47 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 6 Apr 2017 11:33:32 -0700 Subject: [PATCH 100/128] Candidate 0.13.0 (#4475) * Release 0.13.0 * Bump version to 0.14.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 61 +++++++++++------- certbot-compatibility-test/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 15 +++-- letsencrypt-auto | 61 +++++++++++------- letsencrypt-auto-source/certbot-auto.asc | 16 ++--- letsencrypt-auto-source/letsencrypt-auto | 26 ++++---- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/letsencrypt-auto-requirements.txt | 24 +++---- 12 files changed, 124 insertions(+), 89 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index df9937032..a640ae6bb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.13.0.dev0' +version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index cb35d2686..9a473c584 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.13.0.dev0' +version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index 54cc429cf..fc8007c9e 100755 --- a/certbot-auto +++ b/certbot-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.12.0" +LE_AUTO_VERSION="0.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -576,21 +586,23 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -615,7 +627,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 @@ -833,18 +845,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.12.0 \ - --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ - --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf -certbot==0.12.0 \ - --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ - --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 -certbot-apache==0.12.0 \ - --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ - --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 -certbot-nginx==0.12.0 \ - --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ - --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1093,6 +1105,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1209,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 238d7a2d5..aecae329f 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.13.0.dev0' +version = '0.14.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index bdc45b9b4..786b5a1a1 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.13.0.dev0' +version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 0c667378d..228a1a2a6 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.13.0.dev0' +__version__ = '0.14.0.dev0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 9ef9d9e6c..a5f77a3a1 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -70,6 +70,8 @@ optional arguments: because they may be necessary to accurately simulate renewal. --renew-hook commands are not called. (default: False) + --debug-challenges After setting up challenges, wait for user input + before submitting to CA (default: False) --preferred-challenges PREF_CHALLS A sorted, comma delimited list of the preferred challenge to use during authorization with the most @@ -86,7 +88,7 @@ optional arguments: statistics about success rates by OS and plugin. If you wish to hide your server OS version from the Let's Encrypt server, set this to "". (default: - CertbotACMEClient/0.12.0 (Ubuntu 16.04.2 LTS) + CertbotACMEClient/0.13.0 (Ubuntu 16.04.2 LTS) Authenticator/XXX Installer/YYY) automation: @@ -128,6 +130,10 @@ automation: --no-self-upgrade (certbot-auto only) prevent the certbot-auto script from upgrading itself to newer released versions (default: Upgrade automatically) + --no-bootstrap (certbot-auto only) prevent the certbot-auto script + from installing OS-level dependencies (default: Prompt + to install OS-wide dependencies, but exit if the user + says 'No') -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -193,6 +199,9 @@ paths: installed from, or revoked. (default: None) --key-path KEY_PATH Path to private key for cert installation or revocation (if account key is missing) (default: None) + --fullchain-path FULLCHAIN_PATH + Accompanying path to a full certificate chain (cert + plus chain). (default: None) --chain-path CHAIN_PATH Accompanying path to a certificate chain. (default: None) @@ -315,10 +324,6 @@ unregister: install: Options for modifying how a cert is deployed - --fullchain-path FULLCHAIN_PATH - Accompanying path to a full certificate chain (cert - plus chain). (default: None) - config_changes: Options for controlling which changes are displayed diff --git a/letsencrypt-auto b/letsencrypt-auto index 54cc429cf..fc8007c9e 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.12.0" +LE_AUTO_VERSION="0.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -576,21 +586,23 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -615,7 +627,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 @@ -833,18 +845,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.12.0 \ - --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ - --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf -certbot==0.12.0 \ - --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ - --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 -certbot-apache==0.12.0 \ - --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ - --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 -certbot-nginx==0.12.0 \ - --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ - --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1093,6 +1105,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1209,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 417d43387..bf267dc25 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v1 +Version: GnuPG v2 -iQEcBAABAgAGBQJYuJdQAAoJEE0XyZXNl3Xyw+oH/1AQ90P3397rKB0jP+5MchtR -Nz4ScKL86x9s+o/OzAN76gLhJNj/gOVWoyeK8wVkJ07MpbGyLBiYFsXPZWYUcJ77 -LRj4sGAxJatptHG+PnzIquAf+swynqVu0QdBv8ImKwYrqOlULR+Kr8QZE95Ena51 -JPkbm5o0ipSbByIpraAYabCOHj7SrsFQtMx+tPTd7xaliO8VkguzLQt93QQC7CNj -JIO/yURnfKzutTOe3OPzBzbb6e2yhHcHZcSyv8S0DCIAoB08N9Bs8aAbVwmD89Fq -fwYxLZherXRZ2VtJ2Sf/hUP2ZrEH/mvCkKjzznZokFGJXLvTEc8fC/O6c/q/nLw= -=YiSx +iQEcBAABCAAGBQJY5WxEAAoJEE0XyZXNl3XyoDYH/joyJ/7cS4+SoTEiPpVcDnK+ +YJVhxP6pir6GaRvl+ebWlo7ichS4c0Kye8e5BPVj5RtZbDT88iplMZ2EyUmeA579 +8Z96p9qoEANeGWiPe+KCDXRHJfCAsphcHSLTeS8lXgG8SP13p7hsML6hn3gosRdu +OG4/SnFBDLLwu4YwUVom4U+Z+dYS1jQstge4sexr85jCX/Lds7M5WM/lFiYMBsJ8 +uZd/IGKwb7jvsc4u58Ruj9xiTcchaxn15NMJR7R967Mt5ortSvZ3C6Cv3NyubJmB +hmGQVU+eNBTeEwPSIN8xAf3fcwh2wlRMaTZOy5nJ3IoDdSQuwO9IGxxdkNDSegE= +=8KUq -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index af278b118..84235cb68 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.13.0.dev0" +LE_AUTO_VERSION="0.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -845,18 +845,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.12.0 \ - --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ - --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf -certbot==0.12.0 \ - --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ - --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 -certbot-apache==0.12.0 \ - --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ - --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 -certbot-nginx==0.12.0 \ - --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ - --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 711300dda497b4f0de469e0d5f9fdeffcc0ed791..723cc2f8ca6c97bde3befac7184eaa55565dfd11 100644 GIT binary patch literal 256 zcmV+b0ssCL!AY$O$G=(YPL-?HISZVL;uLSvo4ezuc`p!goY|_rlo`sQjXmnLA??J3 zG3#@-hsQdZ3;EW6kL#){9%73pZgfPO^V{n`wAQwiKV_n~hA*hATMAtDbe^FV?7KHMU7&ibA zY)(7YTayW2J7xLFKD;j4uGs>>g0Ud^f#2rW8lI&1oBpWdE0zySF(Jdneg#Byz|O6G z%PaA(4sIzeHzBn34||(^k7n*P^+ z5OMKIYIwR>5CX?W?*GvZf6nD9_+<;knP+ME`_3x{Odk_e&tozlwSx$&=Gd}j4{T;d zN=0Y**I$U*TkAErI2%6WFOxeoL@LtX^L?L>(^-}P0vSB0ac_x}l1H9mca`Z>jI7HK ze*M3p)+_=!N|Mdxjmt%{u8Za)+;9^WpG(BR|)b z`MuJ4TFdF^IQv0a>mH6LZ6Uwm>Ao3CY`8B~Piue4_sWH{fV;aQFnC4iPVzu|)T9g< GmtWvEc7D?U diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index d70d24e2a..325bdf84f 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -171,15 +171,15 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.12.0 \ - --hash=sha256:a6050619b3e07b41d197992bb15b32c755dfa0665cfa1c20faa82806a798265b \ - --hash=sha256:a05cba6b5b0fffdfa246b32492a44769011d45205f3ee8efde1f37ee9843fbdf -certbot==0.12.0 \ - --hash=sha256:d018d13665eb4cfe7038c2df636e3f4928742b83769b95edfdb0311277f0eb48 \ - --hash=sha256:4a71925c035b62dfb7c3343c619ee090add76188b47225272b57798ad63388b7 -certbot-apache==0.12.0 \ - --hash=sha256:de86907ea60e7bc35d252b87dec04eab3c7f3a1ea768774876e7ff582d89d640 \ - --hash=sha256:77dde63cf97292b09da8ae09ef8a7a6d83a3b1ee0f8d1fefe513fc77a6292509 -certbot-nginx==0.12.0 \ - --hash=sha256:c66d848c4577f1f91a06a8119b40f1ab90af1546addea27905434bd070f3924d \ - --hash=sha256:4dab2c93304c80d8d0d2e5214939f016804fd46859dd7a39b892d8b7195ab5ec +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 From da1cfa85fc57d2fba0ce81ee1a2d693d57ab5ae5 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 6 Apr 2017 12:07:17 -0700 Subject: [PATCH 101/128] Update changelog for 0.13.0 release (#4476) --- CHANGELOG.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2325019bf..572a9a6b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,27 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.12.0 - 2017-02-03 +## 0.13.0 - 2017-04-06 + +### Added + +* `--debug-challenges` pauses Certbot after setting up challenges for debugging. +* The Nginx parser can handle all valid directives in configuration files. +* Nginx ciphersuites changed to Mozilla Intermediate. +* `certbot-auto --no-bootstrap` won't install OS dependencies. + +### Fixed + +* `--register-unsafely-without-email` respects `--quiet`. +* Hyphenated renewalparams are now saved in renewal config files. +* `--dry-run` no longer persists keys and csrs. +* No longer hangs when trying to start Nginx in Arch Linux. +* Apache rewrite rules no longer double-encode characters. + +A full list of changes is available on GitHub: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.13.0%20is%3Aclosed%20 + +## 0.12.0 - 2017-03-02 ### Added From 2e8a5ef477caaf4ca267496096231b8b4daedc1f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 6 Apr 2017 16:05:54 -0700 Subject: [PATCH 102/128] Call certbot client cb_client rather than acme_client (#4357) In some sense, certbot.client.Client is an ACME client, but it's the not the client in the ACME library and this leads to confusion. Let's make what this is clear. * call certbot client cb_client rather than acme_client * update tests --- certbot/main.py | 10 +++++----- certbot/tests/main_test.py | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index 090479d20..8c3ecd6bc 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -407,10 +407,10 @@ def unregister(config, unused_plugins): return "Deactivation aborted." acc, acme = _determine_account(config) - acme_client = client.Client(config, acc, None, None, acme=acme) + cb_client = client.Client(config, acc, None, None, acme=acme) # delete on boulder - acme_client.acme.deactivate_registration(acc.regr) + cb_client.acme.deactivate_registration(acc.regr) account_files = account.AccountFileStorage(config) # delete local account files account_files.delete(config.account) @@ -452,11 +452,11 @@ def register(config, unused_plugins): config.email = display_ops.get_email(optional=False) acc, acme = _determine_account(config) - acme_client = client.Client(config, acc, None, None, acme=acme) + cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. - acc.regr = acme_client.acme.update_registration(acc.regr.update( + acc.regr = cb_client.acme.update_registration(acc.regr.update( body=acc.regr.body.update(contact=('mailto:' + config.email,)))) - account_storage.save_regr(acc, acme_client.acme) + account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 1afe95924..562c4bb9d 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1093,8 +1093,8 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] mocked_det.return_value = (mock.MagicMock(), "foo") - acme_client = mock.MagicMock() - mocked_client.Client.return_value = acme_client + cb_client = mock.MagicMock() + mocked_client.Client.return_value = cb_client x = self._call_no_clientmock( ["register", "--update-registration"]) # When registration change succeeds, the return value @@ -1103,7 +1103,7 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me # and we got supposedly did update the registration from # the server self.assertTrue( - acme_client.acme.update_registration.called) + cb_client.acme.update_registration.called) # and we saved the updated registration on disk self.assertTrue(mocked_storage.save_regr.called) self.assertTrue( @@ -1143,8 +1143,8 @@ class UnregisterTest(unittest.TestCase): self.mocks['account'].AccountFileStorage.return_value = mocked_storage self.mocks['_determine_account'].return_value = (mock.MagicMock(), "foo") - acme_client = mock.MagicMock() - self.mocks['client'].Client.return_value = acme_client + cb_client = mock.MagicMock() + self.mocks['client'].Client.return_value = cb_client config = mock.MagicMock() unused_plugins = mock.MagicMock() @@ -1152,7 +1152,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) self.assertTrue(res is None) - self.assertTrue(acme_client.acme.deactivate_registration.called) + self.assertTrue(cb_client.acme.deactivate_registration.called) m = "Account deactivated." self.assertTrue(m in self.mocks['get_utility']().add_message.call_args[0][0]) @@ -1161,8 +1161,8 @@ class UnregisterTest(unittest.TestCase): mocked_storage.find_all.return_value = [] self.mocks['account'].AccountFileStorage.return_value = mocked_storage - acme_client = mock.MagicMock() - self.mocks['client'].Client.return_value = acme_client + cb_client = mock.MagicMock() + self.mocks['client'].Client.return_value = cb_client config = mock.MagicMock() unused_plugins = mock.MagicMock() @@ -1170,7 +1170,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) m = "Could not find existing account to deactivate." self.assertEqual(res, m) - self.assertFalse(acme_client.acme.deactivate_registration.called) + self.assertFalse(cb_client.acme.deactivate_registration.called) if __name__ == '__main__': From cacee80c51fe276c8284769229a9b5e6565d9988 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 7 Apr 2017 09:43:33 -0700 Subject: [PATCH 103/128] Move make_csr into acme.crypto_util (#4165) It's relatively finicky to make a CSR appropriate to pass to poll_and_request_issuance. I think most users want to be able to give a list of domains and a private key, and get back a CSR. This branch adds that functionality to crypto_util. Note that the two new functions take arguments, and return values, as PEM-encoded buffers. This is a departure from some existing ACME interfaces that take PyOpenSSL types. I've discussed with the Certbot team, and we agree that this is broadly the direction the ACME API should take, so that users of the module don't need to import PyOpenSSL themselves, or use its primitives. * Add make_csr. * accept privkey * Tweak API. * Remove make_csr from certbot package. * Skip test in older Pythons. * Move get_Extensions call under protection. * Remove assertIn because not backwards-compatible. * Fix encoding, and use PEM. * Fix test * Fix tests on py35. * Fix error in test. * Make import_csr_file always return PEM. Also delete get_sans_from_csr (unused) and get_names_from_csr (newly unused). * Fix function doc. * Fix indent * Fix call of obtain_certificate_from_Csr * lint * Handle review feedback. * Fix test. --- acme/acme/crypto_util.py | 30 +++++++ acme/acme/crypto_util_test.py | 47 +++++++++++ certbot/client.py | 7 +- certbot/crypto_util.py | 107 ++++++------------------ certbot/main.py | 4 +- certbot/tests/client_test.py | 8 +- certbot/tests/crypto_util_test.py | 126 +++-------------------------- certbot/tests/testdata/csr-san.der | Bin 370 -> 0 bytes 8 files changed, 122 insertions(+), 207 deletions(-) delete mode 100644 certbot/tests/testdata/csr-san.der diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 6a33b3e52..f86a9971a 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -149,6 +149,36 @@ def probe_sni(name, host, port=443, timeout=300, raise errors.Error(error) return client_ssl.get_peer_certificate() +def make_csr(private_key_pem, domains, must_staple=False): + """Generate a CSR containing a list of domains as subjectAltNames. + + :param buffer private_key_pem: Private key, in PEM PKCS#8 format. + :param list domains: List of DNS names to include in subjectAltNames of CSR. + :param bool must_staple: Whether to include the TLS Feature extension (aka + OCSP Must Staple: https://tools.ietf.org/html/rfc7633). + :returns: buffer PEM-encoded Certificate Signing Request. + """ + private_key = OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, private_key_pem) + csr = OpenSSL.crypto.X509Req() + extensions = [ + OpenSSL.crypto.X509Extension( + b'subjectAltName', + critical=False, + value=', '.join('DNS:' + d for d in domains).encode('ascii') + ), + ] + if must_staple: + extensions.append(OpenSSL.crypto.X509Extension( + b"1.3.6.1.5.5.7.1.24", + critical=False, + value=b"DER:30:03:02:01:05")) + csr.add_extensions(extensions) + csr.set_pubkey(private_key) + csr.set_version(2) + csr.sign(private_key, 'sha256') + return OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr) def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 9cf1f7deb..845f43914 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -151,6 +151,53 @@ class RandomSnTest(unittest.TestCase): self.serial_num.append(cert.get_serial_number()) self.assertTrue(len(set(self.serial_num)) > 1) +class MakeCSRTest(unittest.TestCase): + """Test for standalone functions.""" + + @classmethod + def _call_with_key(cls, *args, **kwargs): + privkey = OpenSSL.crypto.PKey() + privkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + privkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey) + from acme.crypto_util import make_csr + return make_csr(privkey_pem, *args, **kwargs) + + def test_make_csr(self): + csr_pem = self._call_with_key(["a.example", "b.example"]) + self.assertTrue(b'--BEGIN CERTIFICATE REQUEST--' in csr_pem) + self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr + # objects don't have a get_extensions() method, so we skip this test if + # the method isn't available. + if hasattr(csr, 'get_extensions'): + self.assertEquals(len(csr.get_extensions()), 1) + self.assertEquals(csr.get_extensions()[0].get_data(), + OpenSSL.crypto.X509Extension( + b'subjectAltName', + critical=False, + value=b'DNS:a.example, DNS:b.example', + ).get_data(), + ) + + def test_make_csr_must_staple(self): + csr_pem = self._call_with_key(["a.example"], must_staple=True) + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_pem) + + # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr + # objects don't have a get_extensions() method, so we skip this test if + # the method isn't available. + if hasattr(csr, 'get_extensions'): + self.assertEquals(len(csr.get_extensions()), 2) + # NOTE: Ideally we would filter by the TLS Feature OID, but + # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID, + # and the shortname field is just "UNDEF" + must_staple_exts = [e for e in csr.get_extensions() + if e.get_data() == b"0\x03\x02\x01\x05"] + self.assertEqual(len(must_staple_exts), 1, + "Expected exactly one Must Staple extension") if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/client.py b/certbot/client.py index f6de26e3d..3e9caad40 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -207,15 +207,14 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, - typ=OpenSSL.crypto.FILETYPE_ASN1, authzr=None): + def obtain_certificate_from_csr(self, domains, csr, authzr=None): """Obtain certificate. Internal function with precondition that `domains` are consistent with identifiers present in the `csr`. :param list domains: Domain names. - :param .util.CSR csr: DER-encoded Certificate Signing + :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. :param list authzr: List of @@ -241,7 +240,7 @@ class Client(object): certr = self.acme.request_issuance( jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(typ, csr.data)), + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), authzr) notify = zope.component.getUtility(interfaces.IDisplay).notification diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 1ad76d503..5671341e8 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -6,7 +6,6 @@ """ import logging import os -import traceback import OpenSSL import pyrfc3339 @@ -66,7 +65,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): return util.Key(key_path, key_pem) -def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): +def init_save_csr(privkey, names, path): """Initialize a CSR with the given private key. :param privkey: Key to include in the CSR @@ -82,8 +81,8 @@ def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): """ config = zope.component.getUtility(interfaces.IConfig) - csr_pem, csr_der = make_csr(privkey.pem, names, - must_staple=config.must_staple) + csr_pem = acme_crypto_util.make_csr( + privkey.pem, names, must_staple=config.must_staple) # Save CSR util.make_or_verify_dir(path, 0o755, os.geteuid(), @@ -93,53 +92,12 @@ def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): logger.info("Creating CSR: not saving to file") else: csr_f, csr_filename = util.unique_file( - os.path.join(path, csrname), 0o644, "wb") + os.path.join(path, "csr-certbot.pem"), 0o644, "wb") with csr_f: csr_f.write(csr_pem) logger.info("Creating CSR: %s", csr_filename) - return util.CSR(csr_filename, csr_der, "der") - - -# Lower level functions -def make_csr(key_str, domains, must_staple=False): - """Generate a CSR. - - :param str key_str: PEM-encoded RSA key. - :param list domains: Domains included in the certificate. - - .. todo:: Detect duplicates in `domains`? Using a set doesn't - preserve order... - - :returns: new CSR in PEM and DER form containing all domains - :rtype: tuple - - """ - assert domains, "Must provide one or more hostnames for the CSR." - pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, key_str) - req = OpenSSL.crypto.X509Req() - req.get_subject().CN = domains[0] - # TODO: what to put into req.get_subject()? - # TODO: put SAN if len(domains) > 1 - extensions = [ - OpenSSL.crypto.X509Extension( - b"subjectAltName", - critical=False, - value=", ".join("DNS:%s" % d for d in domains).encode('ascii') - ) - ] - if must_staple: - extensions.append(OpenSSL.crypto.X509Extension( - b"1.3.6.1.5.5.7.1.24", - critical=False, - value=b"DER:30:03:02:01:05")) - req.add_extensions(extensions) - req.set_version(2) - req.set_pubkey(pkey) - req.sign(pkey, "sha256") - return tuple(OpenSSL.crypto.dump_certificate_request(method, req) - for method in (OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.FILETYPE_ASN1)) + return util.CSR(csr_filename, csr_pem, "pem") # WARNING: the csr and private key file are possible attack vectors for TOCTOU @@ -193,22 +151,27 @@ def import_csr_file(csrfile, data): :param str csrfile: CSR filename :param str data: contents of the CSR file - :returns: (`OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`, + :returns: (`OpenSSL.crypto.FILETYPE_PEM`, util.CSR object representing the CSR, list of domains requested in the CSR) :rtype: tuple """ - for form, typ in (("der", OpenSSL.crypto.FILETYPE_ASN1,), - ("pem", OpenSSL.crypto.FILETYPE_PEM,),): + PEM = OpenSSL.crypto.FILETYPE_PEM + load = OpenSSL.crypto.load_certificate_request + try: + # Try to parse as DER first, then fall back to PEM. + csr = load(OpenSSL.crypto.FILETYPE_ASN1, data) + except OpenSSL.crypto.Error: try: - domains = get_names_from_csr(data, typ) + csr = load(PEM, data) except OpenSSL.crypto.Error: - logger.debug("CSR parse error (form=%s, typ=%s):", form, typ) - logger.debug(traceback.format_exc()) - continue - return typ, util.CSR(file=csrfile, data=data, form=form), domains - raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) + raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) + + domains = _get_names_from_loaded_cert_or_req(csr) + # Internally we always use PEM, so re-encode as PEM before returning. + data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr) + return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains def make_key(bits): @@ -290,22 +253,12 @@ def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): cert, OpenSSL.crypto.load_certificate, typ) -def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): - """Get a list of Subject Alternative Names from a CSR. - - :param str csr: CSR (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` - - :returns: A list of Subject Alternative Names. - :rtype: list - - """ - return _get_sans_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate_request, typ) - - def _get_names_from_cert_or_req(cert_or_req, load_func, typ): loaded_cert_or_req = _load_cert_or_req(cert_or_req, load_func, typ) + return _get_names_from_loaded_cert_or_req(loaded_cert_or_req) + + +def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) @@ -330,20 +283,6 @@ def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): csr, OpenSSL.crypto.load_certificate, typ) -def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): - """Get a list of domains from a CSR, including the CN if it is set. - - :param str csr: CSR (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` - - :returns: A list of domain names. - :rtype: list - - """ - return _get_names_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate_request, typ) - - def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. diff --git a/certbot/main.py b/certbot/main.py index 8c3ecd6bc..64e7d07a4 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -617,8 +617,8 @@ def _csr_get_and_save_cert(config, le_client): have the privkey, and therefore can't construct the files for a lineage. So we just save the cert & chain to disk :/ """ - csr, typ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) + csr, _ = config.actual_csr + certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 8b72e1df7..391a407ac 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -18,7 +18,7 @@ import certbot.tests.util as test_util KEY = test_util.load_vector("rsa512_key.pem") -CSR_SAN = test_util.load_vector("csr-san.der") +CSR_SAN = test_util.load_vector("csr-san.pem") class ConfigHelper(object): @@ -165,7 +165,7 @@ class ClientTest(ClientTestCommon): self.acme.request_issuance.assert_called_once_with( jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, CSR_SAN)), + OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), authzr) self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) @@ -175,7 +175,7 @@ class ClientTest(ClientTestCommon): def test_obtain_certificate_from_csr(self, unused_mock_get_utility, mock_logger): self._mock_obtain_certificate() - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler authzr = auth_handler.get_authorizations(self.eg_domains, False) @@ -246,7 +246,7 @@ class ClientTest(ClientTestCommon): mock_crypto_util): self._mock_obtain_certificate() - csr = util.CSR(form="der", file=None, data=CSR_SAN) + csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key domains = ["example.com", "www.example.com"] diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index df5d9f0f6..c83ad96b1 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -74,21 +74,20 @@ class InitSaveCSRTest(test_util.TempDirTestCase): mock.Mock(strict_permissions=True, dry_run=False), interfaces.IConfig) - @mock.patch('certbot.crypto_util.make_csr') + @mock.patch('acme.crypto_util.make_csr') @mock.patch('certbot.crypto_util.util.make_or_verify_dir') def test_success(self, unused_mock_verify, mock_csr): from certbot.crypto_util import init_save_csr - mock_csr.return_value = (b'csr_pem', b'csr_der') + mock_csr.return_value = b'csr_pem' csr = init_save_csr( - mock.Mock(pem='dummy_key'), 'example.com', self.tempdir, - 'csr-certbot.pem') + mock.Mock(pem='dummy_key'), 'example.com', self.tempdir) - self.assertEqual(csr.data, b'csr_der') + self.assertEqual(csr.data, b'csr_pem') self.assertTrue('csr-certbot.pem' in csr.file) - @mock.patch('certbot.crypto_util.make_csr') + @mock.patch('acme.crypto_util.make_csr') @mock.patch('certbot.crypto_util.util.make_or_verify_dir') def test_success_dry_run(self, unused_mock_verify, mock_csr): from certbot.crypto_util import init_save_csr @@ -96,55 +95,15 @@ class InitSaveCSRTest(test_util.TempDirTestCase): zope.component.provideUtility( mock.Mock(strict_permissions=True, dry_run=True), interfaces.IConfig) - mock_csr.return_value = (b'csr_pem', b'csr_der') + mock_csr.return_value = b'csr_pem' csr = init_save_csr( - mock.Mock(pem='dummy_key'), 'example.com', self.tempdir, - 'csr-certbot.pem') + mock.Mock(pem='dummy_key'), 'example.com', self.tempdir) - self.assertEqual(csr.data, b'csr_der') + self.assertEqual(csr.data, b'csr_pem') self.assertTrue(csr.file is None) -class MakeCSRTest(unittest.TestCase): - """Tests for certbot.crypto_util.make_csr.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot.crypto_util import make_csr - return make_csr(*args, **kwargs) - - def test_san(self): - from certbot.crypto_util import get_sans_from_csr - # TODO: Fails for RSA256_KEY - csr_pem, csr_der = self._call( - RSA512_KEY, ['example.com', 'www.example.com']) - self.assertEqual( - ['example.com', 'www.example.com'], get_sans_from_csr(csr_pem)) - self.assertEqual( - ['example.com', 'www.example.com'], get_sans_from_csr( - csr_der, OpenSSL.crypto.FILETYPE_ASN1)) - - def test_must_staple(self): - # TODO: Fails for RSA256_KEY - csr_pem, _ = self._call( - RSA512_KEY, ['example.com', 'www.example.com'], must_staple=True) - csr = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, csr_pem) - - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. - if hasattr(csr, 'get_extensions'): - # NOTE: Ideally we would filter by the TLS Feature OID, but - # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID, - # and the shortname field is just "UNDEF" - must_staple_exts = [e for e in csr.get_extensions() - if e.get_data() == b"0\x03\x02\x01\x05"] - self.assertEqual(len(must_staple_exts), 1, - "Expected exactly one Must Staple extension") - - class ValidCSRTest(unittest.TestCase): """Tests for certbot.crypto_util.valid_csr.""" @@ -162,9 +121,6 @@ class ValidCSRTest(unittest.TestCase): def test_valid_der_false(self): self.assertFalse(self._call(test_util.load_vector('csr.der'))) - def test_valid_der_san_false(self): - self.assertFalse(self._call(test_util.load_vector('csr-san.der'))) - def test_empty_false(self): self.assertFalse(self._call('')) @@ -200,12 +156,13 @@ class ImportCSRFileTest(unittest.TestCase): def test_der_csr(self): csrfile = test_util.vector_path('csr.der') data = test_util.load_vector('csr.der') + data_pem = test_util.load_vector('csr.pem') self.assertEqual( - (OpenSSL.crypto.FILETYPE_ASN1, + (OpenSSL.crypto.FILETYPE_PEM, util.CSR(file=csrfile, - data=data, - form="der"), + data=data_pem, + form="pem"), ["example.com"],), self._call(csrfile, data)) @@ -272,36 +229,6 @@ class GetSANsFromCertTest(unittest.TestCase): self._call(test_util.load_vector('cert-san.pem'))) -class GetSANsFromCSRTest(unittest.TestCase): - """Tests for certbot.crypto_util.get_sans_from_csr.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot.crypto_util import get_sans_from_csr - return get_sans_from_csr(*args, **kwargs) - - def test_extract_one_san(self): - self.assertEqual(['example.com'], self._call( - test_util.load_vector('csr.pem'))) - - def test_extract_two_sans(self): - self.assertEqual(['example.com', 'www.example.com'], self._call( - test_util.load_vector('csr-san.pem'))) - - def test_extract_six_sans(self): - self.assertEqual(self._call(test_util.load_vector('csr-6sans.pem')), - ["example.com", "example.org", "example.net", - "example.info", "subdomain.example.com", - "other.subdomain.example.com"]) - - def test_parse_non_csr(self): - self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there") - - def test_parse_no_sans(self): - self.assertEqual( - [], self._call(test_util.load_vector('csr-nosans.pem'))) - - class GetNamesFromCertTest(unittest.TestCase): """Tests for certbot.crypto_util.get_names_from_cert.""" @@ -327,36 +254,9 @@ class GetNamesFromCertTest(unittest.TestCase): ['example.com'] + ['{0}.example.com'.format(c) for c in 'abcd'], self._call(test_util.load_vector('cert-5sans.pem'))) - -class GetNamesFromCSRTest(unittest.TestCase): - """Tests for certbot.crypto_util.get_names_from_csr.""" - @classmethod - def _call(cls, *args, **kwargs): - from certbot.crypto_util import get_names_from_csr - return get_names_from_csr(*args, **kwargs) - - def test_extract_one_san(self): - self.assertEqual(['example.com'], self._call( - test_util.load_vector('csr.pem'))) - - def test_extract_two_sans(self): - self.assertEqual(set(('example.com', 'www.example.com',)), set( - self._call(test_util.load_vector('csr-san.pem')))) - - def test_extract_six_sans(self): - self.assertEqual( - set(self._call(test_util.load_vector('csr-6sans.pem'))), - set(("example.com", "example.org", "example.net", - "example.info", "subdomain.example.com", - "other.subdomain.example.com",))) - - def test_parse_non_csr(self): + def test_parse_non_cert(self): self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there") - def test_parse_no_sans(self): - self.assertEqual(["example.org"], - self._call(test_util.load_vector('csr-nosans.pem'))) - class CertLoaderTest(unittest.TestCase): """Tests for certbot.crypto_util.pyopenssl_load_certificate""" diff --git a/certbot/tests/testdata/csr-san.der b/certbot/tests/testdata/csr-san.der deleted file mode 100644 index 68fd38723ddd62a2d60fd6c27e67925d273d51f8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 370 zcmXqLV$3sWVw7NFWH6{S`GxkDEsS*Ho7m>sw0v z-yV9(#LURRxWLN50&Y4dpP{yarhz)p5we0T3I=jb$PQ`ZFE20GLv|UGySRbwa%4zJ z@cA$|+wI(oIT5 Date: Sat, 8 Apr 2017 00:46:27 +0800 Subject: [PATCH 104/128] Fix unorderable types error (#4409) If the updated datetime collides, the comparator of heapq will move onto the AuthorizationResource value and throws an "unorderable type" error. This adds an index value to the element tuple to ensure that they are always strictly ordered. --- acme/acme/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index 0c8886fc6..a069876d5 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -360,14 +360,18 @@ class Client(object): # pylint: disable=too-many-instance-attributes # priority queue with datetime.datetime (based on Retry-After) as key, # and original Authorization Resource as value - waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] + waiting = [ + (datetime.datetime.now(), index, authzr) + for index, authzr in enumerate(authzrs) + ] + heapq.heapify(waiting) # mapping between original Authorization Resource and the most # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) while waiting: # find the smallest Retry-After, and sleep if necessary - when, authzr = heapq.heappop(waiting) + when, index, authzr = heapq.heappop(waiting) now = datetime.datetime.now() if when > now: seconds = (when - now).seconds @@ -386,7 +390,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes if attempts[authzr] < max_attempts: # push back to the priority queue, with updated retry_after heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + response, default=mintime), index, authzr)) else: exhausted.add(authzr) From d557c39c999605dc726d9c866b688843ff4c7081 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 7 Apr 2017 09:52:12 -0700 Subject: [PATCH 105/128] Remove obsolete fields from Registration. (#4339) Authorizations and certificates were in the original spec but have since been deleted. --- acme/acme/messages.py | 22 ---------------------- acme/acme/messages_test.py | 3 +-- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4070290ad..5784e8e11 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -237,10 +237,6 @@ class Registration(ResourceBody): :ivar tuple contact: Contact information following ACME spec, `tuple` of `unicode`. :ivar unicode agreement: - :ivar unicode authorizations: URI where - `messages.Registration.Authorizations` can be found. - :ivar unicode certificates: URI where - `messages.Registration.Certificates` can be found. """ # on new-reg key server ignores 'key' and populates it based on @@ -248,26 +244,8 @@ class Registration(ResourceBody): key = jose.Field('key', omitempty=True, decoder=jose.JWK.from_json) contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) - authorizations = jose.Field('authorizations', omitempty=True) - certificates = jose.Field('certificates', omitempty=True) status = jose.Field('status', omitempty=True) - class Authorizations(jose.JSONObjectWithFields): - """Authorizations granted to Account in the process of registration. - - :ivar tuple authorizations: URIs to Authorization Resources. - - """ - authorizations = jose.Field('authorizations') - - class Certificates(jose.JSONObjectWithFields): - """Certificates granted to Account in the process of registration. - - :ivar tuple certificates: URIs to Certificate Resources. - - """ - certificates = jose.Field('certificates') - phone_prefix = 'tel:' email_prefix = 'mailto:' diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index e84c3e992..ab05a89b7 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -170,8 +170,7 @@ class RegistrationTest(unittest.TestCase): from acme.messages import Registration self.reg = Registration(key=key, contact=contact, agreement=agreement) - self.reg_none = Registration(authorizations='uri/authorizations', - certificates='uri/certificates') + self.reg_none = Registration() self.jobj_to = { 'contact': contact, From 4b91f7fbbc70cf9f824c0c41d8d0aaa23a378860 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 8 Apr 2017 00:58:19 +0800 Subject: [PATCH 106/128] Replace e.message with str(e) for Python 3 compatibility (#4416) --- certbot/display/ops.py | 7 +------ certbot/main.py | 4 ++-- certbot/renewal.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 85343fdc3..4bbf9e5b8 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -192,12 +192,7 @@ def _choose_names_manually(prompt_prefix=""): try: domain_list[i] = util.enforce_domain_sanity(domain) except errors.ConfigurationError as e: - try: # Python 2 - # pylint: disable=no-member - err_msg = e.message.encode('utf-8') - except AttributeError: - err_msg = str(e) - invalid_domains[domain] = err_msg + invalid_domains[domain] = str(e) if len(invalid_domains): retry_message = ( diff --git a/certbot/main.py b/certbot/main.py index 64e7d07a4..97b1bcd8a 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -571,7 +571,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config try: acme.revoke(jose.ComparableX509(cert), config.reason) except acme_errors.ClientError as e: - return e.message + return str(e) display_ops.success_revocation(config.cert_path[0]) @@ -583,7 +583,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals try: installer, authenticator = plug_sel.choose_configurator_plugins(config, plugins, "run") except errors.PluginSelectionError as e: - return e.message + return str(e) # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) diff --git a/certbot/renewal.py b/certbot/renewal.py index 6eb171763..2dd804671 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -79,7 +79,7 @@ def _reconstitute(config, full_path): except (ValueError, errors.Error) as error: logger.warning( "An error occurred while parsing %s. The error was %s. " - "Skipping the file.", full_path, error.message) + "Skipping the file.", full_path, str(error)) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None From a08e9599f5f70b059285b48d984b898a69a25092 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 7 Apr 2017 10:15:52 -0700 Subject: [PATCH 107/128] Review feedback. --- docs/contributing.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index d6b77facf..23cd59aa7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -39,6 +39,7 @@ Then in each shell where you're working on the client, do: .. code-block:: shell source ./venv/bin/activate + export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh After that, your shell will be using the virtual environment, and you run the @@ -116,6 +117,14 @@ and working. Fetch and start Boulder using: If you have problems with Docker, you may want to try `removing all containers and volumes`_ and making sure you have at least 1GB of memory. +Set up a certbot_test alias that enables easily running against the local +Boulder: + +.. code-block:: shell + + export SERVER=http://localhost:4000/directory + source tests/integration/_common.sh + Run the integration tests using: .. code-block:: shell From dfd4d0c10e90beaf733dfea7acacd7902b9a68df Mon Sep 17 00:00:00 2001 From: Alex Jordan Date: Fri, 7 Apr 2017 13:16:59 -0400 Subject: [PATCH 108/128] Document hook error handling (#4418) * Document hook error handling * Clarify wording around hook failure exit codes --- docs/using.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 549a3479c..a325ff413 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -378,10 +378,16 @@ then restart it after the plugin is finished. Example:: certbot renew --pre-hook "service nginx stop" --post-hook "service nginx start" -The hooks will only be -run if a certificate is due for renewal, so you can run this command -frequently without unnecessarily stopping your webserver. More -information about renewal hooks can be found by running +If a hook exits with a non-zero exit code, the error will be printed +to ``stderr`` but renewal will be attempted anyway. A failing hook +doesn't directly cause Certbot to exit with a non-zero exit code, but +since Certbot exits with a non-zero exit code when renewals fail, a +failed hook causing renewal failures will indirectly result in a +non-zero exit code. Hooks will only be run if a certificate is due for +renewal, so you can run the above command frequently without +unnecessarily stopping your webserver. + + More information about renewal hooks can be found by running ``certbot --help renew``. If you're sure that this command executes successfully without human From 28495b13361d0d7ffa1e8c914f7ceb9c47cf05e3 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 11 Apr 2017 16:57:42 -0700 Subject: [PATCH 109/128] fixed up the changelog --- CHANGELOG.md | 157 +++++++++++++++++++++++++-------------------------- 1 file changed, 78 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 572a9a6b4..69105fa14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,17 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added -* `--debug-challenges` pauses Certbot after setting up challenges for debugging. -* The Nginx parser can handle all valid directives in configuration files. -* Nginx ciphersuites changed to Mozilla Intermediate. -* `certbot-auto --no-bootstrap` won't install OS dependencies. +* `--debug-challenges` now pauses Certbot after setting up challenges for debugging. +* The Nginx parser can now handle all valid directives in configuration files. +* Nginx ciphersuites have changed to Mozilla Intermediate. +* `certbot-auto --no-bootstrap` provides the option to not install OS dependencies. ### Fixed -* `--register-unsafely-without-email` respects `--quiet`. -* Hyphenated renewalparams are now saved in renewal config files. +* `--register-unsafely-without-email` now respects `--quiet`. +* Hyphenated renewal parameters are now saved in renewal config files. * `--dry-run` no longer persists keys and csrs. -* No longer hangs when trying to start Nginx in Arch Linux. +* Certbot no longer hangs when trying to start Nginx in Arch Linux. * Apache rewrite rules no longer double-encode characters. A full list of changes is available on GitHub: @@ -26,12 +26,12 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.13.0%20is ### Added -* Allow non-camelcase Apache VirtualHost names -* Allow more log messages to be silenced +* Certbot now allows non-camelcase Apache VirtualHost names. +* Certbot now allows more log messages to be silenced. ### Fixed -* Fix a regression around using `--cert-name` when getting new certificates +* Fixed a regression around using `--cert-name` when getting new certificates More information about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.12.0 @@ -40,9 +40,9 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.12.0 ### Fixed -* Resolve a problem where Certbot would crash while parsing command line +* Resolved a problem where Certbot would crash while parsing command line arguments in some cases. -* Fix a typo. +* Fixed a typo. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Aclosed @@ -51,10 +51,9 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac ### Added -* The UI has been improved in the standalone plugin. When using the -plugin while running Certbot interactively and a required port is bound -by another process, Certbot will give you the option to retry to grab -the port rather than immediately exiting. +* When using the standalone plugin while running Certbot interactively +and a required port is bound by another process, Certbot will give you +the option to retry to grab the port rather than immediately exiting. * You are now able to deactivate your account with the Let's Encrypt server using the `unregister` subcommand. * When revoking a certificate using the `revoke` subcommand, you now @@ -67,7 +66,7 @@ to Let's Encrypt with `--reason`. ### Removed -* Removal of the optional `dnspython` dependency in our `acme` package. +* Removed the optional `dnspython` dependency in our `acme` package. Now the library does not support client side verification of the DNS challenge. @@ -78,17 +77,17 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.11.0+is%3Ac ### Added -* If Certbot receives a request with a `badNonce` error, we -automatically retry the request. Since nonces from Let's Encrypt expire, +* If Certbot receives a request with a `badNonce` error, it now +automatically retries the request. Since nonces from Let's Encrypt expire, this helps people performing the DNS challenge with the `manual` plugin who may have to wait an extended period of time for their DNS changes to propagate. ### Fixed -* We now save `--preferred-challenges` values for renewal. Previously -these values were discarded causing a different challenge type to be -used when renewing certs in some cases. +* Certbot now saves the `--preferred-challenges` values for renewal. Previously +these values were discarded causing a different challenge type to be used when +renewing certs in some cases. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed @@ -126,9 +125,9 @@ certificate with the `certonly` and `run` subcommands so a full list of domains in the certificate does not have to be provided. * Added subcommand `certificates` for listing the certificates managed by Certbot and their properties. -* Added `delete` subcommand for removing certificates managed by Certbot +* Added the `delete` subcommand for removing certificates managed by Certbot from the configuration directory. -* Support requesting internationalized domain names (IDNs). +* Certbot now supports requesting internationalized domain names (IDNs). * Hooks provided to Certbot are now saved to be reused during renewal. If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` flags when obtaining a certificate, the provided commands will @@ -171,7 +170,7 @@ as the tests were failing in some cases. ### Changed -* Adopt more conservative behavior about reporting a needed port as +* Certbot adopted more conservative behavior about reporting a needed port as unavailable when using the standalone plugin. More details about these changes can be found on our GitHub repo: @@ -181,17 +180,17 @@ https://github.com/certbot/certbot/milestone/27?closed=1 ### Added -* Stop requiring that all possibly required ports are available when -using the standalone plugin. Only verify the ports are available when -you know they are necessary. +* Certbot stoped requiring that all possibly required ports are available when +using the standalone plugin. It now only verifies that the ports are available +when they are necessary. ### Fixed -* Verify that our optional dependencies version matches what is +* Certbot now verifies that our optional dependencies version matches what is required by Certbot. -* Ensure we properly copy `ssl on;` directives as necessary when +* Certnot now properly copies the `ssl on;` directives as necessary when performing domain validation in the Nginx plugin. -* Fix problems where symlinks were becoming files when they were +* Fixed problem where symlinks were becoming files when they were packaged, causing errors during testing and OS packaging. More details about these changes can be found on our GitHub repo: @@ -201,7 +200,7 @@ https://github.com/certbot/certbot/milestone/26?closed=1 ### Fixed -* Fix a bug that was introduced in version 0.9.0 where the command +* Fixed a bug that was introduced in version 0.9.0 where the command line flag -q/--quiet wasn't respected in some cases. More details about these changes can be found on our GitHub repo: @@ -211,7 +210,7 @@ https://github.com/certbot/certbot/milestone/25?closed=1 ### Added -* Add an alpha version of the Nginx plugin. This plugin fully automates the +* Added an alpha version of the Nginx plugin. This plugin fully automates the process of obtaining and installing certificates with Nginx. Additionally, it is able to automatically configure security enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use @@ -220,12 +219,12 @@ is installed automatically when using `certbot-auto`) and provide `--nginx` on the command line. This plugin is still in its early stages so we recommend you use it with some caution and make sure you have a backup of your Nginx configuration. -* Support the `DNS` challenge in the `acme` library and `DNS` in +* Added support for the `DNS` challenge in the `acme` library and `DNS` in Certbot's `manual` plugin. This allows you to create DNS records to prove to Let's Encrypt you control the requested domain name. To use this feature, include `--manual --preferred-challenges dns` on the command line. -* Help with enabling Extra Packages for Enterprise Linux (EPEL) on +* Certbot now helps with enabling Extra Packages for Enterprise Linux (EPEL) on CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, the EPEL repository has to be enabled. `certbot-auto` will now prompt users asking them if they would like the script to enable this for them @@ -240,14 +239,14 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Acl ### Added -* Preserve a certificate's common name when using `renew` -* Save webroot values for renewal when they are entered interactively -* Gracefully report the Apache plugin isn't usable when Augeas is not installed -* Experimental support for Mageia has been added to `certbot-auto` +* Certbot now preserves a certificate's common name when using `renew`. +* Certbot now saves webroot values for renewal when they are entered interactively. +* Certbot now gracefully reports that the Apache plugin isn't usable when Augeas is not installed. +* Added experimental support for Mageia has been added to `certbot-auto`. ### Fixed -* Fix problems with an invalid user-agent string on OS X +* Fixed problems with an invalid user-agent string on OS X. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ @@ -256,9 +255,9 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ ### Added -* The main new feature in this release is the `register` subcommand which -can be used to register an account with the Let's Encrypt CA. -* Additionally, you can run `certbot register --update-registration` to +* Added the `register` subcommand which can be used to register an account +with the Let's Encrypt CA. +* You can now run `certbot register --update-registration` to change the e-mail address associated with your registration. More details about these changes can be found on our GitHub repo: @@ -269,14 +268,14 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ ### Added * Added `--must-staple` to request certificates from Let's Encrypt -with the OCSP must staple extension -* Automatically configure OSCP stapling for Apache -* Allow requesting certificates for domains found in the common name -of a custom CSR +with the OCSP must staple extension. +* Certbot now automatically configures OSCP stapling for Apache. +* Certbot now allows requesting certificates for domains found in the common name +of a custom CSR. ### Fixed -* Miscellaneous bug fixes +* Fixed a number of miscellaneous bugs More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue @@ -285,17 +284,17 @@ https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue ### Added -* Versioned the datetime dependency in setup.py +* Versioned the datetime dependency in setup.py. ### Changed -* Renamed the client from `letsencrypt` to `certbot` +* Renamed the client from `letsencrypt` to `certbot`. ### Fixed -* Fixed a small json deserialization error -* Preserve domain order in generated CSRs -* Some minor bug fixes +* Fixed a small json deserialization error. +* Certbot now preserves domain order in generated CSRs. +* Fixed some minor bugs. More details about these changes can be found on our GitHub repo: https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is%3Aclosed%20 @@ -304,21 +303,21 @@ https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is% ### Added -* Add the ability to use the webroot plugin interactively. -* The flags --pre-hook, --post-hook, and --renew-hook can be used with +* Added the ability to use the webroot plugin interactively. +* Added the flags --pre-hook, --post-hook, and --renew-hook which can be used with the renew subcommand to register shell commands to run in response to renewal events. Pre-hook commands will be run before any certs are renewed, post-hook commands will be run after any certs are renewed, and renew-hook commands will be run after each cert is renewed. If no certs are due for renewal, no command is run. -* A -q/--quiet flag which silences all output except errors. -* An --allow-subset-of-domains flag which can be used with the renew +* Added a -q/--quiet flag which silences all output except errors. +* Added an --allow-subset-of-domains flag which can be used with the renew command to prevent renewal failures for a subset of the requested domains from causing the client to exit. ### Changed -* Use cleaner renewal configuration files. In /etc/letsencrypt/renewal +* Certbot now uses renewal configuration files. In /etc/letsencrypt/renewal by default, these files can be used to control what parameters are used when renewing a specific certificate. @@ -329,10 +328,10 @@ https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue ### Fixed -* Resolves problems encountered when compiling letsencrypt +* Resolved problems encountered when compiling letsencrypt against the new OpenSSL release. -* A patch fixing problems of using `letsencrypt renew` with configuration files -from private beta has been added. +* Fixed problems encountered when using `letsencrypt renew` with configuration files +from the private beta. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 @@ -341,10 +340,10 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 ### Fixed -* Fix Apache parsing errors with some configurations -* Fix Werkzeug dependency problems on some Red Hat systems -* Fix bootstrapping failures when using letsencrypt-auto with --no-self-upgrade -* Fix problems with parsing renewal config files from private beta +* Fixed Apache parsing errors encountered with some configurations. +* Fixed Werkzeug dependency problems encountered on some Red Hat systems. +* Fixed bootstrapping failures when using letsencrypt-auto with --no-self-upgrade. +* Fixed problems with parsing renewal config files from private beta. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 @@ -353,23 +352,23 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 ### Added -* The new verb/subcommand `renew` can be used to renew your existing +* Added the verb/subcommand `renew` which can be used to renew your existing certificates as they approach expiration. Running `letsencrypt renew` will examine all existing certificate lineages and determine if any are less than 30 days from expiration. If so, the client will use the settings provided when you previously obtained the certificate to renew it. The subcommand finishes by printing a summary of which renewals were successful, failed, or not yet due. -* A `--dry-run` flag has been added to help with testing configuration +* Added a `--dry-run` flag to help with testing configuration without affecting production rate limits. Currently supported by the `renew` and `certonly` subcommands, providing `--dry-run` on the command line will obtain certificates from the staging server without saving the resulting certificates to disk. -* Major improvements have been added to letsencrypt-auto. This script +* Added major improvements to letsencrypt-auto. This script has been rewritten to include full support for Python 2.6, the ability for letsencrypt-auto to update itself, and improvements to the stability, security, and performance of the script. -* Support for Apache 2.2 has been added to the Apache plugin. +* Added support for Apache 2.2 to the Apache plugin. More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 @@ -378,10 +377,10 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 ### Added -* Add a non-interactive mode which can be enabled by including `-n` or +* Added a non-interactive mode which can be enabled by including `-n` or `--non-interactive` on the command line. This can be used to guarantee the client will not prompt when run automatically using cron/systemd. -* Preparation for the new letsencrypt-auto script. Over the past +* Added preparation for the new letsencrypt-auto script. Over the past couple months, we've been working on increasing the reliability and security of letsencrypt-auto. A number of changes landed in this release to prepare for the new version of this script. @@ -393,7 +392,7 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 ### Added -* Apache plugin support for non-Debian based systems. Support has been +* Added Apache plugin support for non-Debian based systems. Support has been added for modern Red Hat based systems such as Fedora 23, Red Hat 7, and CentOS 7 running Apache 2.4. In theory, this plugin should be able to be configured to run on any Unix-like OS running Apache 2.4. @@ -403,7 +402,7 @@ with PyOpenSSL versions 0.13 or 0.14. ### Fixed -* Resolves issues with the Apache plugin enabling an HTTP to HTTPS +* Resolved issues with the Apache plugin enabling an HTTP to HTTPS redirect on some systems. More details about these changes can be found on our GitHub repo: @@ -413,16 +412,16 @@ https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 ### Added -* Avoids attempting to issue for unqualified domain names like -"localhost" +* Added a check that avoids attempting to issue for unqualified domain names like +"localhost". ### Fixed -* Fix a confusing UI path that caused some users to repeatedly renew +* Fixed a confusing UI path that caused some users to repeatedly renew their certs while experimenting with the client, in some cases hitting -issuance rate limits -* Fix numerous Apache configuration parser problems -* Fix --webroot permission handling for non-root users +issuance rate limits. +* Fixed numerous Apache configuration parser problems +* Fixed --webroot permission handling for non-root users More details about these changes can be found on our GitHub repo: https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 From fbaf145c29d4a10c63e08b5d7eef577dba73f249 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 12 Apr 2017 15:48:58 -0700 Subject: [PATCH 110/128] "Renew" now ignores domains in cli.ini (#4479) * "Renew" now ignores domains in cli.ini * Document things slightly better * fix variable names * py3fix --- certbot/cli.py | 9 +++++++++ certbot/tests/cli_test.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/certbot/cli.py b/certbot/cli.py index cb769872e..e100c7715 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -515,6 +515,13 @@ class HelpfulArgumentParser(object): return usage + def remove_config_file_domains_for_renewal(self, parsed_args): + """Make "certbot renew" safe if domains are set in cli.ini.""" + # Works around https://github.com/certbot/certbot/issues/4096 + if self.verb == "renew": + for source, flags in self.parser._source_to_settings.items(): # pylint: disable=protected-access + if source.startswith("config_file") and "domains" in flags: + parsed_args.domains = _Default() if self.detect_defaults else [] def parse_args(self): """Parses command line arguments and returns the result. @@ -527,6 +534,8 @@ class HelpfulArgumentParser(object): parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb + self.remove_config_file_domains_for_renewal(parsed_args) + if self.detect_defaults: return parsed_args diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 498bd309d..0a5e959c2 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -39,6 +39,7 @@ class TestReadFile(TempDirTestCase): self.assertEqual(contents, test_contents) + class ParseTest(unittest.TestCase): '''Test the cli args entrypoint''' @@ -61,6 +62,22 @@ class ParseTest(unittest.TestCase): self.assertRaises(SystemExit, self.parse, args, output) return output.getvalue() + @mock.patch("certbot.cli.flag_default") + def test_cli_ini_domains(self, mock_flag_default): + tmp_config = tempfile.NamedTemporaryFile() + # use a shim to get ConfigArgParse to pick up tmp_config + shim = lambda v: constants.CLI_DEFAULTS[v] if v != "config_files" else [tmp_config.name] + mock_flag_default.side_effect = shim + + namespace = self.parse(["certonly"]) + self.assertEqual(namespace.domains, []) + tmp_config.write(b"domains = example.com") + tmp_config.flush() + namespace = self.parse(["certonly"]) + self.assertEqual(namespace.domains, ["example.com"]) + namespace = self.parse(["renew"]) + self.assertEqual(namespace.domains, []) + def test_no_args(self): namespace = self.parse([]) for d in ('config_dir', 'logs_dir', 'work_dir'): From 42d732d3c262a0e4387c83b4b1b73bd9aa094ef3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 12 Apr 2017 18:38:45 -0700 Subject: [PATCH 111/128] make sphinx happy --- certbot/storage.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/certbot/storage.py b/certbot/storage.py index a1462b72d..f0c9a2719 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -30,7 +30,14 @@ CURRENT_VERSION = util.get_strict_version(certbot.__version__) def renewal_conf_files(config): - """Return /path/to/*.conf in the renewal conf directory""" + """Build a list of all renewal configuration files. + + :param certbot.interfaces.IConfig config: Configuration object + + :returns: list of renewal configuration files + :rtype: `list` of `str` + + """ return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def renewal_file_for_certname(config, certname): From fef32627590dfcc1a86b24024c1154bcd9a9e7aa Mon Sep 17 00:00:00 2001 From: John Morrissey Date: Wed, 11 Jan 2017 20:42:09 -0500 Subject: [PATCH 112/128] add example of --renew-hook envvar values and hook script (#3502) --- certbot/cli.py | 6 ++++-- docs/cli-help.txt | 10 ++++++---- docs/using.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index e100c7715..fea83da29 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1064,9 +1064,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "renew", "--renew-hook", help="Command to be run in a shell once for each successfully renewed" " certificate. For this command, the shell variable $RENEWED_LINEAGE" - " will point to the config live subdirectory containing the new certs" + " will point to the config live subdirectory (for example," + " \"/etc/letsencrypt/live/example.com\") containing the new certs" " and keys; the shell variable $RENEWED_DOMAINS will contain a" - " space-delimited list of renewed cert domains") + " space-delimited list of renewed cert domains (for example," + " \"example.com www.example.com\"") helpful.add( "renew", "--disable-hook-validation", action='store_false', dest='validate_hooks', default=True, diff --git a/docs/cli-help.txt b/docs/cli-help.txt index a5f77a3a1..91041458e 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -265,10 +265,12 @@ renew: Command to be run in a shell once for each successfully renewed certificate. For this command, the shell variable $RENEWED_LINEAGE will point to the - config live subdirectory containing the new certs and - keys; the shell variable $RENEWED_DOMAINS will contain - a space-delimited list of renewed cert domains - (default: None) + config live subdirectory (for example, + "/etc/letsencrypt/live/example.com") containing the + new certs and keys; the shell variable + $RENEWED_DOMAINS will contain a space-delimited list + of renewed cert domains (for example, + "example.com www.example.com") (default: None) --disable-hook-validation Ordinarily the commands specified for --pre-hook /--post-hook/--renew-hook will be checked for diff --git a/docs/using.rst b/docs/using.rst index a325ff413..7eaa92f84 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -387,6 +387,49 @@ non-zero exit code. Hooks will only be run if a certificate is due for renewal, so you can run the above command frequently without unnecessarily stopping your webserver. +``--pre-hook`` and ``--post-hook`` hooks run before and after every renewal +attempt. If you want your hook to run only after a successful renewal, use +``--renew-hook`` in a command like this. + +``certbot renew --renew-hook /path/to/renew-hook-script`` + +For example, if you have a daemon that does not read its certificates as the +root user, a renew hook like this can copy them to the correct location and +apply appropriate file permissions. + +/path/to/renew-hook-script + +.. code-block:: none + + #!/bin/sh + + set -e + + for domain in $RENEWED_DOMAINS; do + case $domain in + example.com) + daemon_cert_root=/etc/some-daemon/certs + + # Make sure the certificate and private key files are + # never world readable, even just for an instant while + # we're copying them into daemon_cert_root. + umask 077 + + cp "$RENEWED_LINEAGE/fullchain.pem" "$daemon_cert_root/$domain.cert" + cp "$RENEWED_LINEAGE/privkey.pem" "$daemon_cert_root/$domain.key" + + # Apply the proper file ownership and permissions for + # the daemon to read its certificate and key. + chown some-daemon "$daemon_cert_root/$domain.cert" \ + "$daemon_cert_root/$domain.key" + chmod 400 "$daemon_cert_root/$domain.cert" \ + "$daemon_cert_root/$domain.key" + + service some-daemon restart >/dev/null + ;; + esac + done + More information about renewal hooks can be found by running ``certbot --help renew``. From 3381cc10eca33f9e6e566d8cfa4ebf2de30bb912 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 13 Apr 2017 11:57:14 -0700 Subject: [PATCH 113/128] fix 3664 (#4495) --- certbot/cli.py | 2 +- docs/cli-help.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index fea83da29..06b95b43b 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -904,7 +904,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "'run' subcommand this means reinstall the existing cert). (default: Ask)") helpful.add( "automation", "--expand", action="store_true", - help="If an existing cert covers some subset of the requested names, " + help="If an existing cert is a strict subset of the requested names, " "always expand and replace it with the additional names. (default: Ask)") helpful.add( "automation", "--version", action="version", diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 91041458e..3154bbda6 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -99,7 +99,7 @@ automation: keep the existing one until it is due for renewal (for the 'run' subcommand this means reinstall the existing cert). (default: Ask) - --expand If an existing cert covers some subset of the + --expand If an existing cert is a strict subset of the requested names, always expand and replace it with the additional names. (default: Ask) --version show program's version number and exit From bd8c31021ac640d4be0efc5d333da40126766d51 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 13 Apr 2017 13:42:01 -0700 Subject: [PATCH 114/128] Logging setup before argument parsing (#4446) Second part of #4443. Built on #4444. Fixes #3148. This fixes an old problem with code logging messages before logging has been set up. How this works is explained in the docstring of certbot.log.pre_arg_setup. * add memory handler * Add exit_with_log_path * add new_except_hook * pre_arg_parse_setup++ and remove old except_hook * Rewrite post_arg_setup * test restricted permissions * move changes to main * Use .name of NamedTemporaryFile * use better assertions * set exc_info in except_hook * Make post_arg_setup more robust * final cleanup * Add TempHandler * undo main_test changes * improve documentation * use decorators instead of with for mock.patch * add inline comment about logging.shutdown --- certbot/cli.py | 6 + certbot/log.py | 261 +++++++++++++++++++++++++-------- certbot/main.py | 25 +--- certbot/tests/log_test.py | 295 ++++++++++++++++++++++++++++++-------- 4 files changed, 448 insertions(+), 139 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 06b95b43b..c25153477 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -19,6 +19,7 @@ import certbot from certbot import constants from certbot import crypto_util from certbot import errors +from certbot import hooks from certbot import interfaces from certbot import util @@ -565,6 +566,11 @@ class HelpfulArgumentParser(object): if parsed_args.must_staple: parsed_args.staple = True + if parsed_args.validate_hooks: + hooks.validate_hooks(parsed_args) + + possible_deprecation_warning(parsed_args) + return parsed_args def set_test_server(self, parsed_args): diff --git a/certbot/log.py b/certbot/log.py index 92b35ed51..7660846a6 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -1,15 +1,29 @@ -"""Logging utilities for Certbot.""" +"""Logging utilities for Certbot. + +The best way to use this module is through `pre_arg_parse_setup` and +`post_arg_parse_setup`. `pre_arg_parse_setup` configures a minimal +terminal logger and ensures a detailed log is written to a secure +temporary file if Certbot exits before `post_arg_parse_setup` is called. +`post_arg_parse_setup` relies on the parsed command line arguments and +does the full logging setup with terminal and rotating file handling as +configured by the user. Any logged messages before +`post_arg_parse_setup` is called are sent to the rotating file handler. +Special care is taken by both methods to ensure all errors are logged +and properly flushed before program exit. + +""" from __future__ import print_function import functools import logging +import logging.handlers import os import sys +import tempfile import time import traceback from acme import messages -from certbot import cli from certbot import constants from certbot import errors from certbot import util @@ -23,36 +37,85 @@ logger = logging.getLogger(__name__) def pre_arg_parse_setup(): - """Ensures fatal exceptions are logged and reported to the user.""" - sys.excepthook = functools.partial(except_hook, config=None) + """Setup logging before command line arguments are parsed. + + Terminal logging is setup using + `certbot.constants.QUIET_LOGGING_LEVEL` so Certbot is as quiet as + possible. File logging is setup so that logging messages are + buffered in memory. If Certbot exits before `post_arg_parse_setup` + is called, these buffered messages are written to a temporary file. + If Certbot doesn't exit, `post_arg_parse_setup` writes the messages + to the normal log files. + + This function also sets `logging.shutdown` to be called on program + exit which automatically flushes logging handlers and + `sys.excepthook` to properly log/display fatal exceptions. + + """ + temp_handler = TempHandler() + temp_handler.setFormatter(logging.Formatter(FILE_FMT)) + temp_handler.setLevel(logging.DEBUG) + memory_handler = MemoryHandler(temp_handler) + + stream_handler = ColoredStreamHandler() + stream_handler.setFormatter(logging.Formatter(CLI_FMT)) + stream_handler.setLevel(constants.QUIET_LOGGING_LEVEL) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG) # send all records to handlers + root_logger.addHandler(memory_handler) + root_logger.addHandler(stream_handler) + + # logging.shutdown will flush the memory handler because flush() and + # close() are explicitly called + util.atexit_register(logging.shutdown) + sys.excepthook = functools.partial( + except_hook, debug='--debug' in sys.argv, log_path=temp_handler.path) def post_arg_parse_setup(config): """Setup logging after command line arguments are parsed. + This function assumes `pre_arg_parse_setup` was called earlier and + the root logging configuration has not been modified. A rotating + file logging handler is created and the buffered log messages are + sent to that handler. Terminal logging output is set to the level + requested by the user. + :param certbot.interface.IConfig config: Configuration object """ file_handler, file_path = setup_log_file_handler( config, 'letsencrypt.log', FILE_FMT) + logs_dir = os.path.dirname(file_path) + + root_logger = logging.getLogger() + memory_handler = stderr_handler = None + for handler in root_logger.handlers: + if isinstance(handler, ColoredStreamHandler): + stderr_handler = handler + elif isinstance(handler, MemoryHandler): + memory_handler = handler + msg = 'Previously configured logging handlers have been removed!' + assert memory_handler is not None and stderr_handler is not None, msg + + root_logger.addHandler(file_handler) + root_logger.removeHandler(memory_handler) + temp_handler = memory_handler.target + memory_handler.setTarget(file_handler) + memory_handler.close() + temp_handler.delete_and_close() if config.quiet: level = constants.QUIET_LOGGING_LEVEL else: level = -config.verbose_count * 10 - stderr_handler = ColoredStreamHandler() - stderr_handler.setFormatter(logging.Formatter(CLI_FMT)) stderr_handler.setLevel(level) - - root_logger = logging.getLogger() - root_logger.setLevel(logging.DEBUG) # send all records to handlers - root_logger.addHandler(stderr_handler) - root_logger.addHandler(file_handler) - logger.debug('Root logging level set at %d', level) logger.info('Saving debug log to %s', file_path) - sys.excepthook = functools.partial(except_hook, config=config) + sys.excepthook = functools.partial( + except_hook, debug=config.debug, log_path=logs_dir) def setup_log_file_handler(config, logfile, fmt): @@ -91,17 +154,17 @@ class ColoredStreamHandler(logging.StreamHandler): """Sends colored logging output to a stream. If the specified stream is not a tty, the class works like the - standard logging.StreamHandler. Default red_level is logging.WARNING. + standard `logging.StreamHandler`. Default red_level is + `logging.WARNING`. :ivar bool colored: True if output should be colored :ivar bool red_level: The level at which to output """ - def __init__(self, stream=None): - if sys.version_info < (2, 7): - # pragma: no cover - # pylint: disable=non-parent-init-called + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover logging.StreamHandler.__init__(self, stream) else: super(ColoredStreamHandler, self).__init__(stream) @@ -127,54 +190,132 @@ class ColoredStreamHandler(logging.StreamHandler): return out -def except_hook(exc_type, exc_value, trace, config): - """Logs exceptions and reports them to the user. +class MemoryHandler(logging.handlers.MemoryHandler): + """Buffers logging messages in memory until the buffer is flushed. - Config is used to determine how to display exceptions to the user. In - general, if config.debug is True, then the full exception and traceback is - shown to the user, otherwise it is suppressed. If config itself is None, - then the traceback and exception is attempted to be written to a logfile. - If this is successful, the traceback is suppressed, otherwise it is shown - to the user. sys.exit is always called with a nonzero status. + This differs from `logging.handlers.MemoryHandler` in that flushing + only happens when it is done explicitly by calling flush() or + close(). """ - tb_str = "".join(traceback.format_exception(exc_type, exc_value, trace)) - logger.debug("Exiting abnormally:%s%s", os.linesep, tb_str) + def __init__(self, target=None): + # capacity doesn't matter because should_flush() is overridden + capacity = float('inf') + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover + logging.handlers.MemoryHandler.__init__( + self, capacity, target=target) + else: + super(MemoryHandler, self).__init__(capacity, target=target) - if issubclass(exc_type, Exception) and (config is None or not config.debug): - if config is None: - logfile = "certbot.log" - try: - with open(logfile, "w") as logfd: - traceback.print_exception( - exc_type, exc_value, trace, file=logfd) - assert "--debug" not in sys.argv # config is None if this explodes - except: # pylint: disable=bare-except - sys.exit(tb_str) - if "--debug" in sys.argv: - sys.exit(tb_str) + def shouldFlush(self, record): + """Should the buffer be automatically flushed? + :param logging.LogRecord record: log record to be considered + + :returns: False because the buffer should never be auto-flushed + :rtype: bool + + """ + return False + + +class TempHandler(logging.StreamHandler): + """Safely logs messages to a temporary file. + + The file is created with permissions 600. + + :ivar str path: file system path to the temporary log file + + """ + def __init__(self): + stream = tempfile.NamedTemporaryFile('w', delete=False) + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover + logging.StreamHandler.__init__(self, stream) + else: + super(TempHandler, self).__init__(stream) + self.path = stream.name + + def delete_and_close(self): + """Close the handler and delete the temporary log file.""" + self._close(delete=True) + + def close(self): + """Close the handler and the temporary log file.""" + self._close(delete=False) + + def _close(self, delete): + """Close the handler and the temporary log file. + + :param bool delete: True if the log file should be deleted + + """ + self.acquire() + try: + # StreamHandler.close() doesn't close the stream to allow a + # stream like stderr to be used + self.stream.close() + if delete: + os.remove(self.path) + if sys.version_info < (2, 7): # pragma: no cover + logging.StreamHandler.close(self) + else: + super(TempHandler, self).close() + finally: + self.release() + + +def except_hook(exc_type, exc_value, trace, debug, log_path): + """Logs fatal exceptions and reports them to the user. + + If debug is True, the full exception and traceback is shown to the + user, otherwise, it is suppressed. sys.exit is always called with a + nonzero status. + + :param type exc_type: type of the raised exception + :param BaseException exc_value: raised exception + :param traceback trace: traceback of where the exception was raised + :param bool debug: True if the traceback should be shown to the user + :param str log_path: path to file or directory containing the log + + """ + exc_info = (exc_type, exc_value, trace) + # constants.QUIET_LOGGING_LEVEL or higher should be used to + # display message the user, otherwise, a lower level like + # logger.DEBUG should be used + if debug or not issubclass(exc_type, Exception): + assert constants.QUIET_LOGGING_LEVEL <= logging.ERROR + logger.error('Exiting abnormally:', exc_info=exc_info) + else: + logger.debug('Exiting abnormally:', exc_info=exc_info) if issubclass(exc_type, errors.Error): sys.exit(exc_value) + print('An unexpected error occurred:', file=sys.stderr) + if messages.is_acme_error(exc_value): + # Remove the ACME error prefix from the exception + _, _, exc_str = str(exc_value).partition(':: ') + print(exc_str, file=sys.stderr) else: - # Here we're passing a client or ACME error out to the client at the shell - # Tell the user a bit about what happened, without overwhelming - # them with a full traceback - err = traceback.format_exception_only(exc_type, exc_value)[0] - # Typical error from the ACME module: - # acme.messages.Error: urn:ietf:params:acme:error:malformed :: The - # request message was malformed :: Error creating new registration - # :: Validation of contact mailto:none@longrandomstring.biz failed: - # Server failure at resolver - if (messages.is_acme_error(err) and ":: " in err and - config.verbose_count <= cli.flag_default("verbose_count")): - # prune ACME error code, we have a human description - _code, _sep, err = err.partition(":: ") - msg = "An unexpected error occurred:\n" + err + "Please see the " - if config is None: - msg += "logfile '{0}' for more details.".format(logfile) - else: - msg += "logfiles in {0} for more details.".format(config.logs_dir) - sys.exit(msg) + traceback.print_exception(exc_type, exc_value, None) + exit_with_log_path(log_path) + + +def exit_with_log_path(log_path): + """Print a message about the log location and exit. + + The message is printed to stderr and the program will exit with a + nonzero status. + + :param str log_path: path to file or directory containing the log + + """ + msg = 'Please see the ' + if os.path.isdir(log_path): + msg += 'logfiles in {0} '.format(log_path) else: - sys.exit(tb_str) + msg += "logfile '{0}' ".format(log_path) + msg += 'for more details.' + sys.exit(msg) diff --git a/certbot/main.py b/certbot/main.py index 97b1bcd8a..023c09aee 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -714,37 +714,24 @@ def set_displayer(config): config.force_interactive) zope.component.provideUtility(displayer) -def _post_logging_setup(config, plugins, cli_args): - """Perform any setup or configuration tasks that require a logger.""" - - # This needs logging, but would otherwise be in HelpfulArgumentParser - if config.validate_hooks: - hooks.validate_hooks(config) - - cli.possible_deprecation_warning(config) - - logger.debug("certbot version: %s", certbot.__version__) - # do not log `config`, as it contains sensitive data (e.g. revoke --key)! - logger.debug("Arguments: %r", cli_args) - logger.debug("Discovered plugins: %r", plugins) - def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" log.pre_arg_parse_setup() + plugins = plugins_disco.PluginsRegistry.find_all() + logger.debug("certbot version: %s", certbot.__version__) + # do not log `config`, as it contains sensitive data (e.g. revoke --key)! + logger.debug("Arguments: %r", cli_args) + logger.debug("Discovered plugins: %r", plugins) # note: arg parser internally handles --help (and exits afterwards) args = cli.prepare_and_parse_args(plugins, cli_args) config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) - make_or_verify_needed_dirs(config) - log.post_arg_parse_setup(config) - - _post_logging_setup(config, plugins, cli_args) - + make_or_verify_needed_dirs(config) set_displayer(config) # Reporter diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 745147f0d..13021220b 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -1,6 +1,5 @@ """Tests for certbot.log.""" import logging -import traceback import logging.handlers import os import sys @@ -26,13 +25,34 @@ class PreArgParseSetupTest(unittest.TestCase): from certbot.log import pre_arg_parse_setup return pre_arg_parse_setup(*args, **kwargs) - def test_it(self): - with mock.patch('certbot.log.except_hook') as mock_except_hook: - with mock.patch('certbot.log.sys') as mock_sys: - self._call() + @mock.patch('certbot.log.sys') + @mock.patch('certbot.log.except_hook') + @mock.patch('certbot.log.logging.getLogger') + @mock.patch('certbot.log.util.atexit_register') + def test_it(self, mock_register, mock_get, mock_except_hook, mock_sys): + mock_sys.argv = ['--debug'] + mock_sys.version_info = sys.version_info + self._call() + mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) - mock_except_hook.assert_called_once_with(1, 2, 3, config=None) + mock_except_hook.assert_called_once_with( + 1, 2, 3, debug=True, log_path=mock.ANY) + + mock_root_logger = mock_get() + mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) + self.assertEqual(mock_root_logger.addHandler.call_count, 2) + + MemoryHandler = logging.handlers.MemoryHandler + memory_handler = None + for call in mock_root_logger.addHandler.call_args_list: + handler = call[0][0] + if memory_handler is None and isinstance(handler, MemoryHandler): + memory_handler = handler + else: + self.assertTrue(isinstance(handler, logging.StreamHandler)) + self.assertTrue( + isinstance(memory_handler.target, logging.StreamHandler)) class PostArgParseSetupTest(test_util.TempDirTestCase): @@ -46,9 +66,24 @@ class PostArgParseSetupTest(test_util.TempDirTestCase): def setUp(self): super(PostArgParseSetupTest, self).setUp() self.config = mock.MagicMock( - logs_dir=self.tempdir, quiet=False, + debug=False, logs_dir=self.tempdir, quiet=False, verbose_count=constants.CLI_DEFAULTS['verbose_count']) - self.root_logger = mock.MagicMock() + self.devnull = open(os.devnull, 'w') + + from certbot.log import ColoredStreamHandler + self.stream_handler = ColoredStreamHandler(six.StringIO()) + from certbot.log import MemoryHandler, TempHandler + self.temp_handler = TempHandler() + self.temp_path = self.temp_handler.path + self.memory_handler = MemoryHandler(self.temp_handler) + self.root_logger = mock.MagicMock( + handlers=[self.memory_handler, self.stream_handler]) + + def tearDown(self): + self.memory_handler.close() + self.stream_handler.close() + self.temp_handler.close() + super(PostArgParseSetupTest, self).tearDown() def test_common(self): with mock.patch('certbot.log.logging.getLogger') as mock_get_logger: @@ -58,19 +93,26 @@ class PostArgParseSetupTest(test_util.TempDirTestCase): mock_sys.version_info = sys.version_info self._call(self.config) - self.assertEqual(self.root_logger.addHandler.call_count, 2) + self.root_logger.removeHandler.assert_called_once_with( + self.memory_handler) + self.assertTrue(self.root_logger.addHandler.called) self.assertTrue(os.path.exists(os.path.join( self.config.logs_dir, 'letsencrypt.log'))) + self.assertFalse(os.path.exists(self.temp_path)) mock_sys.excepthook(1, 2, 3) - mock_except_hook.assert_called_once_with(1, 2, 3, config=self.config) + mock_except_hook.assert_called_once_with( + 1, 2, 3, debug=self.config.debug, log_path=self.tempdir) - stderr_handler = self.root_logger.addHandler.call_args_list[0][0][0] - level = stderr_handler.level + level = self.stream_handler.level if self.config.quiet: self.assertEqual(level, constants.QUIET_LOGGING_LEVEL) else: self.assertEqual(level, -self.config.verbose_count * 10) + def test_debug(self): + self.config.debug = True + self.test_common() + def test_quiet(self): self.config.quiet = True self.test_common() @@ -107,22 +149,25 @@ class SetupLogFileHandlerTest(test_util.TempDirTestCase): expected_path = os.path.join(self.config.logs_dir, log_file) self.assertEqual(log_path, expected_path) + handler.close() class ColoredStreamHandlerTest(unittest.TestCase): - """Tests for certbot.log.""" + """Tests for certbot.log.ColoredStreamHandler""" def setUp(self): - from certbot import log - self.stream = six.StringIO() self.stream.isatty = lambda: True - self.handler = log.ColoredStreamHandler(self.stream) - self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) + + from certbot.log import ColoredStreamHandler + self.handler = ColoredStreamHandler(self.stream) self.logger.addHandler(self.handler) + def tearDown(self): + self.handler.close() + def test_format(self): msg = 'I did a thing' self.logger.debug(msg) @@ -139,6 +184,75 @@ class ColoredStreamHandlerTest(unittest.TestCase): util.ANSI_SGR_RESET)) +class MemoryHandlerTest(unittest.TestCase): + """Tests for certbot.log.MemoryHandler""" + def setUp(self): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.DEBUG) + self.msg = 'hi there' + self.stream = six.StringIO() + + self.stream_handler = logging.StreamHandler(self.stream) + from certbot.log import MemoryHandler + self.handler = MemoryHandler(self.stream_handler) + self.logger.addHandler(self.handler) + + def tearDown(self): + self.handler.close() + self.stream_handler.close() + + def test_flush(self): + self._test_log_debug() + self.handler.flush() + self.assertEqual(self.stream.getvalue(), self.msg + '\n') + + def test_not_flushed(self): + # By default, logging.ERROR messages and higher are flushed + self.logger.critical(self.msg) + self.assertEqual(self.stream.getvalue(), '') + + def test_target_reset(self): + self._test_log_debug() + + new_stream = six.StringIO() + new_stream_handler = logging.StreamHandler(new_stream) + self.handler.setTarget(new_stream_handler) + self.handler.flush() + self.assertEqual(self.stream.getvalue(), '') + self.assertEqual(new_stream.getvalue(), self.msg + '\n') + new_stream_handler.close() + + def _test_log_debug(self): + self.logger.debug(self.msg) + + +class TempHandlerTest(unittest.TestCase): + """Tests for certbot.log.TempHandler.""" + def setUp(self): + self.closed = False + from certbot.log import TempHandler + self.handler = TempHandler() + + def tearDown(self): + if not self.closed: + self.handler.delete_and_close() + + def test_permissions(self): + self.assertTrue( + util.check_permissions(self.handler.path, 0o600, os.getuid())) + + def test_delete(self): + self.handler.delete_and_close() + self.closed = True + self.assertFalse(os.path.exists(self.handler.path)) + + def test_no_delete(self): + self.handler.close() + self.closed = True + self.assertTrue(os.path.exists(self.handler.path)) + os.remove(self.handler.path) + + class ExceptHookTest(unittest.TestCase): """Tests for certbot.log.except_hook.""" @classmethod @@ -146,53 +260,114 @@ class ExceptHookTest(unittest.TestCase): from certbot.log import except_hook return except_hook(*args, **kwargs) - @mock.patch('certbot.log.sys') - def test_except_hook(self, mock_sys): - config = mock.MagicMock() - mock_open = mock.mock_open() + def setUp(self): + self.error_msg = 'test error message' + self.log_path = 'foo.log' - with mock.patch('certbot.log.open', mock_open, create=True): - exception = Exception('detail') - config.verbose_count = 1 - self._call( - Exception, exc_value=exception, trace=None, config=None) - mock_open().write.assert_any_call(''.join( - traceback.format_exception_only(Exception, exception))) - error_msg = mock_sys.exit.call_args_list[0][0][0] - self.assertTrue('unexpected error' in error_msg) + def test_base_exception(self): + exc_type = KeyboardInterrupt + mock_logger, output = self._test_common(exc_type, debug=False) + self._assert_exception_logged(mock_logger.error, exc_type) + self._assert_logfile_output(output) - with mock.patch('certbot.log.open', mock_open, create=True): - mock_open.side_effect = [KeyboardInterrupt] - error = errors.Error('detail') - self._call( - errors.Error, exc_value=error, trace=None, config=None) - # assert_any_call used because sys.exit doesn't exit in cli.py - mock_sys.exit.assert_any_call(''.join( - traceback.format_exception_only(errors.Error, error))) + def test_debug(self): + exc_type = ValueError + mock_logger, output = self._test_common(exc_type, debug=True) + self._assert_exception_logged(mock_logger.error, exc_type) + self._assert_logfile_output(output) - bad_typ = messages.ERROR_PREFIX + 'triffid' - exception = messages.Error(detail='alpha', typ=bad_typ, title='beta') - config = mock.MagicMock(debug=False, verbose_count=-3) - self._call( - messages.Error, exc_value=exception, trace=None, config=config) - error_msg = mock_sys.exit.call_args_list[-1][0][0] - self.assertTrue('unexpected error' in error_msg) - self.assertTrue('acme:error' not in error_msg) - self.assertTrue('alpha' in error_msg) - self.assertTrue('beta' in error_msg) - config = mock.MagicMock(debug=False, verbose_count=1) - self._call( - messages.Error, exc_value=exception, trace=None, config=config) - error_msg = mock_sys.exit.call_args_list[-1][0][0] - self.assertTrue('unexpected error' in error_msg) - self.assertTrue('acme:error' in error_msg) - self.assertTrue('alpha' in error_msg) + def test_custom_error(self): + exc_type = errors.PluginError + mock_logger, output = self._test_common(exc_type, debug=False) + self._assert_exception_logged(mock_logger.debug, exc_type) + self._assert_quiet_output(mock_logger, output) - interrupt = KeyboardInterrupt('detail') - self._call( - KeyboardInterrupt, exc_value=interrupt, trace=None, config=None) - mock_sys.exit.assert_called_with(''.join( - traceback.format_exception_only(KeyboardInterrupt, interrupt))) + def test_acme_error(self): + # Get an arbitrary error code + acme_code = next(six.iterkeys(messages.ERROR_CODES)) + + def get_acme_error(msg): + """Wraps ACME errors so the constructor takes only a msg.""" + return messages.Error.with_code(acme_code, detail=msg) + + mock_logger, output = self._test_common(get_acme_error, debug=False) + self._assert_exception_logged(mock_logger.debug, messages.Error) + self._assert_quiet_output(mock_logger, output) + self.assertFalse(messages.ERROR_PREFIX in output) + + def test_other_error(self): + exc_type = ValueError + mock_logger, output = self._test_common(exc_type, debug=False) + self._assert_exception_logged(mock_logger.debug, exc_type) + self._assert_quiet_output(mock_logger, output) + + def _test_common(self, error_type, debug): + """Returns the mocked logger and stderr output.""" + mock_err = six.StringIO() + try: + raise error_type(self.error_msg) + except BaseException: + exc_info = sys.exc_info() + with mock.patch('certbot.log.logger') as mock_logger: + with mock.patch('certbot.log.sys.stderr', mock_err): + try: + # pylint: disable=star-args + self._call( + *exc_info, debug=debug, log_path=self.log_path) + except SystemExit as exit_err: + mock_err.write(str(exit_err)) + else: # pragma: no cover + self.fail('SystemExit not raised.') + + output = mock_err.getvalue() + return mock_logger, output + + def _assert_exception_logged(self, log_func, exc_type): + self.assertTrue(log_func.called) + call_kwargs = log_func.call_args[1] + self.assertTrue('exc_info' in call_kwargs) + + actual_exc_info = call_kwargs['exc_info'] + expected_exc_info = (exc_type, mock.ANY, mock.ANY) + self.assertEqual(actual_exc_info, expected_exc_info) + + def _assert_logfile_output(self, output): + self.assertTrue('Please see the logfile' in output) + self.assertTrue(self.log_path in output) + + def _assert_quiet_output(self, mock_logger, output): + self.assertFalse(mock_logger.exception.called) + self.assertTrue(mock_logger.debug.called) + self.assertTrue(self.error_msg in output) + + +class ExitWithLogPathTest(test_util.TempDirTestCase): + """Tests for certbot.log.exit_with_log_path.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.log import exit_with_log_path + return exit_with_log_path(*args, **kwargs) + + def test_log_file(self): + log_file = os.path.join(self.tempdir, 'test.log') + open(log_file, 'w').close() + + err_str = self._test_common(log_file) + self.assertTrue('logfiles' not in err_str) + self.assertTrue(log_file in err_str) + + def test_log_dir(self): + err_str = self._test_common(self.tempdir) + self.assertTrue('logfiles' in err_str) + self.assertTrue(self.tempdir in err_str) + + def _test_common(self, *args, **kwargs): + try: + self._call(*args, **kwargs) + except SystemExit as err: + return str(err) + else: # pragma: no cover + self.fail('SystemExit was not raised.') if __name__ == "__main__": From d11443ac050ab31ca547214b05cd8fcbac0e1903 Mon Sep 17 00:00:00 2001 From: St-Ranger Date: Thu, 13 Apr 2017 20:36:32 -0500 Subject: [PATCH 115/128] Missing word is inserted. --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f3879f53f..6237ad3ba 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -427,7 +427,7 @@ For squeeze you will need to: FreeBSD ------- -Packages can be installed FreeBSD using ``pkg``, +Packages can be installed on FreeBSD using ``pkg``, or any other port-management tool (``portupgrade``, ``portmanager``, etc.) from the pre-built package or can be built and installed from ports. Either way will ensure proper installation of all the dependencies required From f54280d9b98a59c33dabb2499f687b63a93043f4 Mon Sep 17 00:00:00 2001 From: Carl Michael Skog Date: Fri, 14 Apr 2017 04:10:12 +0200 Subject: [PATCH 116/128] Use binary flag when writing cert and key files (#4462) * Use binary flag when writing cert and key files Add binary flag to mode argument when opening files for writing key and certificate files. On Python 3 the data buffers use for writing are bytes objects not strings, and the write fails accordingly. As far as I understand, it the "b" flag will not hurt things in Python 2 either. * Update the tests for RenewableCert::save_successor Update the tests for RenewableCert::save_successor after changing three parameters to be called with bytes objects instead of strings. Also, update the doc comment of the function. --- certbot/storage.py | 14 +++++++------- certbot/tests/storage_test.py | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/certbot/storage.py b/certbot/storage.py index f0c9a2719..c35a268e3 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -1047,10 +1047,10 @@ class RenewableCert(object): is regarded as a successor (used to choose a privkey, if the key has not changed, but otherwise this information is not permanently recorded anywhere) - :param str new_cert: the new certificate, in PEM format - :param str new_privkey: the new private key, in PEM format, + :param bytes new_cert: the new certificate, in PEM format + :param bytes new_privkey: the new private key, in PEM format, or ``None``, if the private key has not changed - :param str new_chain: the new chain, in PEM format + :param bytes new_chain: the new chain, in PEM format :param .NamespaceConfig cli_config: parsed command line arguments @@ -1086,18 +1086,18 @@ class RenewableCert(object): logger.debug("Writing symlink to old private key, %s.", old_privkey) os.symlink(old_privkey, target["privkey"]) else: - with open(target["privkey"], "w") as f: + with open(target["privkey"], "wb") as f: logger.debug("Writing new private key to %s.", target["privkey"]) f.write(new_privkey) # Save everything else - with open(target["cert"], "w") as f: + with open(target["cert"], "wb") as f: logger.debug("Writing certificate to %s.", target["cert"]) f.write(new_cert) - with open(target["chain"], "w") as f: + with open(target["chain"], "wb") as f: logger.debug("Writing chain to %s.", target["chain"]) f.write(new_chain) - with open(target["fullchain"], "w") as f: + with open(target["fullchain"], "wb") as f: logger.debug("Writing full chain to %s.", target["fullchain"]) f.write(new_cert + new_chain) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6635461aa..d8fe98536 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -489,8 +489,8 @@ class RenewableCertTests(BaseRenewableCertTest): self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) self.assertEqual( - 6, self.test_rc.save_successor(3, "new cert", None, - "new chain", self.cli_config)) + 6, self.test_rc.save_successor(3, b'new cert', None, + b'new chain', self.cli_config)) with open(self.test_rc.version("cert", 6)) as f: self.assertEqual(f.read(), "new cert") with open(self.test_rc.version("chain", 6)) as f: @@ -502,11 +502,11 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(os.path.islink(self.test_rc.version("privkey", 6))) # Let's try two more updates self.assertEqual( - 7, self.test_rc.save_successor(6, "again", None, - "newer chain", self.cli_config)) + 7, self.test_rc.save_successor(6, b'again', None, + b'newer chain', self.cli_config)) self.assertEqual( - 8, self.test_rc.save_successor(7, "hello", None, - "other chain", self.cli_config)) + 8, self.test_rc.save_successor(7, b'hello', None, + b'other chain', self.cli_config)) # All of the subsequent versions should link directly to the original # privkey. for i in (6, 7, 8): @@ -520,8 +520,8 @@ class RenewableCertTests(BaseRenewableCertTest): # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) self.assertEqual( - 9, self.test_rc.save_successor(8, "last", None, - "attempt", self.cli_config)) + 9, self.test_rc.save_successor(8, b'last', None, + b'attempt', self.cli_config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), list(six.moves.range(1, 10))) @@ -535,8 +535,8 @@ class RenewableCertTests(BaseRenewableCertTest): # Test updating when providing a new privkey. The key should # be saved in a new file rather than creating a new symlink. self.assertEqual( - 10, self.test_rc.save_successor(9, "with", "a", - "key", self.cli_config)) + 10, self.test_rc.save_successor(9, b'with', b'a', + b'key', self.cli_config)) self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.exists(temp_config_file)) From 29d25f0915dad93cd7f49eba89557a88c730eb23 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 16 Mar 2017 17:52:58 +0800 Subject: [PATCH 117/128] Enable boulder tests on Python 3 --- .travis.yml | 28 ++++++++++++++++++++++++---- tests/boulder-integration.sh | 4 ++-- tests/manual-http-auth.sh | 4 +++- tests/run_http_server.py | 11 +++++++++++ 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 tests/run_http_server.py diff --git a/.travis.yml b/.travis.yml index 577dcbc40..22bde836e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,13 +78,33 @@ matrix: env: TOXENV=apacheconftest sudo: required - python: "3.3" - env: TOXENV=py33 + env: TOXENV=py33 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.4" - env: TOXENV=py34 + env: TOXENV=py34 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.5" - env: TOXENV=py35 + env: TOXENV=py35 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.6" - env: TOXENV=py36 + env: TOXENV=py36 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "2.7" env: TOXENV=nginxroundtrip diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 6612b2e67..08c482676 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -82,7 +82,7 @@ CheckHooks() { # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python -m SimpleHTTPServer $http_01_port & +python ./tests/run_http_server.py $http_01_port & python_server_pid=$! common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ @@ -90,7 +90,7 @@ common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --renew-hook 'echo renew >> "$HOOK_TEST"' kill $python_server_pid -python -m SimpleHTTPServer $tls_sni_01_port & +python ./tests/run_http_server.py $tls_sni_01_port & python_server_pid=$! common --domains le2.wtf --preferred-challenges http-01 run \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ diff --git a/tests/manual-http-auth.sh b/tests/manual-http-auth.sh index c4730392b..48c33f04b 100755 --- a/tests/manual-http-auth.sh +++ b/tests/manual-http-auth.sh @@ -1,10 +1,12 @@ #!/bin/sh uri_path=".well-known/acme-challenge/$CERTBOT_TOKEN" +# This script should be run from the top level. e.g. ./tests/manual-http-auth.sh +source_dir="$(pwd)" cd $(mktemp -d) mkdir -p $(dirname $uri_path) echo $CERTBOT_VALIDATION > $uri_path -python -m SimpleHTTPServer $http_01_port >/dev/null 2>&1 & +python "$source_dir/tests/run_http_server.py" $http_01_port >/dev/null 2>&1 & server_pid=$! while ! curl "http://localhost:$http_01_port/$uri_path" >/dev/null 2>&1; do sleep 1s diff --git a/tests/run_http_server.py b/tests/run_http_server.py new file mode 100644 index 000000000..fd1163816 --- /dev/null +++ b/tests/run_http_server.py @@ -0,0 +1,11 @@ +import runpy +import sys + +# Run Python's built-in HTTP server +# Usage: python ./tests/run_http_server.py port_num +# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ + +# sys.argv (port number) is passed as-is to the HTTP server module +runpy.run_module( + 'http.server' if sys.version_info[0] == 3 else 'SimpleHTTPServer', + run_name='__main__') From 031a8dd837c49fa848e2f1caac82b8edfb9c7eb2 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 16 Mar 2017 21:57:50 +0800 Subject: [PATCH 118/128] Fix nginx integration tests on Python 3 --- certbot-nginx/certbot_nginx/configurator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index b36f638ab..afa701a75 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -629,7 +629,8 @@ class NginxConfigurator(common.Plugin): proc = subprocess.Popen( [self.conf('ctl'), "-c", self.nginx_conf, "-V"], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) text = proc.communicate()[1] # nginx prints output to stderr except (OSError, ValueError) as error: logger.debug(error, exc_info=True) From 5b6c3c2c297505ba4ad87b3f8ecb29578c75a2a5 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 14 Apr 2017 16:01:55 -0700 Subject: [PATCH 119/128] Attempt to document challenge types --- docs/challenges.rst | 156 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 docs/challenges.rst diff --git a/docs/challenges.rst b/docs/challenges.rst new file mode 100644 index 000000000..0c923c45b --- /dev/null +++ b/docs/challenges.rst @@ -0,0 +1,156 @@ +Digital certificates can only be issued to people who are entitled to them. For example, assuming you don't run google.com, you're not entitled to a certificate for it. Nor is someone else entitled to receive a certificate for your web site. + +In order to receive a certificate from Let's Encrypt certificate authority (CA), you have to prove your control over each of the domain names that will be listed in the certificate. You can do so by making certain publicly-visible changes, proving that the person who's requested a particular certificate is the same person who controls the site(s) that the certificate will refer to. + +Let’s Encrypt specifies three different ways to prove your control over a domain (each of which Certbot may be able to do for you). These are called "challenges," because you are being challenged to perform tasks that only someone who controls the domain should be able to accomplish. + +When you use Certbot, it will attempt to help you prove control over your domains automatically in a way that's acceptable to the CA. Especially if this doesn't work the way you expected, it can be helpful to understand what Certbot is trying to do in each case. + +The three ways to prove your control over a domain for the Let’s Encrypt CA are: + +* Posting a specified file on a web site + +This method is called the HTTP-01 challenge. In this challenge, the certificate authority will expect a specified file to be posted in a specified location on a web site. The file will be downloaded using an HTTP request on TCP port 80. Since part of what this challenge shows is the ability to create a file at an arbitrary location, you cannot choose a different location or port number. + +* Offering a specified certificate on a web site + +This method is called the TLS-SNI-01 challenge. In this challenge, the certificate authority will expect a specified digital certificate to be provided by the web server in response to an HTTPS request using a particular made-up domain name. The request will be made using HTTPS on TCP port 443. You cannot choose a different port number. + +This certificate is a self-signed certificate created by Certbot. You use it only temporarily to prove your control over a domain name. It’s not the same as the certificate for your site that will later be issued by Let's Encrypt once you've proven that you control the site. + +* Posting a specified DNS record in the domain name system + +This method is called the DNS-01 challenge. In this challenge, the certificate authority will expect a specified DNS record to be present in your DNS zone when queried for. The record will be a TXT record for a specific subdomain of the name you're proving your control over. + +For each kind of challenge, the challenge can potentially be completed *automatically* (Certbot directly makes the necessary changes itself, or runs another program that does so), or *manually* (Certbot tells you to make a certain change, and you edit a configuration file of some kind in order to accomplish it). Certbot's design emphasizes performing challenges *automatically*, and this is the normal case for most uses of Certbot. + +Some Certbot *plugins* offer the functionality of an *authenticator*, which simply means that they can satisfy challenges. Different plugins can satisfy different kinds of challenges, as follows: + +apache plugin: Can only use TLS-SNI-01. Tries to edit your Apache configuration files in order to temporarily serve a specified Certbot-generated certificate for a specified name. This can work when you're running Certbot on a web server with an existing installation of Apache that is able to listen on port 443. This makes certain assumptions about your Apache configuration. + +nginx plugin: Can only use TLS-SNI-01. Tries to edit your nginx configuration files in order to temporarily serve a specified Certbot-generated certificate for a specified name. This can work when you're running Certbot on a web server with an existing installation of nginx that is able to listen on port 443. This makes certain assumptions about your nginx configuration. + +webroot plugin: Can only use HTTP-01. Tries to place a file into an appropriate place in order for that file to be served over HTTP on port 80 by an existing web server running on your system. This can work when you're running Certbot on a web server with any existing server application that already listens to web requests on port 80, and that serves files from disk in response. + +standalone plugin: Can use either TLS-SNI-01 or HTTP-01. (You can choose with the `--preferred-challenges` option.) Tries to run its own temporary web server which will speak either HTTP on port 80 (for HTTP-01) or HTTPS on port 443 (for TLS-SNI-01). This can work if either of these ports is free to receive incoming connections at the moment that you run Certbot, because there's no existing program listening to them or because you've temporarily shut down any server application that was listening to them. + +manual plugin: Can use either DNS-01 or HTTP-01. May tell you what changes you are expected to make to your configuration. Or, using an external script, can update your DNS records (for DNS-01) or your webroot (for HTTP-01). This can work if you have appropriate technical knowledge of how to make these kinds of changes yourself when asked to do so. Note that this will prevent automated renewal of your certificate using `certbot renew`. [Can manual also use TLS-SNI-01??] + + +Common problems with passing different challenges + +HTTP-01 challenge: +* (With webroot plugin) You aren't running Certbot on your web server + + Most people should install and run Certbot on their web server hosting their website, not on their laptops or some other computer. While you can use Certbot in manual mode on a laptop and then separately set up the appropriate files on your webserver, it's not likely to be the most convenient way to get a certificate for most users. + +* A domain name you're requesting a certificate for isn't correctly pointed at that web server + + In most cases, every name you're requesting a certificate for should already exist and be pointed to the public IP address of the server where you're requesting that certificate. (Some alternatives exist for complex network configurations, but they're the exception rather than the rule.) + +* A firewall is blocking access to port 80 + + The certificate authority needs to be able to connect to port 80 of your server in order to confirm that you satisfied the HTTP-01 challenge. So that needs to be publicly reachable from the Internet, and not blocked by a router or firewall. + +* (With webroot plugin) You specified the webroot directory incorrectly + + If you used `--webroot`, you need to tell Certbot where it can put + files in order to have them served by your existing web server. + If you said your webroot for example.com was /var/www/example.com, + then a file placed in /var/www/example.com/.well-known/acme-challenge/testfile should appear on + your web site at http://example.com/.well-known/acme-challenge/testfile (which you can test using a web browser). (A redirection to HTTPS + is OK here and should not stop the challenge from working.) + + Note that you should *not* specify the .well-known/acme-challenge directory itself. Instead, you should specify the top level directory that web content is served from. + +* (With webroot plugin) You don't have a webroot directory at all + + In some web server configurations, all pages are dynamically generated by some kind of framework, usually using a database backend. In this case, there might not be a particular directory that files can be directly served from by the existing web server application. Using the webroot plugin in this case requires making a change to your web server configuration first. + +* (With manual plugin) You updated the webroot directory incorrectly + + If you used `--manual`, you need to know where you can put files in order to have them served by your existing web server. If you think your webroot for example.com is /var/www/example.com, then a file placed in /var/www/example.com/.well-known/acme-challenge/testfile should appear on + your web site at http://example.com/.well-known/acme-challenge/testfile. (A redirection to HTTPS + is OK here and should not stop the challenge from working.) You should also make sure that you don't make a typo in the name of the file when creating it. + +* Your existing web server's configuration refuses to serve files + from /.well-known/acme-challenge, or doesn't serve them at the + /.well-known/acme-challenge location on your site, or serves them + with a header or footer, or serves them with an unusual MIME type. + +* (With standalone plugin) + You tried to use `--standalone` when there was already some other + program on your server listening to port 80 + +* (With webroot plugin) + You tried to use `--webroot` when you don't have an existing web + server listening on port 80 + +* Your DNS records aren't valid + Try checking your DNS records with a tool like the DNSchecker at + http://www.dnsstuff.com/ to make sure there are no serious errors. + Sometimes a DNS error still allows your site to load in a web + browser, but prevents the certificate authority from issuing a + certificate. + +TLS-SNI-01 challenge: +* You aren't running Certbot on your web server + + Most people should install and run Certbot on their web server hosting their website, not on their laptops or some other computer. While you can use Certbot in manual mode on a laptop and then separately set up the appropriate files on your webserver, it's not likely to be the most convenient way to get a certificate for most users. + +* A domain name you're requesting a certificate for isn't correctly + pointed at that web server + + In most cases, every name you're requesting a certificate for should + already exist and be pointed to the server where you're requesting + that certificate. (Some alternatives exist for complex network + configurations, but they're the exception rather than the rule.) + +* You're using a content delivery network (CDN) + + TLS-SNI-01 doesn't work with CDNs (like CloudFlare and Akamai). You + have to use a different challenge type. (This is a special case of + the previous problem: the domain name is pointed at the CDN, not + directly at your server.) + +* A firewall is blocking access to port 443 + + The certificate authority needs to be able to connect to port 443 of + your server in order to confirm that you satisfied the TLS-SNI-01 + challenge. So that needs to be publicly reachable from the Internet, + and not blocked by a router or firewall. + +* (With apache plugin) + Certbot thinks you're running Apache, but you aren't running it, or + you're running a different server of some kind on port 443 + +* (With nginx plugin) + Certbot thinks you're running nginx, but you aren't running it, or + you're running a different server of some kind on port 443 + +* (With apache or nginx plugin) + Certbot doesn't know how to modify your web server configuration correctly + +* (With standalone plugin) + You tried to use `--standalone` when there was already some other + program on your server listening to port 443 + +* Your DNS records aren't valid + Try checking your DNS records with a tool like the DNSchecker at + http://www.dnsstuff.com/ to make sure there are no serious errors. + Sometimes a DNS error still allows your site to load in a web + browser, but prevents the certificate authority from issuing a + certificate. + +DNS-01 challenge: + +* (With manual plugin) Your DNS records weren't correctly updated. + You need to be able to make appropriate changes to your DNS zone + in order to pass the challenge. + +* Your DNS records aren't valid. + Try checking your DNS records with a tool like the DNSchecker at + http://www.dnsstuff.com/ to make sure there are no serious errors. + Sometimes a DNS error still allows your site to load in a web + browser, but prevents the certificate authority from issuing a + certificate. From 1ed50497eba1203b952b8709f3757fb9285db6dc Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Mon, 17 Apr 2017 11:41:24 -0700 Subject: [PATCH 120/128] [certbot-auto]: "renew" implies --non-interactive (#4500) --- letsencrypt-auto-source/letsencrypt-auto | 2 +- letsencrypt-auto-source/letsencrypt-auto.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 84235cb68..8bc4bb112 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -59,7 +59,7 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 4e07bec02..37f84a715 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -59,7 +59,7 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; From 4c0e82f426464bec254fd1b9086bb54716a7dfbb Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Mon, 17 Apr 2017 13:12:46 -0700 Subject: [PATCH 121/128] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69105fa14..a8ee45024 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,7 +180,7 @@ https://github.com/certbot/certbot/milestone/27?closed=1 ### Added -* Certbot stoped requiring that all possibly required ports are available when +* Certbot stopped requiring that all possibly required ports are available when using the standalone plugin. It now only verifies that the ports are available when they are necessary. From 36891033dd13737943ab548d8138be125b6e4a57 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 17 Apr 2017 14:39:08 -0700 Subject: [PATCH 122/128] Set HOME if it's not defined in certbot-auto (#4503) * set HOME if it's not defined * use ~root rather than /root --- letsencrypt-auto-source/letsencrypt-auto | 5 +++++ letsencrypt-auto-source/letsencrypt-auto.template | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8bc4bb112..41cbf2ec0 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 37f84a715..6cc110f2c 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi From b0600483fd25e0c054713c324bfeddf106e7e98b Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Tue, 18 Apr 2017 18:18:56 +0000 Subject: [PATCH 123/128] interface: IConfig is missing fields used in client.py (#4453) Signed-off-by: Arthur Gautier --- certbot/interfaces.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 213992993..b75c2a44f 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -235,6 +235,22 @@ class IConfig(zope.interface.Interface): "This only affects the port Certbot listens on. " "A conforming ACME server will still attempt to connect on port 80.") + pref_challs = zope.interface.Attribute( + "Sorted user specified preferred challenges" + "type strings with the most preferred challenge listed first") + + allow_subset_of_names = zope.interface.Attribute( + "When performing domain validation, do not consider it a failure " + "if authorizations can not be obtained for a strict subset of " + "the requested domains. This may be useful for allowing renewals for " + "multiple domains to succeed even if some domains no longer point " + "at this system. This is a boolean") + + strict_permissions = zope.interface.Attribute( + "Require that all configuration files are owned by the current " + "user; only needed if your config is somewhere unsafe like /tmp/." + "This is a boolean") + class IInstaller(IPlugin): """Generic Certbot Installer Interface. From d54cb3c59d709c018e95ba01c7a3877e76652e7d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 18 Apr 2017 17:09:22 -0700 Subject: [PATCH 124/128] Improve Account.repr() (#4325) * Include more detail in Account's repr. In particular, regr and meta. * Fix test. * Review feedback. * Lint * Test prefix only. --- certbot/account.py | 3 ++- certbot/tests/account_test.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/certbot/account.py b/certbot/account.py index 1928b90d8..389f96791 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -74,7 +74,8 @@ class Account(object): # pylint: disable=too-few-public-methods self.meta.creation_dt), self.meta.creation_host, self.id[:4]) def __repr__(self): - return "<{0}({1})>".format(self.__class__.__name__, self.id) + return "<{0}({1}, {2}, {3})>".format( + self.__class__.__name__, self.regr, self.id, self.meta) def __eq__(self, other): return (isinstance(other, self.__class__) and diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 11d14132d..5950dcda9 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -33,6 +33,7 @@ class AccountTest(unittest.TestCase): creation_dt=datetime.datetime( 2015, 7, 4, 14, 4, 10, tzinfo=pytz.UTC)) self.acc = Account(self.regr, KEY, self.meta) + self.regr.__repr__ = mock.MagicMock(return_value="i_am_a_regr") with mock.patch("certbot.account.socket") as mock_socket: mock_socket.getfqdn.return_value = "test.certbot.org" @@ -54,10 +55,8 @@ class AccountTest(unittest.TestCase): self.acc.slug, "test.certbot.org@2015-07-04T14:04:10Z (bca5)") def test_repr(self): - self.assertEqual( - repr(self.acc), - "") - + self.assertTrue(repr(self.acc).startswith( + " Date: Wed, 19 Apr 2017 11:11:38 -0500 Subject: [PATCH 125/128] Make it easier to honor --quiet (#4292) * say -- echo which honors quiet * error -- echo which does not honor quiet * switch non error echos to say * switch error echos to error * run letsencrypt-auto-source/build.py --- letsencrypt-auto-source/letsencrypt-auto | 150 +++++++++--------- .../letsencrypt-auto.template | 114 ++++++------- .../pieces/bootstrappers/deb_common.sh | 6 +- .../pieces/bootstrappers/mac.sh | 18 +-- .../pieces/bootstrappers/mageia_common.sh | 4 +- .../pieces/bootstrappers/rpm_common.sh | 8 +- 6 files changed, 158 insertions(+), 142 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 41cbf2ec0..db01277dc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -98,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -105,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -150,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -170,7 +180,7 @@ fi BootstrapMessage() { # Arguments: Platform name - echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" } ExperimentalBootstrap() { @@ -181,11 +191,11 @@ ExperimentalBootstrap() { $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - echo "Alternatively, you can install OS dependencies yourself and run this script" - echo "again with --no-bootstrap." + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -196,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -232,7 +242,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -260,7 +270,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -320,7 +330,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } @@ -340,7 +350,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -354,7 +364,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -366,7 +376,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -408,7 +418,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } @@ -513,15 +523,15 @@ BootstrapFreeBsd() { BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -532,26 +542,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } @@ -571,7 +581,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -583,7 +593,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } @@ -610,11 +620,11 @@ Bootstrap() { BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -630,11 +640,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -654,7 +664,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -662,7 +672,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -671,7 +681,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -1027,42 +1037,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -1089,7 +1097,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -1232,9 +1240,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -1243,7 +1251,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 6cc110f2c..566f79307 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -98,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -105,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -150,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -170,7 +180,7 @@ fi BootstrapMessage() { # Arguments: Platform name - echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" } ExperimentalBootstrap() { @@ -181,11 +191,11 @@ ExperimentalBootstrap() { $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - echo "Alternatively, you can install OS dependencies yourself and run this script" - echo "again with --no-bootstrap." + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -196,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -240,11 +250,11 @@ Bootstrap() { BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -260,11 +270,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -284,7 +294,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -292,7 +302,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -301,7 +311,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -326,42 +336,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -388,7 +396,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -402,9 +410,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -413,7 +421,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index 7735933c4..afd279ac2 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -21,7 +21,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -49,7 +49,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -109,7 +109,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh index c101be7d7..b88e96999 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh @@ -1,14 +1,14 @@ BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -19,26 +19,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh index dd1213a4c..1c76bbcac 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh @@ -8,7 +8,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -20,7 +20,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 44c4625d9..dcd535292 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -13,7 +13,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -27,7 +27,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -39,7 +39,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -81,7 +81,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } From 372d2011111a3ca727de601e0417a30fb332b94e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 19 Apr 2017 14:11:18 -0700 Subject: [PATCH 126/128] use say now that if QUIET statement was moved (#4530) --- letsencrypt-auto-source/letsencrypt-auto | 4 ++-- letsencrypt-auto-source/letsencrypt-auto.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index db01277dc..50c80f0a4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1069,8 +1069,8 @@ UNLIKELY_EOF fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 566f79307..f6585a378 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -368,8 +368,8 @@ UNLIKELY_EOF fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop From 779af8db1ec03de1925217da1bedcab072092286 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 21 Apr 2017 00:09:20 +0800 Subject: [PATCH 127/128] Extend tools/venv3.sh to support full certbot development (#4532) --- tools/venv3.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/venv3.sh b/tools/venv3.sh index 35ffac749..08358aa48 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -1,8 +1,18 @@ #!/bin/sh -xe # Developer Python3 virtualenv setup for Certbot -export VENV_NAME="${VENV_NAME:-venv3}" -export VENV_ARGS="--python python3" +if command -v python3; then + export VENV_NAME="${VENV_NAME:-venv3}" + export VENV_ARGS="--python python3" +else + echo "Couldn't find python3 in $PATH" + exit 1 +fi ./tools/_venv_common.sh \ -e acme[dev] \ + -e .[dev,docs] \ + -e certbot-apache \ + -e certbot-nginx \ + -e letshelp-certbot \ + -e certbot-compatibility-test From b3116af5b4046e8966f0786db8b0762a487ec0bf Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 21 Apr 2017 18:49:57 -0700 Subject: [PATCH 128/128] Add url and kid to jws. (#4340) * Add url and kid to jws. This will be required in order to implement the latest ACME spec, which uses these protected header fields. * Add comments and fix lint. * Enforce mutual exclusivity of jwk and kid. --- acme/acme/jws.py | 28 +++++++++++++++++++--------- acme/acme/jws_test.py | 18 ++++++++++++++++-- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 54bc26d94..79e96edcb 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -1,14 +1,18 @@ -"""ACME JOSE JWS.""" +"""ACME-specific JWS. + +The JWS implementation in acme.jose only implements the base JOSE standard. In +order to support the new header fields defined in ACME, this module defines some +ACME-specific classes that layer on top of acme.jose. +""" from acme import jose class Header(jose.Header): - """ACME JOSE Header. - - .. todo:: Implement ``acmePath``. - + """ACME-specific JOSE Header. Implements nonce, kid, and url. """ nonce = jose.Field('nonce', omitempty=True, encoder=jose.encode_b64jose) + kid = jose.Field('kid', omitempty=True) + url = jose.Field('url', omitempty=True) @nonce.decoder def nonce(value): # pylint: disable=missing-docstring,no-self-argument @@ -20,7 +24,7 @@ class Header(jose.Header): class Signature(jose.Signature): - """ACME Signature.""" + """ACME-specific Signature. Uses ACME-specific Header for customer fields.""" __slots__ = jose.Signature._orig_slots # pylint: disable=no-member # TODO: decoder/encoder should accept cls? Otherwise, subclassing @@ -34,11 +38,17 @@ class Signature(jose.Signature): class JWS(jose.JWS): - """ACME JWS.""" + """ACME-specific JWS. Includes none, url, and kid in protected header.""" signature_cls = Signature __slots__ = jose.JWS._orig_slots # pylint: disable=no-member @classmethod - def sign(cls, payload, key, alg, nonce): # pylint: disable=arguments-differ + # pylint: disable=arguments-differ,too-many-arguments + def sign(cls, payload, key, alg, nonce, url=None, kid=None): + # Per ACME spec, jwk and kid are mutually exclusive, so only include a + # jwk field if kid is not provided. + include_jwk = kid is None return super(JWS, cls).sign(payload, key=key, alg=alg, - protect=frozenset(['nonce']), nonce=nonce) + protect=frozenset(['nonce', 'url', 'kid']), + nonce=nonce, url=url, kid=kid, + include_jwk=include_jwk) diff --git a/acme/acme/jws_test.py b/acme/acme/jws_test.py index e8f8e871a..62409227e 100644 --- a/acme/acme/jws_test.py +++ b/acme/acme/jws_test.py @@ -37,16 +37,30 @@ class JWSTest(unittest.TestCase): self.privkey = KEY self.pubkey = self.privkey.public_key() self.nonce = jose.b64encode(b'Nonce') + self.url = 'hi' + self.kid = 'baaaaa' - def test_it(self): + def test_kid_serialize(self): from acme.jws import JWS jws = JWS.sign(payload=b'foo', key=self.privkey, - alg=jose.RS256, nonce=self.nonce) + alg=jose.RS256, nonce=self.nonce, + url=self.url, kid=self.kid) self.assertEqual(jws.signature.combined.nonce, self.nonce) + self.assertEqual(jws.signature.combined.url, self.url) + self.assertEqual(jws.signature.combined.kid, self.kid) + self.assertEqual(jws.signature.combined.jwk, None) # TODO: check that nonce is in protected header self.assertEqual(jws, JWS.from_json(jws.to_json())) + def test_jwk_serialize(self): + from acme.jws import JWS + jws = JWS.sign(payload=b'foo', key=self.privkey, + alg=jose.RS256, nonce=self.nonce, + url=self.url) + self.assertEqual(jws.signature.combined.kid, None) + self.assertEqual(jws.signature.combined.jwk, self.pubkey) + if __name__ == '__main__': unittest.main() # pragma: no cover