diff --git a/.gitignore b/.gitignore index a01d2e1c7..4dff20caf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ tests/letstest/venv/ # pytest cache .cache + +# docker files +.docker diff --git a/.travis.yml b/.travis.yml index 866f7b12a..e9699a616 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,12 +27,12 @@ matrix: env: TOXENV=py27-oldest BOULDER_INTEGRATION=1 sudo: required services: docker - - python: "2.6" - env: TOXENV=py26 BOULDER_INTEGRATION=1 + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 sudo: required services: docker - sudo: required @@ -73,10 +73,6 @@ matrix: - python: "2.7" env: TOXENV=apacheconftest sudo: required - - python: "3.3" - env: TOXENV=py33 BOULDER_INTEGRATION=1 - sudo: required - services: docker - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=1 sudo: required diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfc0401..1369b0907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,69 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.1 - 2018-01-25 + +### Fixed + +* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host + header of the request is set to an expected value before redirecting users to + the domain found in the header. The previous way Certbot configured Nginx + redirects was a potential security issue which you can read more about at + https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493. +* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges + if basic authentication is configured for the domain you request a + certificate for. +* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6 + based systems rather than Python 2.6. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/49?closed=1 + +## 0.21.0 - 2018-01-17 + +### Added + +* Support for the HTTP-01 challenge type was added to our Apache and Nginx + plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge + type which was what was previously being used by our Apache and Nginx plugins + last week due to a security issue. For more information about Let's Encrypt's + change, click + [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188). + Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no + changes need to be made to your Certbot configuration, however, you should + make sure your server is accessible on port 80 and isn't behind an external + proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to + HTTPS redirects inside Apache and Nginx are fine. +* IPv6 support was added to the Nginx plugin. +* Support for automatically creating server blocks based on the default server + block was added to the Nginx plugin. +* The flags --delete-after-revoke and --no-delete-after-revoke were added + allowing users to control whether the revoke subcommand also deletes the + certificates it is revoking. + +### Changed + +* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME + library. Support for these versions of Python will be removed in the next + major release of Certbot. If you are using certbot-auto on a RHEL 6 based + system, it will guide you through the process of installing Python 3. +* We split our implementation of JOSE (Javascript Object Signing and + Encryption) out of our ACME library and into a separate package named josepy. + This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and + on [GitHub](https://github.com/certbot/josepy). +* We updated the ciphersuites used in Apache to the new [values recommended by + Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29). + The major change here is adding ChaCha20 to the list of supported + ciphersuites. + +### Fixed + +* An issue with our Apache plugin on Gentoo due to differences in their + apache2ctl command have been resolved. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/47?closed=1 + ## 0.20.0 - 2017-12-06 ### Added diff --git a/Dockerfile-dev b/Dockerfile-dev index 581b58f11..9e35ebec8 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,70 +1,21 @@ # This Dockerfile builds an image for development. -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington -MAINTAINER Yan +FROM ubuntu:xenial -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt +# Note: this only exposes the port to other docker containers. +EXPOSE 80 443 WORKDIR /opt/certbot/src -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev git -y && \ +COPY . . +RUN apt-get update && \ + apt-get install apache2 git nginx-light -y && \ + letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ - -# all above files are necessary for setup.py, however, package source -# code directory has to be copied separately to a subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/ -COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ -COPY tests /opt/certbot/src/tests/ - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ - /opt/certbot/venv/bin/pip install -U pip && \ - /opt/certbot/venv/bin/pip install -U setuptools && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx \ - -e /opt/certbot/src/letshelp-certbot \ - -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +RUN VENV_NAME="../venv" tools/venv.sh ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 55780c0ac..03917f8ca 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,9 @@ +If you're having trouble using Certbot and aren't sure you've found a bug or +request for a new feature, please first try asking for help at +https://community.letsencrypt.org/. There is a much larger community there of +people familiar with the project who will be able to more quickly answer your +questions. + ## My operating system is (include version): diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..e3f6e845d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -16,6 +16,7 @@ import re import requests import sys +from acme import crypto_util from acme import errors from acme import jws from acme import messages @@ -39,39 +40,24 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. + :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net, acme_version): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + :param int acme_version: ACME protocol version. 1 or 2. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net + self.acme_version = acme_version @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,28 +69,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -116,6 +82,13 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -130,6 +103,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) + self.net.account = updated_regr return updated_regr def deactivate_registration(self, regr): @@ -153,65 +127,14 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), uri=response.headers.get('Location', uri)) - if authzr.body.identifier != identifier: + if identifier is not None and authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri=None): - """Request challenges. - - :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - if new_authzr_uri is not None: - logger.debug("request_challenges with new_authzr_uri deprecated.") - new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) - # TODO: handle errors - assert response.status_code == http_client.CREATED - return self._authzr_from_response(response, identifier) - - def request_domain_challenges(self, domain, new_authzr_uri=None): - """Request challenges for domain names. - - This is simply a convenience function that wraps around - `request_challenges`, but works with domain names instead of - generic identifiers. See ``request_challenges`` for more - documentation. - - :param str domain: Domain name to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) - def answer_challenge(self, challb, response): """Answer challenge. @@ -227,7 +150,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -288,6 +211,132 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response + def _revoke(self, cert, rsn, url): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :param str url: ACME URL to post to + + :raises .ClientError: If revocation is unsuccessful. + + """ + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') + +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `josepy.JWK` (private) + :ivar alg: `josepy.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, + net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + directory = messages.Directory.from_json( + self.net.get(directory).json()) + super(Client, self).__init__(directory=directory, + net=net, acme_version=1) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self._post(self.directory[new_reg], new_reg) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + new_authz = messages.NewAuthorization(identifier=identifier) + response = self._post(self.directory.new_authz, new_authz) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) + def request_issuance(self, csr, authzrs): """Request issuance. @@ -307,7 +356,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, @@ -492,26 +541,313 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + return self._revoke(cert, rsn, self.directory[messages.Revocation]) + + +class ClientV2(ClientBase): + """ACME client for a v2 API. + + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. + """ + + def __init__(self, directory, net): + """Initialize. + + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, + net=net, acme_version=2) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + """ + response = self._post(self.directory['newAccount'], new_account) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + regr = self._regr_from_response(response) + self.net.account = regr + return regr + + def new_order(self, csr_pem): + """Request a new Order object from the server. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + + identifiers = [] + for name in dnsNames: + identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, + value=name)) + order = messages.NewOrder(identifiers=identifiers) + response = self._post(self.directory['newOrder'], order) + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) + return messages.OrderResource( + body=body, + uri=response.headers.get('Location'), + authorizations=authorizations, + csr_pem=csr_pem) + + def poll_and_finalize(self, orderr, deadline=None): + """Poll authorizations and finalize the order. + + If no deadline is provided, this method will timeout after 90 + seconds. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if deadline is None: + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.poll_authorizations(orderr, deadline) + return self.finalize_order(orderr, deadline) + + def poll_authorizations(self, orderr, deadline): + """Poll Order Resource for status.""" + responses = [] + for url in orderr.body.authorizations: + while datetime.datetime.now() < deadline: + authzr = self._authzr_from_response(self.net.get(url), uri=url) + if authzr.body.status != messages.STATUS_PENDING: + responses.append(authzr) + break + time.sleep(1) + # If we didn't get a response for every authorization, we fell through + # the bottom of the loop due to hitting the deadline. + if len(responses) < len(orderr.body.authorizations): + raise errors.TimeoutError() + failed = [] + for authzr in responses: + if authzr.body.status != messages.STATUS_VALID: + for chall in authzr.body.challenges: + if chall.error != None: + failed.append(authzr) + if len(failed) > 0: + raise errors.ValidationError(failed) + return orderr.update(authorizations=responses) + + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) + wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) + self._post(orderr.body.finalize, wrapped_csr) + while datetime.datetime.now() < deadline: + time.sleep(1) + response = self.net.get(orderr.uri) + body = messages.Order.from_json(response.json()) + if body.error is not None: + raise errors.IssuanceError(body.error) + if body.certificate is not None: + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text + return orderr.update(body=body, fullchain_pem=certificate_response) + raise errors.TimeoutError() + + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + + +class BackwardsCompatibleClientV2(object): + """ACME client wrapper that tends towards V2-style calls, but + supports V1 servers. + + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 + """ + + def __init__(self, net, key, server): + directory = messages.Directory.from_json(net.get(server).json()) + self.acme_version = self._acme_version_from_directory(directory) + if self.acme_version == 1: + self.client = Client(directory, key=key, net=net) + else: + self.client = ClientV2(directory, net=net) + + def __getattr__(self, name): + if name in vars(self.client): + return getattr(self.client, name) + elif name in dir(ClientBase): + return getattr(self.client, name) + else: + raise AttributeError() + + def new_account_and_tos(self, regr, check_tos_cb=None): + """Combined register and agree_tos for V1, new_account for V2 + + :param .NewRegistration regr: + :param callable check_tos_cb: callback that raises an error if + the check does not work + """ + def _assess_tos(tos): + if check_tos_cb is not None: + check_tos_cb(tos) + if self.acme_version == 1: + regr = self.client.register(regr) + if regr.terms_of_service is not None: + _assess_tos(regr.terms_of_service) + return self.client.agree_to_tos(regr) + return regr + else: + if "terms_of_service" in self.client.directory.meta: + _assess_tos(self.client.directory.meta.terms_of_service) + regr = regr.update(terms_of_service_agreed=True) + return self.client.new_account(regr) + + def new_order(self, csr_pem): + """Request a new Order object from the server. + + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) + else: + return self.client.new_order(csr_pem) + + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) + chain = crypto_util.dump_pyopenssl_chain(chain) + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + + def _acme_version_from_directory(self, directory): + if hasattr(directory, 'newNonce'): + return 2 + else: + return 1 class ClientNetwork(object): # pylint: disable=too-many-instance-attributes - """Client network.""" + """Wrapper around requests that signs POSTs for authentication. + + Also adds user agent, and handles Content-Type. + """ JSON_CONTENT_TYPE = 'application/json' JOSE_CONTENT_TYPE = 'application/jose+json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, + """Initialize. + + :param josepy.JWK key: Account private key + :param messages.RegistrationResource account: Account object. Required if you are + planning to use .post() with acme_version=2 for anything other than + creating a new account; may be set later after registering. + :param josepy.JWASignature alg: Algoritm to use in signing JWS. + :param bool verify_ssl: Whether to verify certificates on SSL connections. + :param str user_agent: String to send as User-Agent header. + :param float timeout: Timeout for requests. + """ + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() @@ -527,21 +863,31 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url, acme_version): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. - :param .JSONDeSerializable obj: + :param josepy.JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: - :rtype: `.JWS` + :rtype: `josepy.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if acme_version == 2: + kwargs["url"] = url + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -714,8 +1060,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise - def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, + acme_version=1, **kwargs): + data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..1e4db2884 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +import copy import datetime import json import unittest @@ -7,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -18,13 +20,32 @@ from acme import test_util CERT_DER = test_util.load_vector('cert.der') +CERT_SAN_PEM = test_util.load_vector('cert-san.pem') +CSR_SAN_PEM = test_util.load_vector('csr-san.pem') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) +DIRECTORY_V1 = messages.Directory({ + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', +}) -class ClientTest(unittest.TestCase): - """Tests for acme.client.Client.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods +DIRECTORY_V2 = messages.Directory({ + 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', + 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', +}) + + +class ClientTestBase(unittest.TestCase): + """Base for tests in acme.client.""" def setUp(self): self.response = mock.MagicMock( @@ -33,21 +54,6 @@ class ClientTest(unittest.TestCase): self.net.post.return_value = self.response self.net.get.return_value = self.response - self.directory = messages.Directory({ - messages.NewRegistration: - 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: - 'https://www.letsencrypt-demo.org/acme/revoke-cert', - messages.NewAuthorization: - 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', - }) - - from acme.client import Client - self.client = Client( - directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) - self.identifier = messages.Identifier( typ=messages.IDENTIFIER_FQDN, value='example.com') @@ -57,8 +63,7 @@ class ClientTest(unittest.TestCase): contact=self.contact, key=KEY.public_key()) self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( - body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - terms_of_service='https://www.letsencrypt-demo.org/tos') + body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') # Authorization authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1' @@ -75,14 +80,217 @@ class ClientTest(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + + +class BackwardsCompatibleClientV2Test(ClientTestBase): + """Tests for acme.client.BackwardsCompatibleClientV2.""" + + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped) + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded) + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + + def _init(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + return BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + + def test_init_downloads_directory(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + self.net.get.assert_called_once_with(uri) + + def test_init_acme_version(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.acme_version, 1) + + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.assertEqual(client.acme_version, 2) + + def test_forwarding(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.directory, client.client.directory) + self.assertEqual(client.key, KEY) + self.assertEqual(client.update_registration, client.client.update_registration) + self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') + self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') + self.assertRaises(AttributeError, client.__getattr__, 'new_account') + + def test_new_account_and_tos(self): + # v2 no tos + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().new_account.assert_called_with(self.new_reg) + + # v2 tos good + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + client.new_account_and_tos(self.new_reg, lambda x: True) + mock_client().new_account.assert_called_with( + self.new_reg.update(terms_of_service_agreed=True)) + + # v2 tos bad + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + def _tos_cb(tos): + raise errors.Error + self.assertRaises(errors.Error, client.new_account_and_tos, + self.new_reg, _tos_cb) + mock_client().new_account.assert_not_called() + + # v1 yes tos + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service="TOS") + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_called_once_with(regr) + + # v1 no tos + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service=None) + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_not_called() + + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + unused_mock_load_certificate_request): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth + client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + +class ClientTest(ClientTestBase): + """Tests for acme.client.Client.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientTest, self).setUp() + + self.directory = DIRECTORY_V1 + + # Registration + self.regr = self.regr.update( + terms_of_service='https://www.letsencrypt-demo.org/tos') + # Request issuance self.certr = messages.CertificateResource( body=messages_test.CERT, authzrs=(self.authzr,), uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 + from acme.client import Client + self.client = Client( + directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) def test_init_downloads_directory(self): uri = 'http://www.letsencrypt-demo.org/directory' @@ -142,20 +350,23 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_deprecated_arg(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier, new_authzr_uri="hi") self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY, + acme_version=1) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -417,7 +628,8 @@ class ClientTest(unittest.TestCase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None) + self.directory[messages.Revocation], mock.ANY, content_type=None, + acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -432,9 +644,151 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + + def setUp(self): + super(ClientV2Test, self).setUp() + + self.directory = DIRECTORY_V2 + + from acme.client import ClientV2 + self.client = ClientV2(self.directory, self.net) + + self.new_reg = self.new_reg.update(terms_of_service_agreed=True) + + self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2' + self.authz2 = self.authz.update(identifier=messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='www.example.com'), + status=messages.STATUS_PENDING) + self.authzr2 = messages.AuthorizationResource( + body=self.authz2, uri=self.authzr_uri2) + + self.order = messages.Order( + identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, + authorizations=(self.authzr.uri, self.authzr_uri2), + finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') + self.orderr = messages.OrderResource( + body=self.order, + uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1', + authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM) + + def test_new_account(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + + def test_new_order(self): + order_response = copy.deepcopy(self.response) + order_response.status_code = http_client.CREATED + order_response.json.return_value = self.order.to_json() + order_response.headers['Location'] = self.orderr.uri + self.net.post.return_value = order_response + + authz_response = copy.deepcopy(self.response) + authz_response.json.return_value = self.authz.to_json() + authz_response.headers['Location'] = self.authzr.uri + authz_response2 = self.response + authz_response2.json.return_value = self.authz2.to_json() + authz_response2.headers['Location'] = self.authzr2.uri + self.net.get.side_effect = (authz_response, authz_response2) + + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + + @mock.patch('acme.client.datetime') + def test_poll_and_finalize(self, mock_datetime): + mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15) + mock_datetime.timedelta = datetime.timedelta + expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90) + + self.client.poll_authorizations = mock.Mock(return_value=self.orderr) + self.client.finalize_order = mock.Mock(return_value=self.orderr) + + self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr) + self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline) + self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline) + + @mock.patch('acme.client.datetime') + def test_poll_authorizations_timeout(self, mock_datetime): + now_side_effect = [datetime.datetime(2018, 2, 15), + datetime.datetime(2018, 2, 16), + datetime.datetime(2018, 2, 17)] + mock_datetime.datetime.now.side_effect = now_side_effect + self.response.json.side_effect = [ + self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()] + + self.assertRaises( + errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1]) + + def test_poll_authorizations_failure(self): + deadline = datetime.datetime(9999, 9, 9) + challb = self.challr.body.update(status=messages.STATUS_INVALID, + error=messages.Error.with_code('unauthorized')) + authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,)) + self.response.json.return_value = authz.to_json() + + self.assertRaises( + errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline) + + def test_poll_authorizations_success(self): + deadline = datetime.datetime(9999, 9, 9) + updated_authz2 = self.authz2.update(status=messages.STATUS_VALID) + updated_authzr2 = messages.AuthorizationResource( + body=updated_authz2, uri=self.authzr_uri2) + updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2]) + + self.response.json.side_effect = ( + self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json()) + self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr) + + def test_finalize_order_success(self): + updated_order = self.order.update( + certificate='https://www.letsencrypt-demo.org/acme/cert/') + updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM) + + self.response.json.return_value = updated_order.to_json() + self.response.text = CERT_SAN_PEM + + deadline = datetime.datetime(9999, 9, 9) + self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr) + + def test_finalize_order_error(self): + updated_order = self.order.update(error=messages.Error.with_code('unauthorized')) + self.response.json.return_value = updated_order.to_json() + + deadline = datetime.datetime(9999, 9, 9) + self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline) + + def test_finalize_order_timeout(self): + deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, content_type=None, + acme_version=2) + + +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +807,27 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=1) jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=2) + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +1057,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..07b55ae33 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,9 +5,10 @@ import logging import os import re import socket -import sys import OpenSSL +import josepy as jose + from acme import errors @@ -130,8 +131,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host @@ -186,6 +186,15 @@ def make_csr(private_key_pem, domains, must_staple=False): return OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr) +def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): + common_name = loaded_cert_or_req.get_subject().CN + sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. @@ -271,3 +280,23 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..3874ba9d9 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -65,6 +65,30 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): + """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" + + @classmethod + def _call(cls, loader, name): + # pylint: disable=protected-access + from acme.crypto_util import _pyopenssl_cert_or_req_all_names + return _pyopenssl_cert_or_req_all_names(loader(name)) + + def _call_cert(self, name): + return self._call(test_util.load_cert, name) + + def test_cert_one_san_no_common(self): + self.assertEqual(self._call_cert('cert-nocn.der'), + ['no-common-name.badssl.com']) + + def test_cert_no_sans_yes_common(self): + self.assertEqual(self._call_cert('cert.pem'), ['example.com']) + + def test_cert_two_sans_yes_common(self): + self.assertEqual(self._call_cert('cert-san.pem'), + ['example.com', 'www.example.com']) + + class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" @@ -170,9 +194,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +212,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but @@ -201,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..991335958 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -83,6 +83,28 @@ class PollError(ClientError): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) +class ValidationError(Error): + """Error for authorization failures. Contains a list of authorization + resources, each of which is invalid and should have an error field. + """ + def __init__(self, failed_authzrs): + self.failed_authzrs = failed_authzrs + super(ValidationError, self).__init__() + +class TimeoutError(Error): + """Error for when polling an authorization or an order times out.""" + +class IssuanceError(Error): + """Error sent by the server after requesting issuance of a certificate.""" + + def __init__(self, error): + """Initialize. + + :param messages.Error error: The error provided by the server. + """ + self.error = error + super(IssuanceError, self).__init__() + class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6daf55094..23cd66c63 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -171,9 +171,30 @@ class Directory(jose.JSONDeSerializable): class Meta(jose.JSONObjectWithFields): """Directory Meta.""" - terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) - caa_identities = jose.Field('caa-identities', omitempty=True) + caa_identities = jose.Field('caaIdentities', omitempty=True) + + def __init__(self, **kwargs): + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) + # pylint: disable=star-args + super(Directory.Meta, self).__init__(**kwargs) + + @property + def terms_of_service(self): + """URL for the CA TOS""" + return self._terms_of_service or self._terms_of_service_v2 + + def __iter__(self): + # When iterating over fields, use the external name 'terms_of_service' instead of + # the internal '_terms_of_service'. + for name in super(Directory.Meta, self).__iter__(): + yield name[1:] if name == '_terms_of_service' else name + + def _internal_name(self, name): + return '_' + name if name == 'terms_of_service' else name + @classmethod def _canon_key(cls, key): @@ -251,6 +272,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) + terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' @@ -482,3 +504,50 @@ class Revocation(jose.JSONObjectWithFields): certificate = jose.Field( 'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert) reason = jose.Field('reason') + + +class Order(ResourceBody): + """Order Resource Body. + + :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: + :ivar list of str authorizations: URLs of authorizations. + :ivar str certificate: URL to download certificate as a fullchain PEM. + :ivar str finalize: URL to POST to to request issuance once all + authorizations have "valid" status. + :ivar datetime.datetime expires: When the order expires. + :ivar .Error error: Any error that occurred during finalization, if applicable. + """ + identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + authorizations = jose.Field('authorizations', omitempty=True) + certificate = jose.Field('certificate', omitempty=True) + finalize = jose.Field('finalize', omitempty=True) + expires = fields.RFC3339Field('expires', omitempty=True) + error = jose.Field('error', omitempty=True, decoder=Error.from_json) + + @identifiers.decoder + def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + return tuple(Identifier.from_json(identifier) for identifier in value) + +class OrderResource(ResourceWithURI): + """Order Resource. + + :ivar acme.messages.Order body: + :ivar str csr_pem: The CSR this Order will be finalized with. + :ivar list of acme.messages.AuthorizationResource authorizations: + Fully-fetched AuthorizationResource objects. + :ivar str fullchain_pem: The fetched contents of the certificate URL + produced once the order was finalized, if it's present. + """ + body = jose.Field('body', decoder=Order.from_json) + csr_pem = jose.Field('csr_pem', omitempty=True) + authorizations = jose.Field('authorizations') + fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + +@Directory.register +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index fa885d3c2..64bc81efd 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -157,7 +157,7 @@ class DirectoryTest(unittest.TestCase): 'meta': { 'terms-of-service': 'https://example.com/acme/terms', 'website': 'https://www.example.com/', - 'caa-identities': ['example.com'], + 'caaIdentities': ['example.com'], }, }) @@ -165,6 +165,13 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + def test_iter_meta(self): + result = False + for k in self.dir.meta: + if k == 'terms_of_service': + result = self.dir.meta[k] == 'https://example.com/acme/terms' + self.assertTrue(result) + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" @@ -401,5 +408,21 @@ class RevocationTest(unittest.TestCase): hash(Revocation.from_json(self.rev.to_json())) +class OrderResourceTest(unittest.TestCase): + """Tests for acme.messages.OrderResource.""" + + def setUp(self): + from acme.messages import OrderResource + self.regr = OrderResource( + body=mock.sentinel.body, uri=mock.sentinel.uri) + + def test_to_partial_json(self): + self.assertEqual(self.regr.to_json(), { + 'body': mock.sentinel.body, + 'uri': mock.sentinel.uri, + 'authorizations': None, + }) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/testdata/cert-nocn.der b/acme/acme/testdata/cert-nocn.der new file mode 100644 index 000000000..59da83ccc Binary files /dev/null and b/acme/acme/testdata/cert-nocn.der differ diff --git a/acme/docs/api/other.rst b/acme/docs/api/other.rst deleted file mode 100644 index eb27a5d53..000000000 --- a/acme/docs/api/other.rst +++ /dev/null @@ -1,5 +0,0 @@ -Other ACME objects ------------------- - -.. automodule:: acme.other - :members: diff --git a/acme/setup.py b/acme/setup.py index 7ebdfe46e..071b56ab3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -19,19 +19,10 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -52,16 +43,15 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 5a33346ea..6377bb114 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -24,9 +25,10 @@ from certbot_apache import apache_util from certbot_apache import augeas_configurator from certbot_apache import constants from certbot_apache import display_ops -from certbot_apache import tls_sni_01 +from certbot_apache import http_01 from certbot_apache import obj from certbot_apache import parser +from certbot_apache import tls_sni_01 from collections import defaultdict @@ -151,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -261,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -279,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -310,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -326,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -390,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -435,12 +558,35 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True return False - def _find_best_vhost(self, target_name): + def find_best_http_vhost(self, target, filter_defaults, port="80"): + """Returns non-HTTPS vhost objects found from the Apache config + + :param str target: Domain name of the desired VirtualHost + :param bool filter_defaults: whether _default_ vhosts should be + included if it is the best match + :param str port: port number the vhost should be listening on + + :returns: VirtualHost object that's the best match for target name + :rtype: `obj.VirtualHost` or None + """ + filtered_vhosts = [] + for vhost in self.vhosts: + if any(a.is_wildcard() or a.get_port() == port for a in vhost.addrs) and not vhost.ssl: + filtered_vhosts.append(vhost) + return self._find_best_vhost(target, filtered_vhosts, filter_defaults) + + def _find_best_vhost(self, target_name, vhosts=None, filter_defaults=True): """Finds the best vhost for a target_name. This does not upgrade a vhost to HTTPS... it only finds the most appropriate vhost for the given target_name. + :param str target_name: domain handled by the desired vhost + :param vhosts: vhosts to consider + :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost` + :param bool filter_defaults: whether a vhost with a _default_ + addr is acceptable + :returns: VHost or None """ @@ -452,7 +598,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Points 1 - Address name with no SSL best_candidate = None best_points = 0 - for vhost in self.vhosts: + + if vhosts is None: + vhosts = self.vhosts + + for vhost in vhosts: if vhost.modmacro is True: continue names = vhost.get_names() @@ -476,8 +626,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # No winners here... is there only one reasonable vhost? if best_candidate is None: - # reasonable == Not all _default_ addrs - vhosts = self._non_default_vhosts() + if filter_defaults: + vhosts = self._non_default_vhosts(vhosts) # remove mod_macro hosts from reasonable vhosts reasonable_vhosts = [vh for vh in vhosts if vh.modmacro is False] @@ -486,9 +636,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return best_candidate - def _non_default_vhosts(self): + def _non_default_vhosts(self, vhosts): """Return all non _default_ only vhosts.""" - return [vh for vh in self.vhosts if not all( + return [vh for vh in vhosts if not all( addr.get_addr() == "_default_" for addr in vh.addrs )] @@ -736,31 +886,43 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ - # If nonstandard port, add service definition for matching - if port != "443": + self.prepare_https_modules(temp) + self.ensure_listen(port, https=True) + + def ensure_listen(self, port, https=False): + """Make sure that Apache is listening on the port. Checks if the + Listen statement for the port already exists, and adds it to the + configuration if necessary. + + :param str port: Port number to check and add Listen for if not in + place already + :param bool https: If the port will be used for HTTPS + + """ + + # If HTTPS requested for nonstandard port, add service definition + if https and port != "443": port_service = "%s %s" % (port, "https") else: port_service = port - self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] - # In case no Listens are set (which really is a broken apache config) - if not listens: - listens = ["80"] - # Listen already in place if self._has_port_already(listens, port): return listen_dirs = set(listens) + if not listens: + listen_dirs.add(port_service) + for listen in listens: # For any listen statement, check if the machine also listens on - # Port 443. If not, add such a listen statement. + # the given port. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces if port not in listen_dirs and port_service not in listen_dirs: @@ -772,11 +934,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if "%s:%s" % (ip, port_service) not in listen_dirs and ( "%s:%s" % (ip, port_service) not in listen_dirs): listen_dirs.add("%s:%s" % (ip, port_service)) - self._add_listens(listen_dirs, listens, port) + if https: + self._add_listens_https(listen_dirs, listens, port) + else: + self._add_listens_http(listen_dirs, listens, port) - def _add_listens(self, listens, listens_orig, port): - """Helper method for prepare_server_https to figure out which new - listen statements need adding + def _add_listens_http(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTP on port + + :param set listens: Set of all needed Listen statements + :param list listens_orig: List of existing listen statements + :param string port: Port number we're adding + """ + + new_listens = listens.difference(listens_orig) + + if port in new_listens: + # We have wildcard, skip the rest + self.parser.add_dir(parser.get_aug_path(self.parser.loc["listen"]), + "Listen", port) + self.save_notes += "Added Listen %s directive to %s\n" % ( + port, self.parser.loc["listen"]) + else: + for listen in new_listens: + self.parser.add_dir(parser.get_aug_path( + self.parser.loc["listen"]), "Listen", listen.split(" ")) + self.save_notes += ("Added Listen %s directive to " + "%s\n") % (listen, + self.parser.loc["listen"]) + + def _add_listens_https(self, listens, listens_orig, port): + """Helper method for ensure_listen to figure out which new + listen statements need adding for listening HTTPS on port :param set listens: Set of all needed Listen statements :param list listens_orig: List of existing listen statements @@ -1201,7 +1391,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + # Only include the TLS configuration if not already included + existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path) + if not existing_inc: + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path @@ -1305,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise @@ -1855,7 +2051,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] def perform(self, achalls): """Perform the configuration related challenge. @@ -1867,16 +2063,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ self._chall_out.update(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.ApacheTlsSni01(self) + http_doer = http_01.ApacheHttp01(self) + sni_doer = tls_sni_01.ApacheTlsSni01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() - if sni_response: + http_response = http_doer.perform() + sni_response = sni_doer.perform() + if http_response or sni_response: # Must reload in order to activate the challenges. # Handled here because we may be able to load up other challenge # types @@ -1886,14 +2087,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # of identifying when the new configuration is being used. time.sleep(3) - # Go through all of the challenges and assign them to the proper - # place in the responses return value. All responses must be in the - # same order as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + self._update_responses(responses, http_response, http_doer) + self._update_responses(responses, sni_response, sni_doer) return responses + def _update_responses(self, responses, chall_response, chall_doer): + # Go through all of the challenges and assign them to the proper + # place in the responses return value. All responses must be in the + # same order as the original challenges. + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp + def cleanup(self, achalls): """Revert all challenges.""" self._chall_out.difference_update(achalls) diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py new file mode 100644 index 000000000..cce93a646 --- /dev/null +++ b/certbot-apache/certbot_apache/http_01.py @@ -0,0 +1,174 @@ +"""A class that performs HTTP-01 challenges for Apache""" +import logging +import os + +from certbot import errors + +from certbot.plugins import common + +logger = logging.getLogger(__name__) + +class ApacheHttp01(common.TLSSNI01): + """Class that performs HTTP-01 challenges within the Apache configurator.""" + + CONFIG_TEMPLATE22_PRE = """\ + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] + + """ + CONFIG_TEMPLATE22_POST = """\ + + Order Allow,Deny + Allow from all + + + Order Allow,Deny + Allow from all + + """ + + CONFIG_TEMPLATE24_PRE = """\ + RewriteEngine on + RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] + """ + CONFIG_TEMPLATE24_POST = """\ + + Require all granted + + + Require all granted + + """ + + def __init__(self, *args, **kwargs): + super(ApacheHttp01, self).__init__(*args, **kwargs) + self.challenge_conf_pre = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_pre.conf") + self.challenge_conf_post = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_post.conf") + self.challenge_dir = os.path.join( + self.configurator.config.work_dir, + "http_challenges") + self.moded_vhosts = set() + + def perform(self): + """Perform all HTTP-01 challenges.""" + if not self.achalls: + return [] + # Save any changes to the configuration as a precaution + # About to make temporary changes to the config + self.configurator.save("Changes before challenge setup", True) + + self.configurator.ensure_listen(str( + self.configurator.config.http01_port)) + self.prepare_http01_modules() + + responses = self._set_up_challenges() + + self._mod_config() + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def prepare_http01_modules(self): + """Make sure that we have the needed modules available for http01""" + + if self.configurator.conf("handle-modules"): + needed_modules = ["rewrite"] + if self.configurator.version < (2, 4): + needed_modules.append("authz_host") + else: + needed_modules.append("authz_core") + for mod in needed_modules: + if mod + "_module" not in self.configurator.parser.modules: + self.configurator.enable_mod(mod, temp=True) + + def _mod_config(self): + for chall in self.achalls: + vh = self.configurator.find_best_http_vhost( + chall.domain, filter_defaults=False, + port=str(self.configurator.config.http01_port)) + if vh: + self._set_up_include_directives(vh) + else: + for vh in self._relevant_vhosts(): + self._set_up_include_directives(vh) + + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_pre) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_post) + + if self.configurator.version < (2, 4): + config_template_pre = self.CONFIG_TEMPLATE22_PRE + config_template_post = self.CONFIG_TEMPLATE22_POST + else: + config_template_pre = self.CONFIG_TEMPLATE24_PRE + config_template_post = self.CONFIG_TEMPLATE24_POST + + config_text_pre = config_template_pre.format(self.challenge_dir) + config_text_post = config_template_post.format(self.challenge_dir) + + logger.debug("writing a pre config file with text:\n %s", config_text_pre) + with open(self.challenge_conf_pre, "w") as new_conf: + new_conf.write(config_text_pre) + logger.debug("writing a post config file with text:\n %s", config_text_post) + with open(self.challenge_conf_post, "w") as new_conf: + new_conf.write(config_text_post) + + def _relevant_vhosts(self): + http01_port = str(self.configurator.config.http01_port) + relevant_vhosts = [] + for vhost in self.configurator.vhosts: + if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs): + if not vhost.ssl: + relevant_vhosts.append(vhost) + if not relevant_vhosts: + raise errors.PluginError( + "Unable to find a virtual host listening on port {0} which is" + " currently needed for Certbot to prove to the CA that you" + " control your domain. Please add a virtual host for port" + " {0}.".format(http01_port)) + + return relevant_vhosts + + def _set_up_challenges(self): + if not os.path.isdir(self.challenge_dir): + os.makedirs(self.challenge_dir) + os.chmod(self.challenge_dir, 0o755) + + responses = [] + for achall in self.achalls: + responses.append(self._set_up_challenge(achall)) + + return responses + + def _set_up_challenge(self, achall): + response, validation = achall.response_and_validation() + + name = os.path.join(self.challenge_dir, achall.chall.encode("token")) + + self.configurator.reverter.register_file_creation(True, name) + with open(name, 'wb') as f: + f.write(validation.encode()) + os.chmod(name, 0o644) + + return response + + def _set_up_include_directives(self, vhost): + """Includes override configuration to the beginning and to the end of + VirtualHost. Note that this include isn't added to Augeas search tree""" + + if vhost not in self.moded_vhosts: + logger.debug( + "Adding a temporary challenge validation Include for name: %s " + + "in: %s", vhost.name, vhost.filep) + self.configurator.parser.add_dir_beginning( + vhost.path, "Include", self.challenge_conf_pre) + self.configurator.parser.add_dir( + vhost.path, "Include", self.challenge_conf_post) + + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py index 6e2e34ba9..02dffc3f7 100644 --- a/certbot-apache/certbot_apache/override_debian.py +++ b/certbot-apache/certbot_apache/override_debian.py @@ -140,5 +140,5 @@ class DebianConfigurator(configurator.ApacheConfigurator): "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.conf("dismod"), mod_name]) + temp, [self.conf("dismod"), "-f", mod_name]) util.run_script([self.conf("enmod"), mod_name]) diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 7715d2c35..d7da1e55e 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -332,6 +332,23 @@ class ApacheParser(object): else: self.aug.set(aug_conf_path + "/directive[last()]/arg", args) + def add_dir_beginning(self, aug_conf_path, dirname, args): + """Adds the directive to the beginning of defined aug_conf_path. + + :param str aug_conf_path: Augeas configuration path to add directive + :param str dirname: Directive to add + :param args: Value of the directive. ie. Listen 443, 443 is arg + :type args: list or str + """ + first_dir = aug_conf_path + "/directive[1]" + self.aug.insert(first_dir, "directive", True) + self.aug.set(first_dir, dirname) + if isinstance(args, list): + for i, value in enumerate(args, 1): + self.aug.set(first_dir + "/arg[%d]" % (i), value) + else: + self.aug.set(first_dir + "/arg", args) + def find_dir(self, directive, arg=None, start=None, exclude=True): """Finds directive in the configuration. diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 4f85e1e3f..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -126,7 +126,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() self.assertEqual(names, set( ["certbot.demo", "ocspvhost.com", "encryption-example.demo", - "nonsym.link", "vhost.in.rootconf"] + "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"] )) @certbot_util.patch_get_utility() @@ -146,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest): names = self.config.get_all_names() # Names get filtered, only 5 are returned - self.assertEqual(len(names), 7) + self.assertEqual(len(names), 8) self.assertTrue("zombo.com" in names) self.assertTrue("google.com" in names) self.assertTrue("certbot.demo" in names) @@ -260,6 +260,20 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.PluginError, self.config.choose_vhost, "none.com") + def test_find_best_http_vhost_default(self): + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + self.config.vhosts = [vh] + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) + + def test_find_best_http_vhost_port(self): + port = "8080" + vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + False, True, "encryption-example.demo") + self.config.vhosts.append(vh) + self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) + def test_findbest_continues_on_short_domain(self): # pylint: disable=protected-access chosen_vhost = self.config._find_best_vhost("purple.com") @@ -305,7 +319,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_non_default_vhosts(self): # pylint: disable=protected-access - self.assertEqual(len(self.config._non_default_vhosts()), 8) + vhosts = self.config._non_default_vhosts(self.config.vhosts) + self.assertEqual(len(vhosts), 8) def test_deploy_cert_enable_new_vhost(self): # Create @@ -320,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest): "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) + def test_no_duplicate_include(self): + def mock_find_dir(directive, argument, _): + """Mock method for parser.find_dir""" + if directive == "Include" and argument.endswith("options-ssl-apache.conf"): + return ["/path/to/whatever"] + + mock_add = mock.MagicMock() + self.config.parser.add_dir = mock_add + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = False + for a in mock_add.call_args_list: + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + tried_to_add = True + # Include should be added, find_dir is not patched, and returns falsy + self.assertTrue(tried_to_add) + + self.config.parser.find_dir = mock_find_dir + mock_add.reset_mock() + + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = [] + for a in mock_add.call_args_list: + tried_to_add.append(a[0][1] == "Include" and + a[0][2] == self.config.mod_ssl_conf) + # Include shouldn't be added, as patched find_dir "finds" existing one + self.assertFalse(any(tried_to_add)) + def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -424,6 +466,47 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:80")) + def test_add_listen_80(self): + mock_find = mock.Mock() + mock_add_dir = mock.Mock() + mock_find.return_value = [] + self.config.parser.find_dir = mock_find + self.config.parser.add_dir = mock_add_dir + self.config.ensure_listen("80") + self.assertTrue(mock_add_dir.called) + self.assertTrue(mock_find.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + self.assertEqual(mock_add_dir.call_args[0][2], "80") + + def test_add_listen_80_named(self): + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2", "test3"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + mock_add_dir = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir = mock_add_dir + + self.config.ensure_listen("80") + self.assertEqual(mock_add_dir.call_count, 0) + + # Reset return lists and inputs + mock_add_dir.reset_mock() + mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"] + + # Test + self.config.ensure_listen("8080") + self.assertEqual(mock_add_dir.call_count, 3) + self.assertTrue(mock_add_dir.called) + self.assertEqual(mock_add_dir.call_args[0][1], "Listen") + call_found = False + for mock_call in mock_add_dir.mock_calls: + if mock_call[1][2] == ['1.2.3.4:8080']: + call_found = True + self.assertTrue(call_found) + def test_prepare_server_https(self): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -435,7 +518,6 @@ class MultipleVhostsTest(util.ApacheTest): # This will test the Add listen self.config.parser.find_dir = mock_find self.config.parser.add_dir_to_ifmodssl = mock_add_dir - self.config.prepare_server_https("443") # Changing the order these modules are enabled breaks the reverter self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb") @@ -676,23 +758,33 @@ class MultipleVhostsTest(util.ApacheTest): self.config._add_name_vhost_if_necessary(self.vh_truth[0]) self.assertEqual(self.config.add_name_vhost.call_count, 2) + @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform") @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") - def test_perform(self, mock_restart, mock_perform): + def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded - account_key, achall1, achall2 = self.get_achalls() + account_key, achalls = self.get_key_and_achalls() - expected = [ - achall1.response(account_key), - achall2.response(account_key), - ] + all_expected = [] + http_expected = [] + tls_expected = [] + for achall in achalls: + response = achall.response(account_key) + if isinstance(achall.chall, challenges.HTTP01): + http_expected.append(response) + else: + tls_expected.append(response) + all_expected.append(response) - mock_perform.return_value = expected - responses = self.config.perform([achall1, achall2]) + mock_http_perform.return_value = http_expected + mock_tls_perform.return_value = tls_expected - self.assertEqual(mock_perform.call_count, 1) - self.assertEqual(responses, expected) + responses = self.config.perform(achalls) + + self.assertEqual(mock_http_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(responses, all_expected) self.assertEqual(mock_restart.call_count, 1) @@ -700,29 +792,32 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() - self.config._chall_out.add(achall1) # pylint: disable=protected-access - self.config._chall_out.add(achall2) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall1]) - self.assertFalse(mock_restart.called) - - self.config.cleanup([achall2]) - self.assertTrue(mock_restart.called) + for i, achall in enumerate(achalls): + self.config.cleanup([achall]) + if i == len(achalls) - 1: + self.assertTrue(mock_restart.called) + else: + self.assertFalse(mock_restart.called) @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") def test_cleanup_no_errors(self, mock_cfg, mock_restart): mock_cfg.return_value = "" - _, achall1, achall2 = self.get_achalls() + _, achalls = self.get_key_and_achalls() + self.config.http_doer = mock.MagicMock() - self.config._chall_out.add(achall1) # pylint: disable=protected-access + for achall in achalls: + self.config._chall_out.add(achall) # pylint: disable=protected-access - self.config.cleanup([achall2]) + self.config.cleanup([achalls[-1]]) self.assertFalse(mock_restart.called) - self.config.cleanup([achall1, achall2]) + self.config.cleanup(achalls) self.assertTrue(mock_restart.called) @mock.patch("certbot.util.run_script") @@ -1151,7 +1246,7 @@ class MultipleVhostsTest(util.ApacheTest): not_rewriterule = "NotRewriteRule ^ ..." self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule)) - def get_achalls(self): + def get_key_and_achalls(self): """Return testing achallenges.""" account_key = self.rsa512jwk achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -1166,8 +1261,12 @@ class MultipleVhostsTest(util.ApacheTest): token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), "pending"), domain="certbot.demo", account_key=account_key) + achall3 = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=(b'x' * 16)), "pending"), + domain="example.org", account_key=account_key) - return account_key, achall1, achall2 + return account_key, (achall1, achall2, achall3) def test_make_addrs_sni_ready(self): self.config.version = (2, 2) @@ -1238,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py new file mode 100644 index 000000000..9ed4ee509 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -0,0 +1,202 @@ +"""Test for certbot_apache.http_01.""" +import mock +import os +import unittest + +from acme import challenges + +from certbot import achallenges +from certbot import errors + +from certbot.tests import acme_util + +from certbot_apache.tests import util + + +NUM_ACHALLS = 3 + + +class ApacheHttp01TestMeta(type): + """Generates parmeterized tests for testing perform.""" + def __new__(mcs, name, bases, class_dict): + + def _gen_test(num_achalls, minor_version): + def _test(self): + achalls = self.achalls[:num_achalls] + vhosts = self.vhosts[:num_achalls] + self.config.version = (2, minor_version) + self.common_perform_test(achalls, vhosts) + return _test + + for i in range(1, NUM_ACHALLS + 1): + for j in (2, 4): + test_name = "test_perform_{0}_{1}".format(i, j) + class_dict[test_name] = _gen_test(i, j) + return type.__new__(mcs, name, bases, class_dict) + + +class ApacheHttp01Test(util.ApacheTest): + """Test for certbot_apache.http_01.ApacheHttp01.""" + + __metaclass__ = ApacheHttp01TestMeta + + def setUp(self, *args, **kwargs): + super(ApacheHttp01Test, self).setUp(*args, **kwargs) + + self.account_key = self.rsa512jwk + self.achalls = [] + vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multiple_vhosts") + # Takes the vhosts for encryption-example.demo, certbot.demo, and + # vhost.in.rootconf + self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]] + + for i in range(NUM_ACHALLS): + self.achalls.append( + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))), + "pending"), + domain=self.vhosts[i].name, account_key=self.account_key)) + + modules = ["rewrite", "authz_core", "authz_host"] + for mod in modules: + self.config.parser.modules.add("mod_{0}.c".format(mod)) + self.config.parser.modules.add(mod + "_module") + + from certbot_apache.http_01 import ApacheHttp01 + self.http = ApacheHttp01(self.config) + + def test_empty_perform(self): + self.assertFalse(self.http.perform()) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_enable_modules_22(self, mock_enmod): + self.config.version = (2, 2) + self.config.parser.modules.remove("authz_host_module") + self.config.parser.modules.remove("mod_authz_host.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_host") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod") + def test_enable_modules_24(self, mock_enmod): + self.config.parser.modules.remove("authz_core_module") + self.config.parser.modules.remove("mod_authz_core.c") + + enmod_calls = self.common_enable_modules_test(mock_enmod) + self.assertEqual(enmod_calls[0][0][0], "authz_core") + + def common_enable_modules_test(self, mock_enmod): + """Tests enabling mod_rewrite and other modules.""" + self.config.parser.modules.remove("rewrite_module") + self.config.parser.modules.remove("mod_rewrite.c") + + self.http.prepare_http01_modules() + + self.assertTrue(mock_enmod.called) + calls = mock_enmod.call_args_list + other_calls = [] + for call in calls: + if "rewrite" != call[0][0]: + other_calls.append(call) + + # If these lists are equal, we never enabled mod_rewrite + self.assertNotEqual(calls, other_calls) + return other_calls + + def test_same_vhost(self): + vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo") + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain=vhost.name, account_key=self.account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'b' * 16))), + "pending"), + domain=next(iter(vhost.aliases)), account_key=self.account_key) + ] + self.common_perform_test(achalls, [vhost]) + + def test_anonymous_vhost(self): + vhosts = [v for v in self.config.vhosts if not v.ssl] + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=((b'a' * 16))), + "pending"), + domain="something.nonexistent", account_key=self.account_key)] + self.common_perform_test(achalls, vhosts) + + def test_no_vhost(self): + for achall in self.achalls: + self.http.add_chall(achall) + self.config.config.http01_port = 12345 + self.assertRaises(errors.PluginError, self.http.perform) + + def common_perform_test(self, achalls, vhosts): + """Tests perform with the given achalls.""" + challenge_dir = self.http.challenge_dir + self.assertFalse(os.path.exists(challenge_dir)) + for achall in achalls: + self.http.add_chall(achall) + + expected_response = [ + achall.response(self.account_key) for achall in achalls] + self.assertEqual(self.http.perform(), expected_response) + + self.assertTrue(os.path.isdir(self.http.challenge_dir)) + self._has_min_permissions(self.http.challenge_dir, 0o755) + self._test_challenge_conf() + + for achall in achalls: + self._test_challenge_file(achall) + + for vhost in vhosts: + if not vhost.ssl: + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, + vhost.path) + self.assertEqual(len(matches), 1) + + self.assertTrue(os.path.exists(challenge_dir)) + + def _test_challenge_conf(self): + with open(self.http.challenge_conf_pre) as f: + pre_conf_contents = f.read() + + with open(self.http.challenge_conf_post) as f: + post_conf_contents = f.read() + + self.assertTrue("RewriteEngine on" in pre_conf_contents) + self.assertTrue("RewriteRule" in pre_conf_contents) + + self.assertTrue(self.http.challenge_dir in post_conf_contents) + if self.config.version < (2, 4): + self.assertTrue("Allow from all" in post_conf_contents) + else: + self.assertTrue("Require all granted" in post_conf_contents) + + def _test_challenge_file(self, achall): + name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) + validation = achall.validation(self.account_key) + + self._has_min_permissions(name, 0o644) + with open(name, 'rb') as f: + self.assertEqual(f.read(), validation.encode()) + + def _has_min_permissions(self, path, min_mode): + """Tests the given file has at least the permissions in mode.""" + st_mode = os.stat(path).st_mode + self.assertEqual(st_mode, st_mode | min_mode) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py index a9eb129c2..4496781c9 100644 --- a/certbot-apache/certbot_apache/tests/parser_test.py +++ b/certbot-apache/certbot_apache/tests/parser_test.py @@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest): for i, match in enumerate(matches): self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_add_dir_beginning(self): + aug_default = "/files" + self.parser.loc["default"] + self.parser.add_dir_beginning(aug_default, + "AddDirectiveBeginning", + "testBegin") + + self.assertTrue( + self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default)) + + self.assertEqual( + self.parser.aug.get(aug_default+"/directive[1]"), + "AddDirectiveBeginning") + self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"]) + matches = self.parser.find_dir("AddList", None, aug_default) + for i, match in enumerate(matches): + self.assertEqual(self.parser.aug.get(match), str(i + 1)) + def test_empty_arg(self): self.assertEquals(None, self.parser.get_arg("/files/whatever/nonexistent")) diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf index b3147a523..965ca2222 100644 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf @@ -1,5 +1,6 @@ ServerName certbot.demo +ServerAlias www.certbot.demo ServerAdmin webmaster@localhost DocumentRoot /var/www-certbot-reworld/static/ diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 6c37c2ecc..8cea97f04 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -1,6 +1,6 @@ """Test for certbot_apache.tls_sni_01.""" -import unittest import shutil +import unittest import mock @@ -16,8 +16,8 @@ from six.moves import xrange # pylint: disable=redefined-builtin, import-error class TlsSniPerformTest(util.ApacheTest): """Test the ApacheTlsSni01 challenge.""" - auth_key = common_test.TLSSNI01Test.auth_key - achalls = common_test.TLSSNI01Test.achalls + auth_key = common_test.AUTH_KEY + achalls = common_test.ACHALLS def setUp(self): # pylint: disable=arguments-differ super(TlsSniPerformTest, self).setUp() diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index ca667465c..1daaa00c5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -103,6 +103,7 @@ def get_apache_configurator( # pylint: disable=too-many-arguments, too-many-loc apache_challenge_location=config_path, backup_dir=backups, config_dir=config_dir, + http01_port=80, temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) @@ -169,7 +170,7 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, True, - "certbot.demo"), + "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 3270f2c79..336233bd4 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] @@ -32,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-auto b/certbot-auto index 444bee1b9..d3a5c23e5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -696,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -715,11 +775,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -782,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -816,7 +903,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +915,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +944,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1098,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1184,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1437,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1472,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1509,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1535,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1543,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1566,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1581,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1610,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1faf30643..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' install_requires = [ 'certbot', @@ -34,16 +34,15 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 428271045..e5687a9f5 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -39,10 +38,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4a103193f..0ef31a90c 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +40,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 23098d4b6..11c2aea24 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] @@ -32,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4ed5a06ca..414a058fa 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +40,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a0b88aab..18d773347 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +40,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..cea754c06 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -224,4 +224,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..53f84dd6e 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -223,9 +223,13 @@ class GoogleClientTest(unittest.TestCase): response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b00bd1ac3..d5def1bf9 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -15,9 +15,7 @@ install_requires = [ 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' @@ -36,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -44,10 +43,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b8f50254e..13fa742d5 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +40,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 2a388e487..01c9579c1 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +40,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 78007afb5..64b126595 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -12,9 +12,7 @@ install_requires = [ 'certbot=={0}'.format(version), 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +29,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -39,10 +38,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 7d1eb0bc9..e45343f79 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,16 +3,14 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.21.0.dev0' +version = '0.22.0.dev0' install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -24,6 +22,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -32,10 +31,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8af474c5e..9f091c0fd 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -26,20 +26,11 @@ from certbot_nginx import constants from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 +from certbot_nginx import http_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [ - ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - ['\n'] -] - -REDIRECT_COMMENT_BLOCK = [ - ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' return 301 https://$host$request_uri;'], - ['\n'] -] @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) @@ -208,7 +199,8 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -259,9 +251,9 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain): + def _vhost_from_duplicated_default(self, domain, port=None): if self.new_vhost is None: - default_vhost = self._get_default_vhost() + default_vhost = self._get_default_vhost(port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) self.new_vhost.names = set() @@ -276,15 +268,16 @@ class NginxConfigurator(common.Installer): name_block[0].append(name) self.parser.add_server_directives(vhost, name_block, replace=True) - def _get_default_vhost(self): + def _get_default_vhost(self, port): vhost_list = self.parser.get_vhosts() # if one has default_server set, return that one default_vhosts = [] for vhost in vhost_list: for addr in vhost.addrs: if addr.default: - default_vhosts.append(vhost) - break + if port is None or self._port_matches(port, addr.get_port()): + default_vhosts.append(vhost) + break if len(default_vhosts) == 1: return default_vhosts[0] @@ -366,7 +359,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port): + def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -380,12 +373,27 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number + :param bool create_if_no_match: If we should create a new vhost from default + when there is no match found. If we can't choose a default, raise a + MisconfigurationError. + :returns: vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` """ matches = self._get_redirect_ranked_matches(target_name, port) - return self._select_best_name_match(matches) + vhost = self._select_best_name_match(matches) + if not vhost and create_if_no_match: + vhost = self._vhost_from_duplicated_default(target_name, port=port) + return vhost + + def _port_matches(self, test_port, matching_port): + # test_port is a number, matching is a number or "" or None + if matching_port == "" or matching_port is None: + # if no port is specified, Nginx defaults to listening on port 80. + return test_port == self.DEFAULT_LISTEN_PORT + else: + return test_port == matching_port def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -394,20 +402,13 @@ class NginxConfigurator(common.Installer): Rank by how well these match target_name. :param str target_name: The name to match - :param str port: port number + :param str port: port number as a string :returns: list of dicts containing the vhost, the matching name, and the numerical rank :rtype: list """ all_vhosts = self.parser.get_vhosts() - def _port_matches(test_port, matching_port): - # test_port is a number, matching is a number or "" or None - if matching_port == "" or matching_port is None: - # if no port is specified, Nginx defaults to listening on port 80. - return test_port == self.DEFAULT_LISTEN_PORT - else: - return test_port == matching_port def _vhost_matches(vhost, port): found_matching_port = False @@ -417,7 +418,7 @@ class NginxConfigurator(common.Installer): found_matching_port = (port == self.DEFAULT_LISTEN_PORT) else: for addr in vhost.addrs: - if _port_matches(port, addr.get_port()) and addr.ssl == False: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: found_matching_port = True if found_matching_port: @@ -560,24 +561,17 @@ class NginxConfigurator(common.Installer): logger.warning("Failed %s for %s", enhancement, domain) raise - def _has_certbot_redirect(self, vhost): - test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + def _has_certbot_redirect(self, vhost, domain): + test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) - def _has_certbot_redirect_comment(self, vhost): - test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) - return vhost.contains_list(test_redirect_comment_block) - - def _add_redirect_block(self, vhost, active=True): + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ - if active: - redirect_block = REDIRECT_BLOCK - else: - redirect_block = REDIRECT_COMMENT_BLOCK + redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False) + vhost, redirect_block, replace=False, insert_at_top=True) def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -604,6 +598,7 @@ class NginxConfigurator(common.Installer): self.DEFAULT_LISTEN_PORT) return + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, only_directives=['listen', 'server_name']) @@ -620,20 +615,18 @@ class NginxConfigurator(common.Installer): # remove all non-ssl addresses from the existing block self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + # Add this at the bottom to get the right order of directives + return_404_directive = [['\n ', 'return', ' ', '404']] + self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + vhost = new_vhost - if self._has_certbot_redirect(vhost): + if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) else: # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) + self._add_redirect_block(vhost, domain) logger.info("Redirecting all traffic on port %s to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) @@ -840,7 +833,7 @@ class NginxConfigurator(common.Installer): ########################################################################### def get_chall_pref(self, unused_domain): # pylint: disable=no-self-use """Return list of challenge preferences.""" - return [challenges.TLSSNI01] + return [challenges.TLSSNI01, challenges.HTTP01] # Entry point in main.py for performing challenges def perform(self, achalls): @@ -853,15 +846,20 @@ class NginxConfigurator(common.Installer): """ self._chall_out += len(achalls) responses = [None] * len(achalls) - chall_doer = tls_sni_01.NginxTlsSni01(self) + sni_doer = tls_sni_01.NginxTlsSni01(self) + http_doer = http_01.NginxHttp01(self) for i, achall in enumerate(achalls): # Currently also have chall_doer hold associated index of the # challenge. This helps to put all of the responses back together # when they are all complete. - chall_doer.add_chall(achall, i) + if isinstance(achall.chall, challenges.HTTP01): + http_doer.add_chall(achall, i) + else: # tls-sni-01 + sni_doer.add_chall(achall, i) - sni_response = chall_doer.perform() + sni_response = sni_doer.perform() + http_response = http_doer.perform() # Must restart in order to activate the challenges. # Handled here because we may be able to load up other challenge types self.restart() @@ -869,8 +867,9 @@ class NginxConfigurator(common.Installer): # Go through all of the challenges and assign them to the proper place # in the responses return value. All responses must be in the same order # as the original challenges. - for i, resp in enumerate(sni_response): - responses[chall_doer.indices[i]] = resp + for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)): + for i, resp in enumerate(chall_response): + responses[chall_doer.indices[i]] = resp return responses @@ -890,6 +889,14 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] +def _redirect_block_for_domain(domain): + redirect_block = [[ + ['\n ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + '\n ']], + ['\n']] + return redirect_block + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py new file mode 100644 index 000000000..c0dec061a --- /dev/null +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -0,0 +1,203 @@ +"""A class that performs HTTP-01 challenges for Nginx""" + +import logging +import os + +from acme import challenges + +from certbot import errors +from certbot.plugins import common + +from certbot_nginx import obj +from certbot_nginx import nginxparser + + +logger = logging.getLogger(__name__) + + +class NginxHttp01(common.ChallengePerformer): + """HTTP-01 authenticator for Nginx + + :ivar configurator: NginxConfigurator object + :type configurator: :class:`~nginx.configurator.NginxConfigurator` + + :ivar list achalls: Annotated + class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + challenges + + :param list indices: Meant to hold indices of challenges in a + larger array. NginxHttp01 is capable of solving many challenges + at once which causes an indexing issue within NginxConfigurator + who must return all responses in order. Imagine NginxConfigurator + maintaining state about where all of the http-01 Challenges, + TLS-SNI-01 Challenges belong in the response array. This is an + optional utility. + + """ + + def __init__(self, configurator): + super(NginxHttp01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_http_01_cert_challenge.conf") + self._ipv6 = None + self._ipv6only = None + + def perform(self): + """Perform a challenge on Nginx. + + :returns: list of :class:`certbot.acme.challenges.HTTP01Response` + :rtype: list + + """ + if not self.achalls: + return [] + + responses = [x.response(x.account_key) for x in self.achalls] + + # Set up the configuration + self._mod_config() + + # Save reversible changes + self.configurator.save("HTTP Challenge", True) + + return responses + + def _mod_config(self): + """Modifies Nginx config to include server_names_hash_bucket_size directive + and server challenge blocks. + + :raises .MisconfigurationError: + Unable to find a suitable HTTP block in which to include + authenticator hosts. + """ + included = False + include_directive = ['\n', 'include', ' ', self.challenge_conf] + root = self.configurator.parser.config_root + + bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] + + main = self.configurator.parser.parsed[root] + for line in main: + if line[0] == ['http']: + body = line[1] + found_bucket = False + posn = 0 + for inner_line in body: + if inner_line[0] == bucket_directive[1]: + if int(inner_line[1]) < int(bucket_directive[3]): + body[posn] = bucket_directive + found_bucket = True + posn += 1 + if not found_bucket: + body.insert(0, bucket_directive) + if include_directive not in body: + body.insert(0, include_directive) + included = True + break + if not included: + raise errors.MisconfigurationError( + 'Certbot could not find a block to include ' + 'challenges in %s.' % root) + config = [self._make_or_mod_server_block(achall) for achall in self.achalls] + config = [x for x in config if x is not None] + config = nginxparser.UnspacedList(config) + + self.configurator.reverter.register_file_creation( + True, self.challenge_conf) + + with open(self.challenge_conf, "w") as new_conf: + nginxparser.dump(config, new_conf) + + def _default_listen_addresses(self): + """Finds addresses for a challenge block to listen on. + :returns: list of :class:`certbot_nginx.obj.Addr` to apply + :rtype: list + """ + addresses = [] + default_addr = "%s" % self.configurator.config.http01_port + ipv6_addr = "[::]:{0}".format( + self.configurator.config.http01_port) + port = self.configurator.config.http01_port + + if self._ipv6 is None or self._ipv6only is None: + self._ipv6, self._ipv6only = self.configurator.ipv6_info(port) + ipv6, ipv6only = self._ipv6, self._ipv6only + + if ipv6: + # If IPv6 is active in Nginx configuration + if not ipv6only: + # If ipv6only=on is not already present in the config + ipv6_addr = ipv6_addr + " ipv6only=on" + addresses = [obj.Addr.fromstring(default_addr), + obj.Addr.fromstring(ipv6_addr)] + logger.info(("Using default addresses %s and %s for authentication."), + default_addr, + ipv6_addr) + else: + addresses = [obj.Addr.fromstring(default_addr)] + logger.info("Using default address %s for authentication.", + default_addr) + return addresses + + def _get_validation_path(self, achall): + return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) + + def _make_server_block(self, achall): + """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :param list addrs: addresses of challenged domain + :class:`list` of type :class:`~nginx.obj.Addr` + :returns: server block for the challenge host + :rtype: list + """ + addrs = self._default_listen_addresses() + block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs] + + # Ensure we 404 on any other request by setting a root + document_root = os.path.join( + self.configurator.config.work_dir, "http_01_nonexistent") + + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + block.extend([['server_name', ' ', achall.domain], + ['root', ' ', document_root], + [['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]]) + # TODO: do we want to return something else if they otherwise access this block? + return [['server'], block] + + def _make_or_mod_server_block(self, achall): + """Modifies a server block to respond to a challenge. + + :param achall: Annotated HTTP-01 challenge + :type achall: + :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + """ + try: + vhost = self.configurator.choose_redirect_vhost(achall.domain, + '%i' % self.configurator.config.http01_port, create_if_no_match=True) + except errors.MisconfigurationError: + # Couldn't find either a matching name+port server block + # or a port+default_server block, so create a dummy block + return self._make_server_block(achall) + + # Modify existing server block + validation = achall.validation(achall.account_key) + validation_path = self._get_validation_path(achall) + + location_directive = [[['location', ' ', '=', ' ', validation_path], + [['default_type', ' ', 'text/plain'], + ['return', ' ', '200', ' ', validation]]]] + + self.configurator.parser.add_server_directives(vhost, + location_directive, replace=False) + + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, replace=False, insert_at_top=True) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index f5ac5c2e3..e8dc8936d 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,15 +193,6 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False - def has_redirect(self): - """Determine if this vhost has a redirecting statement - """ - for directive_name in REDIRECT_DIRECTIVES: - found = _find_directive(self.raw, directive_name) - if found is not None: - return True - return False - def contains_list(self, test): """Determine if raw server block contains test list at top level """ @@ -225,15 +216,3 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods for a in self.addrs: if not a.ipv6: return True - -def _find_directive(directives, directive_name): - """Find a directive of type directive_name in directives - """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: - return None - - if directives[0] == directive_name: - return directives - - matches = (_find_directive(line, directive_name) for line in directives) - return next((m for m in matches if m is not None), None) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 9f13bc59f..fbd6c0ade 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -276,7 +276,7 @@ class NginxParser(object): return False - def add_server_directives(self, vhost, directives, replace): + def add_server_directives(self, vhost, directives, replace, insert_at_top=False): """Add or replace directives in the server block identified by vhost. This method modifies vhost to be fully consistent with the new directives. @@ -293,10 +293,12 @@ class NginxParser(object): whose information we use to match on :param list directives: The directives to add :param bool replace: Whether to only replace existing directives + :param bool insert_at_top: True if the directives need to be inserted at the top + of the server block instead of the bottom """ self._modify_server_directives(vhost, - functools.partial(_add_directives, directives, replace)) + functools.partial(_add_directives, directives, replace, insert_at_top)) def remove_server_directives(self, vhost, directive_name, match_func=None): """Remove all directives of type directive_name. @@ -521,10 +523,10 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(directives, replace, block): +def _add_directives(directives, replace, insert_at_top, block): """Adds or replaces directives in a config block. - When replace=False, it's an error to try and add a directive that already + When replace=False, it's an error to try and add a nonrepeatable directive that already exists in the config block with a conflicting value. When replace=True and a directive with the same name already exists in the @@ -535,17 +537,18 @@ def _add_directives(directives, replace, block): :param list directives: The new directives. :param bool replace: Described above. + :param bool insert_at_top: Described above. :param list block: The block to replace in """ for directive in directives: - _add_directive(block, directive, replace) + _add_directive(block, directive, replace, insert_at_top) if block and '\n' not in block[-1]: # could be " \n " or ["\n"] ! block.append(nginxparser.UnspacedList('\n')) INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite']) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] @@ -597,7 +600,7 @@ def _find_location(block, directive_name, match_func=None): return next((index for index, line in enumerate(block) \ if line and line[0] == directive_name and (match_func is None or match_func(line))), None) -def _add_directive(block, directive, replace): +def _add_directive(block, directive, replace, insert_at_top): """Adds or replaces a single directive in a config block. See _add_directives for more documentation. @@ -619,7 +622,7 @@ def _add_directive(block, directive, replace): block[location] = directive comment_directive(block, location) return - # Append directive. Fail if the name is not a repeatable directive name, + # Append or prepend directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. @@ -652,8 +655,15 @@ def _add_directive(block, directive, replace): _comment_out_directive(block, included_dir_loc, directive[1]) if can_append(location, directive_name): - block.append(directive) - comment_directive(block, len(block) - 1) + if insert_at_top: + # Add a newline so the comment doesn't comment + # out existing directives + block.insert(0, nginxparser.UnspacedList('\n')) + block.insert(0, directive) + comment_directive(block, 0) + else: + block.append(directive) + comment_directive(block, len(block) - 1) elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index e708b159a..acb7ee282 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser +from certbot_nginx.configurator import _redirect_block_for_domain +from certbot_nginx.nginxparser import UnspacedList from certbot_nginx.tests import util @@ -100,7 +102,7 @@ class NginxConfiguratorTest(util.NginxTest): errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement') def test_get_chall_pref(self): - self.assertEqual([challenges.TLSSNI01], + self.assertEqual([challenges.TLSSNI01, challenges.HTTP01], self.config.get_chall_pref('myhost')) def test_save(self): @@ -291,9 +293,11 @@ class NginxConfiguratorTest(util.NginxTest): parsed_migration_conf[0]) @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform") + @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform") @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart") @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config") - def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform): + def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform, + mock_tls_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -304,7 +308,7 @@ class NginxConfiguratorTest(util.NginxTest): ), domain="localhost", account_key=self.rsa512jwk) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"), + chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", status=messages.Status("pending"), ), domain="example.com", account_key=self.rsa512jwk) @@ -314,10 +318,12 @@ class NginxConfiguratorTest(util.NginxTest): achall2.response(self.rsa512jwk), ] - mock_perform.return_value = expected + mock_tls_perform.return_value = expected[:1] + mock_http_perform.return_value = expected[1:] responses = self.config.perform([achall1, achall2]) - self.assertEqual(mock_perform.call_count, 1) + self.assertEqual(mock_tls_perform.call_count, 1) + self.assertEqual(mock_http_perform.call_count, 1) self.assertEqual(responses, expected) self.config.cleanup([achall1, achall2]) @@ -443,7 +449,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -456,6 +462,8 @@ class NginxConfiguratorTest(util.NginxTest): migration_conf = self.config.parser.abs_path('sites-enabled/migration.com') self.config.enhance("migration.com", "redirect") + expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0] + generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -480,101 +488,27 @@ class NginxConfiguratorTest(util.NginxTest): ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], [], []]], [['server'], [ + [['if', '($host', '=', 'www.example.com)'], [ + ['return', '301', 'https://$host$request_uri']]], + ['#', ' managed by Certbot'], [], ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], - [], []]]], + ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): + def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a # redirect in the block that is managed by certbot # Has a certbot redirect - mock_has_redirect.return_value = True mock_contains_list.return_value = True with mock.patch("certbot_nginx.configurator.logger") as mock_logger: self.config.enhance("www.example.com", "redirect") self.assertEqual(mock_logger.info.call_args[0][0], "Traffic on port %s already redirecting to ssl in %s") - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - self.config.deploy_cert( - "example.org", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block') - def test_redirect_comment_exists(self, mock_add_redirect_block, - mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list): - # Test that we add nothing if there is a non-Certbot redirect and a - # preexisting comment - # Has a non-Certbot redirect and a comment - mock_has_redirect.return_value = True - mock_contains_list.return_value = False # self._has_certbot_redirect(vhost): - mock_has_cb_redirect_comment.return_value = True - - # assert _add_redirect_block not called - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertFalse(mock_add_redirect_block.called) - self.assertTrue(mock_logger.info.called) - def test_redirect_dont_enhance(self): # Test that we don't accidentally add redirect to ssl-only block with mock.patch("certbot_nginx.configurator.logger") as mock_logger: @@ -582,22 +516,18 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_logger.info.call_args[0][0], 'No matching insecure server blocks listening on port %s found.') - def test_no_double_redirect(self): - # Test that we don't also add the commented redirect if we've just added - # a redirect to that vhost this run + def test_double_redirect(self): + # Test that we add one redirect for each domain example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("example.com", "redirect") self.config.enhance("example.org", "redirect") - unexpected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] - ] + expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0] + expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0] + generated_conf = self.config.parser.parsed[example_conf] - for line in unexpected: - self.assertFalse(util.contains_at_depth(generated_conf, line, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2)) def test_staple_ocsp_bad_version(self): self.config.version = (1, 3, 1) @@ -759,7 +689,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py new file mode 100644 index 000000000..0f764e92e --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -0,0 +1,113 @@ +"""Tests for certbot_nginx.http_01""" +import unittest +import shutil + +import mock +import six + +from acme import challenges + +from certbot import achallenges + +from certbot.plugins import common_test +from certbot.tests import acme_util + +from certbot_nginx.tests import util + + +class HttpPerformTest(util.NginxTest): + """Test the NginxHttp01 challenge.""" + + account_key = common_test.AUTH_KEY + achalls = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), + domain="www.example.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01( + token=b"\xba\xa9\xda?=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -32,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 5295e3681..c38180698 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -49,10 +49,10 @@ http { server { # IPv4. - listen 8081; + listen 5002; # IPv6. - listen [::]:8081 default ipv6only=on; - server_name nginx.wtf; + listen [::]:5002 default ipv6only=on; + server_name nginx.wtf nginx2.wtf; root $root/webroot; diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 996cc2201..f236fb103 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -22,13 +22,20 @@ certbot_test_nginx () { "$@" } -certbot_test_nginx --domains nginx.wtf run -echo | openssl s_client -connect localhost:5001 \ - | openssl x509 -out $root/nginx.pem -diff -q $root/nginx.pem $root/conf/live/nginx.wtf/cert.pem +test_deployment_and_rollback() { + # Arguments: certname + echo | openssl s_client -connect localhost:5001 \ + | openssl x509 -out $root/nginx.pem + diff -q $root/nginx.pem "$root/conf/live/$1/cert.pem" -certbot_test_nginx rollback --checkpoints 9001 -diff -q <(echo "$original") $nginx_conf + certbot_test_nginx rollback --checkpoints 9001 + diff -q <(echo "$original") $nginx_conf +} + +certbot_test_nginx --domains nginx.wtf run +test_deployment_and_rollback nginx.wtf +certbot_test_nginx --domains nginx2.wtf --preferred-challenges http +test_deployment_and_rollback nginx2.wtf # note: not reached if anything above fails, hence "killall" at the # top diff --git a/certbot/__init__.py b/certbot/__init__.py index cbea701ee..2869d29b0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.21.0.dev0' +__version__ = '0.22.0.dev0' diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..67d36c8cc 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -24,15 +29,13 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` + :ivar aauthzrs: ACME Authorization Resources and their active challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,18 +45,16 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() + self.aauthzrs = [] self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - - def get_authorizations(self, domains, best_effort=False): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param acme.messages.OrderResource orderr: must have + authorizations filled in :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -62,15 +63,15 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + for authzr in orderr.authorizations: + self.aauthzrs.append(AnnotatedAuthzr(authzr, [])) - self._choose_challenges(domains) + self._choose_challenges() config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: + while self._has_challenges(): resp = self._solve_challenges() logger.info("Waiting for verification...") if config.debug_challenges: @@ -84,8 +85,8 @@ class AuthHandler(object): self.verify_authzr_complete() # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in self.aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -93,35 +94,54 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: - path = gen_challenge_path( - self.authzr[dom].body.challenges, - self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + for aauthzr in self.aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges + if self.acme.acme_version == 1: + combinations = aauthzr.authzr.body.combinations + else: + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + path = gen_challenge_path( + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), + combinations) + + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) + + def _has_challenges(self): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in self.aauthzrs) def _solve_challenges(self): """Get Responses for challenges from authenticators.""" resp = [] + all_achalls = self._get_all_achalls() with error_handler.ErrorHandler(self._cleanup_challenges): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp + def _get_all_achalls(self): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in self.aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + def _respond(self, resp, best_effort): """Send/Receive confirmation of all challenges. @@ -130,69 +150,67 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(resp, chall_update) # Check for updated status... try: self._poll_challenges(chall_update, best_effort) finally: - # This removes challenges from self.achalls self._cleanup_challenges(active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, resps, chall_update): """Send responses and make sure errors are handled. :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(self.aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls def _poll_challenges( self, chall_update, best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + self.aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -201,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = self.aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + self.aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -276,14 +296,17 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls() else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in self.aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break def verify_authzr_complete(self): """Verifies that all authorizations have been decided. @@ -292,15 +315,16 @@ class AuthHandler(object): :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in self.aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -314,8 +338,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls @@ -408,7 +433,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -429,15 +454,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index da85ed783..4240a0523 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -121,7 +121,28 @@ def domains_for_certname(config, certname): return lineage.names() if lineage else None def find_duplicative_certs(config, domains): - """Find existing certs that duplicate the request.""" + """Find existing certs that match the given domain names. + + This function searches for certificates whose domains are equal to + the `domains` parameter and certificates whose domains are a subset + of the domains in the `domains` parameter. If multiple certificates + are found whose names are a subset of `domains`, the one whose names + are the largest subset of `domains` is returned. + + If multiple certificates' domains are an exact match or equally + sized subsets, which matching certificates are returned is + undefined. + + :param config: Configuration. + :type config: :class:`certbot.configuration.NamespaceConfig` + :param domains: List of domain names + :type domains: `list` of `str` + + :returns: lineages representing the identically matching cert and the + largest subset if they exist + :rtype: `tuple` of `storage.RenewableCert` or `None` + + """ def update_certs_for_domain_matches(candidate_lineage, rv): """Return cert as identical_names_cert if it matches, or subset_names_cert if it matches as subset diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7e..1c2273c8a 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,13 +207,15 @@ def set_by_cli(var): # propagate plugin requests: eg --standalone modifies config.authenticator detector.authenticator, detector.installer = ( plugin_selection.cli_plugin_requests(detector)) - logger.debug("Default Detector is %r", detector) if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) return True for modifier in VAR_MODIFIERS.get(var, []): if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) return True return False @@ -597,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args @@ -1276,14 +1283,13 @@ def _paths_parser(helpful): elif verb == "revoke": add(section, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, - help=cph, required=(verb == "install")) + add(section, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): section = verb # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", required=(verb == "install"), + add(section, "--key-path", type=((verb == "revoke" and read_file) or os.path.abspath), help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/client.py b/certbot/client.py index b735421f5..81fc0b802 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -11,7 +12,6 @@ import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util -from acme import errors as acme_errors from acme import messages import certbot @@ -37,12 +37,12 @@ from certbot.plugins import selection as plugin_selection logger = logging.getLogger(__name__) -def acme_from_config_key(config, key): +def acme_from_config_key(config, key, regr=None): "Wrangle ACME client construction" # TODO: Allow for other alg types besides RS256 - net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl), + net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl), user_agent=determine_user_agent(config)) - return acme_client.Client(config.server, key=key, net=net) + return acme_client.BackwardsCompatibleClientV2(net, key, config.server) def determine_user_agent(config): @@ -162,14 +162,7 @@ def register(config, account_storage, tos_cb=None): backend=default_backend()))) acme = acme_from_config_key(config, key) # TODO: add phone? - regr = perform_registration(acme, config) - - if regr.terms_of_service is not None: - if tos_cb is not None and not tos_cb(regr): - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") - regr = acme.agree_to_tos(regr) + regr = perform_registration(acme, config, tos_cb) acc = account.Account(regr, key) account.report_new_account(config) @@ -180,7 +173,7 @@ def register(config, account_storage, tos_cb=None): return acc, acme -def perform_registration(acme, config): +def perform_registration(acme, config, tos_cb): """ Actually register new account, trying repeatedly if there are email problems @@ -192,7 +185,8 @@ def perform_registration(acme, config): :rtype: `acme.messages.RegistrationResource` """ try: - return acme.register(messages.NewRegistration.from_data(email=config.email)) + return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), + tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: @@ -202,7 +196,7 @@ def perform_registration(acme, config): raise errors.Error(msg) else: config.email = display_ops.get_email(invalid=True) - return perform_registration(acme, config) + return perform_registration(acme, config, tos_cb) else: raise @@ -218,8 +212,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ @@ -232,7 +226,7 @@ class Client(object): # Initialize ACME if account is provided if acme is None and self.account is not None: - acme = acme_from_config_key(config, self.account.key) + acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme if auth is not None: @@ -241,21 +235,15 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` + :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM strings :rtype: tuple """ @@ -267,37 +255,17 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + if orderr is None: + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr) + orderr = orderr.update(authorizations=authzr) + authzr = orderr.authorizations - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + return crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -306,20 +274,12 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -332,10 +292,26 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + orderr = self.acme.new_order(csr.data) + authzr = self.auth_handler.handle_authorizations(orderr, self.config.allow_subset_of_names) + orderr = orderr.update(authorizations=authzr) + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: + if not self.config.dry_run: + os.remove(key.file) + os.remove(csr.file) + return self.obtain_certificate(successful_domains) + else: + cert, chain = self.obtain_certificate_from_csr(csr, orderr) + + return cert, chain, key, csr # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -354,7 +330,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -369,19 +345,16 @@ class Client(object): return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -398,8 +371,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -410,20 +381,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): @@ -556,11 +522,11 @@ class Client(object): self.installer.rollback_checkpoints() self.installer.restart() except: - # TODO: suggest letshelp-letsencrypt here reporter.add_message( "An error occurred and we failed to restore your config and " - "restart your server. Please submit a bug report to " - "https://github.com/letsencrypt/letsencrypt", + "restart your server. Please post to " + "https://community.letsencrypt.org/c/server-config " + "with details about your configuration and this error you received.", reporter.HIGH_PRIORITY) raise reporter.add_message(success_msg, reporter.HIGH_PRIORITY) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..11721cc10 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -340,14 +339,8 @@ def _get_names_from_cert_or_req(cert_or_req, load_func, typ): def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): - common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access - sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) - - if common_name is None: - return sans - else: - return [common_name] + [d for d in sans if d != common_name] + return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): @@ -373,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -449,3 +433,16 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 32dd69256..7be852e83 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,10 +1,10 @@ """Certbot main entry point.""" +# pylint: disable=too-many-lines from __future__ import print_function import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -495,17 +495,20 @@ def _determine_account(config): if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - def _tos_cb(regr): + def _tos_cb(terms_of_service): if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( - regr.terms_of_service, config.server)) + terms_of_service, config.server)) obj = zope.component.getUtility(interfaces.IDisplay) - return obj.yesno(msg, "Agree", "Cancel", + result = obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos", force_interactive=True) - + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) @@ -779,11 +782,45 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) - domains, _ = _find_domains_or_certname(config, installer) - le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + # If cert-path is defined, populate missing (ie. not overridden) values. + # Unfortunately this can't be done in argument parser, as certificate + # manager needs the access to renewal directory paths + if config.certname: + config = _populate_from_certname(config) + if config.key_path and config.cert_path: + _check_certificate_and_key(config) + domains, _ = _find_domains_or_certname(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + _install_cert(config, le_client, domains) + else: + raise errors.ConfigurationError("Path to certificate or key was not defined. " + "If your certificate is managed by Certbot, please use --cert-name " + "to define which certificate you would like to install.") +def _populate_from_certname(config): + """Helper function for install to populate missing config values from lineage + defined by --cert-name.""" + lineage = cert_manager.lineage_for_certname(config, config.certname) + if not lineage: + return config + if not config.key_path: + config.namespace.key_path = lineage.key_path + if not config.cert_path: + config.namespace.cert_path = lineage.cert_path + if not config.chain_path: + config.namespace.chain_path = lineage.chain_path + if not config.fullchain_path: + config.namespace.fullchain_path = lineage.fullchain_path + return config + +def _check_certificate_and_key(config): + if not os.path.isfile(os.path.realpath(config.cert_path)): + raise errors.ConfigurationError("Error while reading certificate from path " + "{0}".format(config.cert_path)) + if not os.path.isfile(os.path.realpath(config.key_path)): + raise errors.ConfigurationError("Error while reading private key from path " + "{0}".format(config.key_path)) def plugins_cmd(config, plugins): """List server software plugins. @@ -945,11 +982,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) @@ -1026,13 +1063,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): @@ -1218,17 +1255,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 002d2f225..c281534ca 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -315,23 +315,28 @@ class Addr(object): return result -class TLSSNI01(object): - """Abstract base for TLS-SNI-01 challenge performers""" +class ChallengePerformer(object): + """Abstract base for challenge performers. + + :ivar configurator: Authenticator and installer plugin + :ivar achalls: Annotated challenges + :vartype achalls: `list` of `.KeyAuthorizationAnnotatedChallenge` + :ivar indices: Holds the indices of challenges from a larger array + so the user of the class doesn't have to. + :vartype indices: `list` of `int` + + """ def __init__(self, configurator): self.configurator = configurator self.achalls = [] self.indices = [] - self.challenge_conf = os.path.join( - configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") - # self.completed = 0 def add_chall(self, achall, idx=None): - """Add challenge to TLSSNI01 object to perform at once. + """Store challenge to be performed when perform() is called. :param .KeyAuthorizationAnnotatedChallenge achall: Annotated - TLSSNI01 challenge. - + challenge. :param int idx: index to challenge in a larger array """ @@ -339,6 +344,27 @@ class TLSSNI01(object): if idx is not None: self.indices.append(idx) + def perform(self): + """Perform all added challenges. + + :returns: challenge respones + :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` + + + """ + raise NotImplementedError() + + +class TLSSNI01(ChallengePerformer): + # pylint: disable=abstract-method + """Abstract base for TLS-SNI-01 challenge performers""" + + def __init__(self, configurator): + super(TLSSNI01, self).__init__(configurator) + self.challenge_conf = os.path.join( + configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf") + # self.completed = 0 + def get_cert_path(self, achall): """Returns standardized name for challenge certificate. diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 1a1ca7dcb..103a12499 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -18,6 +18,17 @@ from certbot import errors from certbot.tests import acme_util from certbot.tests import util as test_util +AUTH_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) +ACHALLS = [ + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token1'), "pending"), + domain="encryption-example.demo", account_key=AUTH_KEY), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.TLSSNI01(token=b'token2'), "pending"), + domain="certbot.demo", account_key=AUTH_KEY), +] class NamespaceFunctionsTest(unittest.TestCase): """Tests for certbot.plugins.common.*_namespace functions.""" @@ -261,21 +272,27 @@ class AddrTest(unittest.TestCase): self.assertEqual(set_c, set_d) +class ChallengePerformerTest(unittest.TestCase): + """Tests for certbot.plugins.common.ChallengePerformer.""" + + def setUp(self): + configurator = mock.MagicMock() + + from certbot.plugins.common import ChallengePerformer + self.performer = ChallengePerformer(configurator) + + def test_add_chall(self): + self.performer.add_chall(ACHALLS[0], 0) + self.assertEqual(1, len(self.performer.achalls)) + self.assertEqual([0], self.performer.indices) + + def test_perform(self): + self.assertRaises(NotImplementedError, self.performer.perform) + + class TLSSNI01Test(unittest.TestCase): """Tests for certbot.plugins.common.TLSSNI01.""" - auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) - achalls = [ - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token1'), "pending"), - domain="encryption-example.demo", account_key=auth_key), - achallenges.KeyAuthorizationAnnotatedChallenge( - challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token=b'token2'), "pending"), - domain="certbot.demo", account_key=auth_key), - ] - def setUp(self): self.tempdir = tempfile.mkdtemp() configurator = mock.MagicMock() @@ -288,11 +305,6 @@ class TLSSNI01Test(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - def test_add_chall(self): - self.sni.add_chall(self.achalls[0], 0) - self.assertEqual(1, len(self.sni.achalls)) - self.assertEqual([0], self.sni.indices) - def test_setup_challenge_cert(self): # This is a helper function that can be used for handling # open context managers more elegantly. It avoids dealing with @@ -325,7 +337,7 @@ class TLSSNI01Test(unittest.TestCase): OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) def test_get_z_domain(self): - achall = self.achalls[0] + achall = ACHALLS[0] self.assertEqual(self.sni.get_z_domain(achall), achall.response(achall.account_key).z_domain.decode("utf-8")) diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..5a7e07ec0 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..6328b16ef 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() diff --git a/certbot/renewal.py b/certbot/renewal.py index 7d0240f73..ea5d87a5e 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -294,15 +294,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) @@ -425,7 +422,10 @@ def handle_renewal_request(config): main.renew_cert(lineage_config, plugins, renewal_candidate) renew_successes.append(renewal_candidate.fullchain) else: - renew_skipped.append(renewal_candidate.fullchain) + expiry = crypto_util.notAfter(renewal_candidate.version( + "cert", renewal_candidate.latest_common_version())) + renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, + expiry.strftime("%Y-%m-%d"))) except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " diff --git a/certbot/storage.py b/certbot/storage.py index 67d2155ae..ed3922c58 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -417,7 +417,7 @@ class RenewableCert(object): conf_version = self.configuration.get("version") if (conf_version is not None and util.get_strict_version(conf_version) > CURRENT_VERSION): - logger.warning( + logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " "work.", conf_version, config_filename, certbot.__version__) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..b6af3d0f5 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,36 +29,35 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -81,6 +80,7 @@ class GetAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,20 +90,19 @@ class GetAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) + mock_order = mock.MagicMock(authorizations=[authzr]) - mock_poll.side_effect = self._validate_all - - authzr = self.handler.get_authorizations(["0"]) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -113,22 +112,28 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -140,13 +145,43 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][0] + self.assertEqual(list(six.iterkeys(chall_update)), [0]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + + + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) @@ -154,75 +189,105 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_poll.call_count, 1) chall_update = mock_poll.call_args[0][0] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] + mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + for i, aauthzr in enumerate(self.handler.aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + self.handler.aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -231,7 +296,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -239,40 +304,38 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[0], [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( + [messages.STATUS_PENDING] * 2, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) + self.handler.aauthzrs.append(AnnotatedAuthzr(acme_util.gen_authzr( messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), [])) self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.handler.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid self.handler._poll_challenges(self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.handler._poll_challenges(self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.handler.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -286,7 +349,7 @@ class PollChallengesTest(unittest.TestCase): def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..b51275d9e 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -30,31 +26,27 @@ class RegisterTest(test_util.ConfigTestCase): self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" self.account_storage = account.AccountMemoryStorage() - self.tos_cb = mock.MagicMock() def _call(self): from certbot.client import register - return register(self.config, self.account_storage, self.tos_cb) + tos_cb = mock.MagicMock() + return register(self.config, self.account_storage, tos_cb) def test_no_tos(self): - with mock.patch("certbot.client.acme_client.Client") as mock_client: - mock_client.register().terms_of_service = "http://tos" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client.new_account_and_tos().terms_of_service = "http://tos" with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): - self.tos_cb.return_value = False + mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) self.assertFalse(mock_handle.called) - self.tos_cb.return_value = True + mock_client().new_account_and_tos.side_effect = None self._call() self.assertTrue(mock_handle.called) - self.tos_cb = None - self._call() - self.assertEqual(mock_handle.call_count, 2) - def test_it(self): - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -66,9 +58,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = False msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) self.assertTrue(mock_handle.called) @@ -79,9 +71,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = True msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription"): - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) def test_needs_email(self): @@ -91,7 +83,7 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -104,9 +96,9 @@ class RegisterTest(test_util.ConfigTestCase): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) self.assertFalse(mock_handle.called) @@ -122,7 +114,7 @@ class ClientTestCommon(test_util.ConfigTestCase): self.account = mock.MagicMock(**{"key.pem": KEY}) from certbot.client import Client - with mock.patch("certbot.client.acme_client.Client") as acme: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as acme: self.acme_client = acme self.acme = acme.return_value = mock.MagicMock() self.client = Client( @@ -138,106 +130,77 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock( + authorizations=[None], + fullchain_pem=mock.sentinel.fullchain_pem, + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): - net = self.acme_client.call_args[1]["net"] + net = self.acme_client.call_args[0][0] self.assertTrue(net.verify_ssl) def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.client.auth_handler.handle_authorizations.return_value = [None] + self.acme.finalize_order.return_value = self.eg_order + self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.get_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) - authzr = auth_handler.get_authorizations(self.eg_domains, False) + orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None + # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order) # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler - - authzr = auth_handler.get_authorizations(self.eg_domains, False) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler - - authzr = auth_handler.get_authorizations(self.eg_domains, False) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -245,6 +208,27 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + mock.sentinel.fullchain_pem) + + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + mock_crypto_util.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) + + authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -253,6 +237,8 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + mock_crypto.cert_and_chain_from_fullchain.return_value = (mock.sentinel.cert, + mock.sentinel.chain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -262,38 +248,42 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. - + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr - self.client.auth_handler.get_authorizations.return_value = authzr + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) + + self.eg_order.authorizations = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): domains = ["example.com", "www.example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), @@ -309,7 +299,6 @@ class ClientTest(ClientTestCommon): self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -318,9 +307,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -330,7 +318,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..00303fab3 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT + chain_pem = CERT + SS_CERT + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f..b778f05ea 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -225,7 +226,7 @@ class RevokeTest(test_util.TempDirTestCase): 'cert_512.pem')) self.patches = [ - mock.patch('acme.client.Client', autospec=True), + mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot.client.Client'), mock.patch('certbot.main._determine_account'), mock.patch('certbot.main.display_ops.success_revocation') @@ -267,7 +268,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_with_reason(self, mock_acme_client, mock_delete_if_appropriate): mock_delete_if_appropriate.return_value = False - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): self._call("--reason " + reason) @@ -592,11 +593,30 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met super(MainTest, self).tearDown() - def _call(self, args, stdout=None): - "Run the cli with output streams and actual client mocked out" - with mock.patch('certbot.main.client') as client: - ret, stdout, stderr = self._call_no_clientmock(args, stdout) - return ret, stdout, stderr, client + def _call(self, args, stdout=None, mockisfile=False): + """Run the cli with output streams, actual client and optionally + os.path.isfile() mocked out""" + + if mockisfile: + orig_open = os.path.isfile + def mock_isfile(fn, *args, **kwargs): + """Mock os.path.isfile()""" + if (fn.endswith("cert") or + fn.endswith("chain") or + fn.endswith("privkey")): + return True + else: + return orig_open(fn, *args, **kwargs) + + with mock.patch("os.path.isfile") as mock_if: + mock_if.side_effect = mock_isfile + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client + else: + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client def _call_no_clientmock(self, args, stdout=None): "Run the client with output streams mocked out" @@ -642,10 +662,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - with mock.patch('certbot.main._get_and_save_cert'): - with mock.patch('certbot.main.client.acme_from_config_key'): - args.extend(['--email', 'io@io.is']) - self._cli_missing_flag(args, "--agree-tos") @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @@ -674,15 +690,75 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met ua = "bandersnatch" args += ["--user-agent", ua] self._call_no_clientmock(args) - acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua) + acme_net.assert_called_once_with(mock.ANY, account=mock.ANY, verify_ssl=True, + user_agent=ua) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_selection(self, mock_pick_installer, _rec): self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert', - '--key-path', 'key', '--chain-path', 'chain']) + '--key-path', 'privkey', '--chain-path', 'chain'], mockisfile=True) self.assertEqual(mock_pick_installer.call_count, 1) + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_certname(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_override(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever', + '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.chain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + + mock_install.reset() + + self._call(['install', '--cert-name', 'whatever', + '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_error(self, _inst, _rec): + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--key-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-name', 'notfound', + '--key-path', 'invalid']) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -864,11 +940,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, @@ -951,7 +1022,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False): + quiet_mode=False, expiry_date=datetime.datetime.now()): # pylint: disable=too-many-locals,too-many-arguments cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -984,7 +1055,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_latest = mock.MagicMock() mock_latest.get_issuer.return_value = "Fake fake" mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('certbot.main.renewal.crypto_util'): + with mock.patch('certbot.main.renewal.crypto_util') as mock_crypto_util: + mock_crypto_util.notAfter.return_value = expiry_date if not args: args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: @@ -1052,6 +1124,16 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + @mock.patch('certbot.renewal.should_renew') + def test_renew_skips_recent_certs(self, should_renew): + should_renew.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + expiry = datetime.datetime.now() + datetime.timedelta(days=90) + _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, + args=['renew'], expiry_date=expiry) + self.assertTrue('No renewals were attempted.' in stdout.getvalue()) + self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] @@ -1146,7 +1228,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) @@ -1263,11 +1345,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['--cert-path', SS_CERT_PATH, '--key-path', RSA2048_KEY_PATH, '--server', server, 'revoke']) with open(RSA2048_KEY_PATH, 'rb') as f: - mock_acme_client.Client.assert_called_once_with( - server, key=jose.JWK.load(f.read()), net=mock.ANY) + mock_acme_client.BackwardsCompatibleClientV2.assert_called_once_with( + mock.ANY, jose.JWK.load(f.read()), server) with open(SS_CERT_PATH, 'rb') as f: cert = crypto_util.pyopenssl_load_certificate(f.read())[0] - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke mock_revoke.assert_called_once_with( jose.ComparableX509(cert), mock.ANY) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c8f775e2..6c0970e72 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -158,8 +158,8 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertTrue(mock_logger.warning.called) - self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + self.assertTrue(mock_logger.info.called) + self.assertTrue("version" in mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=too-many-statements,protected-access diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( @@ -340,7 +335,7 @@ class ConfigTestCase(TempDirTestCase): self.config.cert_path = constants.CLI_DEFAULTS['auth_cert_path'] self.config.fullchain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] - self.config.server = "example.com" + self.config.server = "https://example.com" def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) @@ -552,16 +548,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): @@ -615,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. diff --git a/docker-compose.yml b/docker-compose.yml index 00d3d4c72..75a5b9aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: context: . dockerfile: Dockerfile-dev ports: + - "80:80" - "443:443" volumes: - .:/opt/certbot/src diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 000000000..8fd0f127d --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,52 @@ + diff --git a/docs/cli-help.txt b/docs/cli-help.txt index abaa95b9b..abebdb9c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.20.0 (certbot; - Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + "". (default: CertbotACMEClient/0.21.1 (certbot; + darwin 10.13.3) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -331,6 +331,14 @@ revoke: --reason {unspecified,keycompromise,affiliationchanged,superseded,cessationofoperation} Specify reason for revoking certificate. (default: unspecified) + --delete-after-revoke + Delete certificates after revoking them. (default: + None) + --no-delete-after-revoke + Do not delete certificates after revoking them. This + option should be used with caution because the 'renew' + subcommand will attempt to renew undeleted revoked + certificates. (default: None) register: Options for account registration & modification @@ -440,11 +448,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary. (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: - a2dismod) + Path to the Apache 'a2dismod' binary. (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -458,13 +464,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: True) + you.(Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you.(Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/docs/conf.py b/docs/conf.py index 73df47dbd..09bb44285 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,8 @@ master_doc = 'index' # General information about the project. project = u'Certbot' -copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' +# this is now overridden by the footer.html template +#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/contributing.rst b/docs/contributing.rst index c144a4f74..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -32,10 +32,13 @@ a new plugin is introduced. .. code-block:: shell cd certbot - ./certbot-auto --os-packages-only + sudo ./certbot-auto --os-packages-only ./tools/venv.sh -Then in each shell where you're working on the client, do: +You can now run the copy of Certbot from git either by executing +``venv/bin/certbot``, or by activating the virtual environment. If you're +actively modifying and testing the code, you may want to run commands like this in +each shell where you're working: .. code-block:: shell @@ -43,7 +46,8 @@ Then in each shell where you're working on the client, do: export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh -After that, your shell will be using the virtual environment, and you run the +After that, your shell will be using the virtual environment, your copy of +Certbot will default to requesting test (staging) certificates, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that includes several flags useful for testing. For instance, it sets various output directories to point to /tmp/, and uses non-privileged ports for challenges, so @@ -418,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index a914586ff..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver @@ -116,7 +116,7 @@ certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a cert for resolves +from the server that the domain you're requesting a certficate for resolves to, `install Docker`_, then issue the following command: .. code-block:: shell diff --git a/docs/using.rst b/docs/using.rst index ab4670052..e8f84e2d7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -557,8 +557,8 @@ apologize for any inconvenience you encounter in integrating these commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. - This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. - If you write a custom script and expect to run a command only after a cert was actually renewed + This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. + If you write a custom script and expect to run a command only after a certificate was actually renewed you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. diff --git a/letsencrypt-auto b/letsencrypt-auto index 444bee1b9..d3a5c23e5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -68,10 +68,12 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive|renew) - ASSUME_YES=1;; + --noninteractive|--non-interactive) + NONINTERACTIVE=1;; --quiet) QUIET=1;; + renew) + ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -93,7 +95,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - ASSUME_YES=1 + NONINTERACTIVE=1 HELP=0 fi @@ -244,23 +246,42 @@ DeprecationBootstrap() { fi } - +MIN_PYTHON_VERSION="2.6" +MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version DeterminePythonVersion() { - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - if [ "$?" != "0" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 + # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python + # + # If no Python is found, PYVER is set to 0. + if [ "$USE_PYTHON_3" = 1 ]; then + for LE_PYTHON in "$LE_PYTHON" python3; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + else + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + fi + if [ "$?" != "0" ]; then + if [ "$1" != "NOCRASH" ]; then + error "Cannot find any Pythons; please install one!" + exit 1 + else + PYVER=0 + return 0 + fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ "$PYVER" -lt 26 ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version 2.6." - exit 1 + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + if [ "$1" != "NOCRASH" ]; then + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." + exit 1 + fi fi } @@ -384,23 +405,19 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 (EPEL must be installed manually) +# If new packages are installed by BootstrapRpmCommonBase below, version +# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# Sets TOOL to the name of the package manager +# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. +# Enables EPEL if applicable and possible. +InitializeRPMCommonBase() { if type dnf 2>/dev/null then - tool=dnf + TOOL=dnf elif type yum 2>/dev/null then - tool=yum + TOOL=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -408,15 +425,15 @@ BootstrapRpmCommon() { fi if [ "$ASSUME_YES" = 1 ]; then - yes_flag="-y" + YES_FLAG="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $tool list *virtualenv >/dev/null 2>&1; then + if ! $TOOL list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $tool list epel-release >/dev/null 2>&1; then + if ! $TOOL list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -425,14 +442,20 @@ BootstrapRpmCommon() { sleep 1s /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..." sleep 1s fi - if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -444,10 +467,39 @@ BootstrapRpmCommon() { ca-certificates " - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $tool list python >/dev/null 2>&1; then + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then pkgs="$pkgs - python + mod_ssl + " + fi + + if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 + + InitializeRPMCommonBase + + # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. + if $TOOL list python >/dev/null 2>&1; then + python_pkgs="$python python-devel python-virtualenv python-tools @@ -455,9 +507,8 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $tool list python2 >/dev/null 2>&1; then - pkgs="$pkgs - python2 + elif $TOOL list python2 >/dev/null 2>&1; then + python_pkgs="$python2 python2-libs python2-setuptools python2-devel @@ -468,8 +519,7 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - pkgs="$pkgs - python27 + python_pkgs="$python27 python27-devel python27-virtualenv python27-tools @@ -477,16 +527,31 @@ BootstrapRpmCommon() { " fi - if $tool list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi + BootstrapRpmCommonBase "$python_pkgs" +} - if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" +# If new packages are installed by BootstrapRpmPython3 below, this version +# number must be increased. +BOOTSTRAP_RPM_PYTHON3_VERSION=1 + +BootstrapRpmPython3() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + # EPEL uses python34 + if $TOOL list python34 >/dev/null 2>&1; then + python_pkgs="python34 + python34-devel + python34-tools + " + else + error "No supported Python package available to install. Aborting bootstrap!" exit 1 fi + + BootstrapRpmCommonBase "$python_pkgs" } # If new packages are installed by BootstrapSuseCommon below, this version @@ -696,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -715,11 +775,27 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + # Run DeterminePythonVersion to decide on the basis of available Python versions + # whether to use 2.x or 3.x on RedHat-like systems. + # Then, revert LE_PYTHON to its previous state. + prev_le_python="$LE_PYTHON" + unset LE_PYTHON + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapMessage "RedHat-based OSes that will use Python3" + BootstrapRpmPython3 + } + USE_PYTHON_3=1 + BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" + else + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" + fi + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -782,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -816,7 +903,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -824,14 +915,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -841,6 +944,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -858,10 +965,18 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + if [ "$PYVER" -le 27 ]; then + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + if [ "$VERBOSE" = 1 ]; then + "$LE_PYTHON" -m venv "$VENV_PATH" + else + "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null + fi fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -983,9 +1098,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1184,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1078,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1319,9 +1437,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 @@ -1353,17 +1472,22 @@ On failure, return non-zero. """ -from __future__ import print_function +from __future__ import print_function, unicode_literals from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re +import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -from urllib2 import build_opener, HTTPHandler, HTTPSHandler -from urllib2 import HTTPError, URLError +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler + from urllib2 import HTTPError, URLError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1385,8 +1509,11 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9. - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. + if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): + self._opener = build_opener(HTTPSHandler(context=cert_none_context())) + else: + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1408,7 +1535,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'w') as file: + with open(join(dir, filename), 'wb') as file: file.write(contents) @@ -1416,13 +1543,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json'))) + 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in metadata['releases'].iterkeys() + in metadata['releases'].keys() if re.match('^[0-9.]+$', r))) @@ -1439,7 +1566,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY, temp_dir, 'public_key.pem') + write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1454,6 +1581,14 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) +def cert_none_context(): + """Create a SSLContext object to not check hostname.""" + # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + return context + + def main(): get = HttpsGetter().get flag = argv[1] @@ -1475,8 +1610,10 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- - DeterminePythonVersion - if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + DeterminePythonVersion "NOCRASH" + if [ "$PYVER" -lt "$MIN_PYVER" ]; then + error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." + elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index eeab78cd6..f28fd9893 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt -XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf -p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX -OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4 -X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI -/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE= -=Is56 +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X +dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 +JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc +a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 +0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR +zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR +F/j/631OC6yLRasr+xcL1gc+SCryfA== +=tkZT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 712ef6813..9ff1c1386 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0.dev0" +LE_AUTO_VERSION="0.22.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -274,7 +277,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -762,13 +764,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -801,7 +798,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -864,6 +861,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -898,7 +906,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -906,14 +918,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -923,6 +947,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -1171,18 +1199,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1212,6 +1240,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1245,33 +1274,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1292,12 +1320,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1307,8 +1336,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1321,6 +1351,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1328,17 +1376,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: @@ -1403,6 +1458,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else @@ -1412,9 +1473,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index e276aae53..8dd683775 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index b06ac9c80..9d0f27009 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -274,7 +277,6 @@ DeterminePythonVersion() { return 0 fi fi - export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt "$MIN_PYVER" ]; then @@ -301,13 +303,8 @@ DeterminePythonVersion() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -340,7 +337,7 @@ elif [ -f /etc/redhat-release ]; then } BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" fi - export LE_PYTHON="$prev_le_python" + LE_PYTHON="$prev_le_python" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -403,6 +400,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -437,7 +445,11 @@ TempDir() { mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS } - +# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, +# returns a non-zero number. +OldVenvExists() { + [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] +} if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -445,14 +457,26 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg SetPrevBootstrapVersion + if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then + unset LE_PYTHON + fi + INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ]; then + if [ -d "$VENV_PATH" ] || OldVenvExists; then # If the selected Bootstrap function isn't a noop and it differs from the # previously used version if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then # if non-interactive mode or stdin and stdout are connected to a terminal if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - rm -rf "$VENV_PATH" + if [ -d "$VENV_PATH" ]; then + rm -rf "$VENV_PATH" + fi + # In the case the old venv was just a symlink to the new one, + # OldVenvExists is now false because we deleted the venv at VENV_PATH. + if OldVenvExists; then + rm -rf "$OLD_VENV_PATH" + ln -s "$VENV_PATH" "$OLD_VENV_PATH" + fi RerunWithArgs "$@" else error "Skipping upgrade because new OS dependencies may need to be installed." @@ -462,6 +486,10 @@ if [ "$1" = "--le-auto-phase2" ]; then error "install any required packages." # Set INSTALLED_VERSION to be the same so we don't update the venv INSTALLED_VERSION="$LE_AUTO_VERSION" + # Continue to use OLD_VENV_PATH if the new venv doesn't exist + if [ ! -d "$VENV_PATH" ]; then + VENV_BIN="$OLD_VENV_PATH/bin" + fi fi elif [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings @@ -562,6 +590,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else @@ -571,9 +605,10 @@ else # upgrading. Phase 1 checks the version of the latest release of # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. + export PHASE_1_VERSION="$LE_AUTO_VERSION" if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then + if ! OldVenvExists; then if [ "$HELP" = 1 ]; then echo "$USAGE" exit 0 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 74336de7b..0a5994afc 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.20.0 \ - --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ - --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 -acme==0.20.0 \ - --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ - --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 -certbot-apache==0.20.0 \ - --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ - --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f -certbot-nginx==0.20.0 \ - --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ - --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..ed55b37e9 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,33 +57,32 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 0 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), +# Pip has no dependencies, as it vendors everything: +PIP_PACKAGE = [ + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), + '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d')] + + +OTHER_PACKAGES = maybe_argparse + [ # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +103,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +119,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +134,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,17 +159,24 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # We download and install pip first, then the rest, to avoid the bug + # https://github.com/certbot/certbot/issues/4938. + pip_downloads, other_downloads = [ + [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in packages] + for packages in (PIP_PACKAGE, OTHER_PACKAGES)] + for downloads in (pip_downloads, other_downloads): + check_output('pip install --no-index --no-deps -U ' + + # Disable cache since we're not using it and it + # otherwise sometimes throws permission warnings: + ('--no-cache-dir ' if has_pip_cache else '') + + ' '.join(quote(d) for d in downloads), + shell=True) except HashError as exc: print(exc) except Exception: diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index a0e96edf8..2c6dcf734 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -69,5 +69,13 @@ fi echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." echo "" +export VENV_PATH=$(mktemp -d) +"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 +if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then + echo "Python 3 wasn't used with --no-bootstrap!" + exit 1 +fi +unset VENV_PATH + # test using python3 pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', @@ -31,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/setup.py b/setup.py index ce505a62e..9ac1a7ee7 100644 --- a/setup.py +++ b/setup.py @@ -41,24 +41,16 @@ install_requires = [ 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2', # load_pem_x509_certificate + 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -89,6 +81,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', @@ -98,10 +91,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..f2b0dcf60 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -251,7 +251,7 @@ openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/csr/key.pem" + --key-path "${root}/key.pem" CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` @@ -430,6 +430,12 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; @@ -437,4 +443,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 63 -m diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..236090a14 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,6 +16,11 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*," diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index d9491939c..17740cde8 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -38,6 +38,7 @@ from multiprocessing import Manager import urllib2 import yaml import boto3 +from botocore.exceptions import ClientError import fabric from fabric.api import run, execute, local, env, sudo, cd, lcd from fabric.operations import get, put @@ -141,7 +142,7 @@ def make_instance(instance_name, # give instance a name try: new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) - except botocore.exceptions.ClientError as e: + except ClientError as e: if "InvalidInstanceID.NotFound" in str(e): # This seems to be ephemeral... retry time.sleep(1) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index a83cbd826..355fead2e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -64,10 +64,56 @@ iQIDAQAB -----END PUBLIC KEY----- " -if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then +if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + RUN_PYTHON3_TESTS=1 + if command -v python3; then + echo "Didn't expect Python 3 to be installed!" + exit 1 + fi + cp letsencrypt-auto cb-auto + if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then + echo "Certbot shouldn't have updated to a new version!" + exit 1 + fi + if [ -d "/opt/eff.org" ]; then + echo "New directory shouldn't have been created!" + exit 1 + fi + # Create a 2nd venv at the new path to ensure we properly handle this case + export VENV_PATH="/opt/eff.org/certbot/venv" + if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then + echo second installation appeared to fail + exit 1 + fi + unset VENV_PATH +fi + +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi + +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then echo upgrade appeared to fail exit 1 fi + +if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then + echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ + exit 1 +fi + +if [ "$RUN_PYTHON3_TESTS" = 1 ]; then + if ! command -v python3; then + echo "Python3 wasn't properly installed" + exit 1 + fi + if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then + echo "Python3 wasn't used in venv!" + exit 1 + fi +fi echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 4bed2dd3a..e6ab836b8 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,11 +1,13 @@ #!/bin/sh -xe +LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" VENV_NAME=venv # *-auto respects VENV_PATH -letsencrypt/certbot-auto --debug --os-packages-only --non-interactive -LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version +$LE_AUTO --os-packages-only +LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate # change to an empty directory to ensure CWD doesn't affect tests diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index afc362ff8..47440241d 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -21,7 +21,7 @@ futures==3.1.1 google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 -ipdb==0.10.3 +ipdb==0.10.2 ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tox.ini b/tox.ini index 20f5cda32..971aa7631 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,22 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +all_packages = acme[dev] \ .[dev] \ certbot-apache \ certbot-dns-cloudflare \ + certbot-dns-cloudxns \ certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ certbot-dns-rfc2136 \ certbot-dns-route53 \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +51,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -104,7 +84,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will