diff --git a/.gitignore b/.gitignore index a01d2e1c7..4dff20caf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ tests/letstest/venv/ # pytest cache .cache + +# docker files +.docker diff --git a/.travis.yml b/.travis.yml index 35666d8e6..9ec2f724b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,11 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27_install BOULDER_INTEGRATION=v1 + sudo: required + services: docker + - python: "2.7" + env: TOXENV=py27_install BOULDER_INTEGRATION=v2 sudo: required services: docker - python: "2.7" @@ -25,16 +29,12 @@ matrix: addons: - python: "2.7" env: TOXENV=lint - - python: "2.6" - env: TOXENV=py26 - sudo: required - services: docker - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest' sudo: required services: docker - - python: "3.3" - env: TOXENV=py33 + - python: "3.4" + env: TOXENV=py34 sudo: required services: docker - python: "3.6" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4acfc0401..1369b0907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,69 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.21.1 - 2018-01-25 + +### Fixed + +* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host + header of the request is set to an expected value before redirecting users to + the domain found in the header. The previous way Certbot configured Nginx + redirects was a potential security issue which you can read more about at + https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493. +* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges + if basic authentication is configured for the domain you request a + certificate for. +* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6 + based systems rather than Python 2.6. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/49?closed=1 + +## 0.21.0 - 2018-01-17 + +### Added + +* Support for the HTTP-01 challenge type was added to our Apache and Nginx + plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge + type which was what was previously being used by our Apache and Nginx plugins + last week due to a security issue. For more information about Let's Encrypt's + change, click + [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188). + Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no + changes need to be made to your Certbot configuration, however, you should + make sure your server is accessible on port 80 and isn't behind an external + proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to + HTTPS redirects inside Apache and Nginx are fine. +* IPv6 support was added to the Nginx plugin. +* Support for automatically creating server blocks based on the default server + block was added to the Nginx plugin. +* The flags --delete-after-revoke and --no-delete-after-revoke were added + allowing users to control whether the revoke subcommand also deletes the + certificates it is revoking. + +### Changed + +* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME + library. Support for these versions of Python will be removed in the next + major release of Certbot. If you are using certbot-auto on a RHEL 6 based + system, it will guide you through the process of installing Python 3. +* We split our implementation of JOSE (Javascript Object Signing and + Encryption) out of our ACME library and into a separate package named josepy. + This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and + on [GitHub](https://github.com/certbot/josepy). +* We updated the ciphersuites used in Apache to the new [values recommended by + Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29). + The major change here is adding ChaCha20 to the list of supported + ciphersuites. + +### Fixed + +* An issue with our Apache plugin on Gentoo due to differences in their + apache2ctl command have been resolved. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/47?closed=1 + ## 0.20.0 - 2017-12-06 ### Added diff --git a/Dockerfile-dev b/Dockerfile-dev index 581b58f11..9e35ebec8 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,70 +1,21 @@ # This Dockerfile builds an image for development. -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington -MAINTAINER Yan +FROM ubuntu:xenial -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt +# Note: this only exposes the port to other docker containers. +EXPOSE 80 443 WORKDIR /opt/certbot/src -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - # TODO: Install Apache/Nginx for plugin development. -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev git -y && \ +COPY . . +RUN apt-get update && \ + apt-get install apache2 git nginx-light -y && \ + letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ /var/tmp/* -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - -COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ - -# all above files are necessary for setup.py, however, package source -# code directory has to be copied separately to a subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ -COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/ -COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ -COPY tests /opt/certbot/src/tests/ - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ - /opt/certbot/venv/bin/pip install -U pip && \ - /opt/certbot/venv/bin/pip install -U setuptools && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx \ - -e /opt/certbot/src/letshelp-certbot \ - -e /opt/certbot/src/certbot-compatibility-test \ - -e /opt/certbot/src[dev,docs] - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +RUN VENV_NAME="../venv" tools/venv.sh ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 55780c0ac..03917f8ca 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,9 @@ +If you're having trouble using Certbot and aren't sure you've found a bug or +request for a new feature, please first try asking for help at +https://community.letsencrypt.org/. There is a much larger community there of +people familiar with the project who will be able to more quickly answer your +questions. + ## My operating system is (include version): diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index 5850fa955..e8a0b16a8 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,13 +10,3 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ -import sys -import warnings - -for (major, minor) in [(2, 6), (3, 3)]: - if sys.version_info[:2] == (major, minor): - warnings.warn( - "Python {0}.{1} support will be dropped in the next release of " - "acme. Please upgrade your Python version.".format(major, minor), - DeprecationWarning, - ) #pragma: no cover diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..9e2478afe 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -16,6 +16,7 @@ import re import requests import sys +from acme import crypto_util from acme import errors from acme import jws from acme import messages @@ -39,39 +40,24 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. + :ivar int acme_version: ACME protocol version. 1 or 2. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net, acme_version): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + :param int acme_version: ACME protocol version. 1 or 2. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net + self.acme_version = acme_version @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,28 +69,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): - response = self.net.post(regr.uri, body) + response = self._post(regr.uri, body) # TODO: Boulder returns httplib.ACCEPTED #assert response.status_code == httplib.OK @@ -116,6 +82,13 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, uri=regr.uri, terms_of_service=regr.terms_of_service) + def _post(self, *args, **kwargs): + """Wrapper around self.net.post that adds the acme_version. + + """ + kwargs.setdefault('acme_version', self.acme_version) + return self.net.post(*args, **kwargs) + def update_registration(self, regr, update=None): """Update registration. @@ -130,6 +103,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) + self.net.account = updated_regr return updated_regr def deactivate_registration(self, regr): @@ -153,65 +127,14 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - - def _authzr_from_response(self, response, identifier, uri=None): + def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), uri=response.headers.get('Location', uri)) - if authzr.body.identifier != identifier: + if identifier is not None and authzr.body.identifier != identifier: raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri=None): - """Request challenges. - - :param .messages.Identifier identifier: Identifier to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - if new_authzr_uri is not None: - logger.debug("request_challenges with new_authzr_uri deprecated.") - new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(self.directory.new_authz, new_authz) - # TODO: handle errors - assert response.status_code == http_client.CREATED - return self._authzr_from_response(response, identifier) - - def request_domain_challenges(self, domain, new_authzr_uri=None): - """Request challenges for domain names. - - This is simply a convenience function that wraps around - `request_challenges`, but works with domain names instead of - generic identifiers. See ``request_challenges`` for more - documentation. - - :param str domain: Domain name to be challenged. - :param str new_authzr_uri: Deprecated. Do not use. - - :returns: Authorization Resource. - :rtype: `.AuthorizationResource` - - """ - return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) - def answer_challenge(self, challb, response): """Answer challenge. @@ -227,7 +150,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self.net.post(challb.uri, response) + response = self._post(challb.uri, response) try: authzr_uri = response.links['up']['url'] except KeyError: @@ -288,6 +211,141 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response + def _revoke(self, cert, rsn, url): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :param str url: ACME URL to post to + + :raises .ClientError: If revocation is unsuccessful. + + """ + response = self._post(url, + messages.Revocation( + certificate=cert, + reason=rsn)) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') + +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `josepy.JWK` (private) + :ivar alg: `josepy.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, + net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + directory = messages.Directory.from_json( + self.net.get(directory).json()) + super(Client, self).__init__(directory=directory, + net=net, acme_version=1) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self._post(self.directory[new_reg], new_reg) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + :raises errors.WildcardUnsupportedError: if a wildcard is requested + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + + if identifier.value.startswith("*"): + raise errors.WildcardUnsupportedError( + "Requesting an authorization for a wildcard name is" + " forbidden by this version of the ACME protocol.") + + new_authz = messages.NewAuthorization(identifier=identifier) + response = self._post(self.directory.new_authz, new_authz) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + :raises errors.WildcardUnsupportedError: if a wildcard is requested + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) + def request_issuance(self, csr, authzrs): """Request issuance. @@ -307,7 +365,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes req = messages.CertificateRequest(csr=csr) content_type = DER_CONTENT_TYPE # TODO: add 'cert_type 'argument - response = self.net.post( + response = self._post( self.directory.new_cert, req, content_type=content_type, @@ -492,26 +550,317 @@ class Client(object): # pylint: disable=too-many-instance-attributes :raises .ClientError: If revocation is unsuccessful. """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + return self._revoke(cert, rsn, self.directory[messages.Revocation]) + + +class ClientV2(ClientBase): + """ACME client for a v2 API. + + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. + """ + + def __init__(self, directory, net): + """Initialize. + + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, + net=net, acme_version=2) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + """ + response = self._post(self.directory['newAccount'], new_account) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + regr = self._regr_from_response(response) + self.net.account = regr + return regr + + def new_order(self, csr_pem): + """Request a new Order object from the server. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + """ + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + + identifiers = [] + for name in dnsNames: + identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN, + value=name)) + order = messages.NewOrder(identifiers=identifiers) + response = self._post(self.directory['newOrder'], order) + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url), uri=url)) + return messages.OrderResource( + body=body, + uri=response.headers.get('Location'), + authorizations=authorizations, + csr_pem=csr_pem) + + def poll_and_finalize(self, orderr, deadline=None): + """Poll authorizations and finalize the order. + + If no deadline is provided, this method will timeout after 90 + seconds. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if deadline is None: + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.poll_authorizations(orderr, deadline) + return self.finalize_order(orderr, deadline) + + def poll_authorizations(self, orderr, deadline): + """Poll Order Resource for status.""" + responses = [] + for url in orderr.body.authorizations: + while datetime.datetime.now() < deadline: + authzr = self._authzr_from_response(self.net.get(url), uri=url) + if authzr.body.status != messages.STATUS_PENDING: + responses.append(authzr) + break + time.sleep(1) + # If we didn't get a response for every authorization, we fell through + # the bottom of the loop due to hitting the deadline. + if len(responses) < len(orderr.body.authorizations): + raise errors.TimeoutError() + failed = [] + for authzr in responses: + if authzr.body.status != messages.STATUS_VALID: + for chall in authzr.body.challenges: + if chall.error != None: + failed.append(authzr) + if len(failed) > 0: + raise errors.ValidationError(failed) + return orderr.update(authorizations=responses) + + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem) + wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr)) + self._post(orderr.body.finalize, wrapped_csr) + while datetime.datetime.now() < deadline: + time.sleep(1) + response = self.net.get(orderr.uri) + body = messages.Order.from_json(response.json()) + if body.error is not None: + raise errors.IssuanceError(body.error) + if body.certificate is not None: + certificate_response = self.net.get(body.certificate, + content_type=DER_CONTENT_TYPE).text + return orderr.update(body=body, fullchain_pem=certificate_response) + raise errors.TimeoutError() + + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self._revoke(cert, rsn, self.directory['revokeCert']) + + +class BackwardsCompatibleClientV2(object): + """ACME client wrapper that tends towards V2-style calls, but + supports V1 servers. + + .. note:: While this class handles the majority of the differences + between versions of the ACME protocol, if you need to support an + ACME server based on version 3 or older of the IETF ACME draft + that uses combinations in authorizations (or lack thereof) to + signal that the client needs to complete something other than + any single challenge in the authorization to make it valid, the + user of this class needs to understand and handle these + differences themselves. This does not apply to either of Let's + Encrypt's endpoints where successfully completing any challenge + in an authorization will make it valid. + + :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint + :ivar .ClientBase client: either Client or ClientV2 + """ + + def __init__(self, net, key, server): + directory = messages.Directory.from_json(net.get(server).json()) + self.acme_version = self._acme_version_from_directory(directory) + if self.acme_version == 1: + self.client = Client(directory, key=key, net=net) + else: + self.client = ClientV2(directory, net=net) + + def __getattr__(self, name): + if name in vars(self.client): + return getattr(self.client, name) + elif name in dir(ClientBase): + return getattr(self.client, name) + else: + raise AttributeError() + + def new_account_and_tos(self, regr, check_tos_cb=None): + """Combined register and agree_tos for V1, new_account for V2 + + :param .NewRegistration regr: + :param callable check_tos_cb: callback that raises an error if + the check does not work + """ + def _assess_tos(tos): + if check_tos_cb is not None: + check_tos_cb(tos) + if self.acme_version == 1: + regr = self.client.register(regr) + if regr.terms_of_service is not None: + _assess_tos(regr.terms_of_service) + return self.client.agree_to_tos(regr) + return regr + else: + if "terms_of_service" in self.client.directory.meta: + _assess_tos(self.client.directory.meta.terms_of_service) + regr = regr.update(terms_of_service_agreed=True) + return self.client.new_account(regr) + + def new_order(self, csr_pem): + """Request a new Order object from the server. + + If using ACMEv1, returns a dummy OrderResource with only + the authorizations field filled in. + + :param str csr_pem: A CSR in PEM format. + + :returns: The newly created order. + :rtype: OrderResource + + :raises errors.WildcardUnsupportedError: if a wildcard domain is + requested but unsupported by the ACME version + + """ + if self.acme_version == 1: + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # pylint: disable=protected-access + dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) + authorizations = [] + for domain in dnsNames: + authorizations.append(self.client.request_domain_challenges(domain)) + return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) + else: + return self.client.new_order(csr_pem) + + def finalize_order(self, orderr, deadline): + """Finalize an order and obtain a certificate. + + :param messages.OrderResource orderr: order to finalize + :param datetime.datetime deadline: when to stop polling and timeout + + :returns: finalized order + :rtype: messages.OrderResource + + """ + if self.acme_version == 1: + csr_pem = orderr.csr_pem + certr = self.client.request_issuance( + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), + orderr.authorizations) + + chain = None + while datetime.datetime.now() < deadline: + try: + chain = self.client.fetch_chain(certr) + break + except errors.Error: + time.sleep(1) + + if chain is None: + raise errors.TimeoutError( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + cert = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode() + chain = crypto_util.dump_pyopenssl_chain(chain).decode() + + return orderr.update(fullchain_pem=(cert + chain)) + else: + return self.client.finalize_order(orderr, deadline) + + def revoke(self, cert, rsn): + """Revoke certificate. + + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` + + :param int rsn: Reason code for certificate revocation. + + :raises .ClientError: If revocation is unsuccessful. + + """ + return self.client.revoke(cert, rsn) + + def _acme_version_from_directory(self, directory): + if hasattr(directory, 'newNonce'): + return 2 + else: + return 1 class ClientNetwork(object): # pylint: disable=too-many-instance-attributes - """Client network.""" + """Wrapper around requests that signs POSTs for authentication. + + Also adds user agent, and handles Content-Type. + """ JSON_CONTENT_TYPE = 'application/json' JOSE_CONTENT_TYPE = 'application/jose+json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, + """Initialize. + + :param josepy.JWK key: Account private key + :param messages.RegistrationResource account: Account object. Required if you are + planning to use .post() with acme_version=2 for anything other than + creating a new account; may be set later after registering. + :param josepy.JWASignature alg: Algoritm to use in signing JWS. + :param bool verify_ssl: Whether to verify certificates on SSL connections. + :param str user_agent: String to send as User-Agent header. + :param float timeout: Timeout for requests. + """ + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() @@ -527,21 +876,31 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url, acme_version): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. - :param .JSONDeSerializable obj: + :param josepy.JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: - :rtype: `.JWS` + :rtype: `josepy.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if acme_version == 2: + kwargs["url"] = url + # newAccount and revokeCert work without the kid + if self.account is not None: + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -714,8 +1073,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes else: raise - def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, + acme_version=1, **kwargs): + data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..00b9e19dd 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -1,4 +1,5 @@ """Tests for acme.client.""" +import copy import datetime import json import unittest @@ -7,6 +8,7 @@ from six.moves import http_client # pylint: disable=import-error import josepy as jose import mock +import OpenSSL import requests from acme import challenges @@ -18,13 +20,32 @@ from acme import test_util CERT_DER = test_util.load_vector('cert.der') +CERT_SAN_PEM = test_util.load_vector('cert-san.pem') +CSR_SAN_PEM = test_util.load_vector('csr-san.pem') KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) +DIRECTORY_V1 = messages.Directory({ + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', + messages.CertificateRequest: + 'https://www.letsencrypt-demo.org/acme/new-cert', +}) -class ClientTest(unittest.TestCase): - """Tests for acme.client.Client.""" - # pylint: disable=too-many-instance-attributes,too-many-public-methods +DIRECTORY_V2 = messages.Directory({ + 'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account', + 'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce', + 'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order', + 'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert', +}) + + +class ClientTestBase(unittest.TestCase): + """Base for tests in acme.client.""" def setUp(self): self.response = mock.MagicMock( @@ -33,21 +54,6 @@ class ClientTest(unittest.TestCase): self.net.post.return_value = self.response self.net.get.return_value = self.response - self.directory = messages.Directory({ - messages.NewRegistration: - 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: - 'https://www.letsencrypt-demo.org/acme/revoke-cert', - messages.NewAuthorization: - 'https://www.letsencrypt-demo.org/acme/new-authz', - messages.CertificateRequest: - 'https://www.letsencrypt-demo.org/acme/new-cert', - }) - - from acme.client import Client - self.client = Client( - directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) - self.identifier = messages.Identifier( typ=messages.IDENTIFIER_FQDN, value='example.com') @@ -57,8 +63,7 @@ class ClientTest(unittest.TestCase): contact=self.contact, key=KEY.public_key()) self.new_reg = messages.NewRegistration(**dict(reg)) self.regr = messages.RegistrationResource( - body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1', - terms_of_service='https://www.letsencrypt-demo.org/tos') + body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') # Authorization authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1' @@ -75,14 +80,217 @@ class ClientTest(unittest.TestCase): self.authzr = messages.AuthorizationResource( body=self.authz, uri=authzr_uri) + # Reason code for revocation + self.rsn = 1 + + +class BackwardsCompatibleClientV2Test(ClientTestBase): + """Tests for acme.client.BackwardsCompatibleClientV2.""" + + def setUp(self): + super(BackwardsCompatibleClientV2Test, self).setUp() + # contains a loaded cert + self.certr = messages.CertificateResource( + body=messages_test.CERT) + + loaded = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM) + wrapped = jose.ComparableX509(loaded) + self.chain = [wrapped, wrapped] + + self.cert_pem = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode() + + single_chain = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, loaded).decode() + self.chain_pem = single_chain + single_chain + + self.fullchain_pem = self.cert_pem + self.chain_pem + + self.orderr = messages.OrderResource( + csr_pem=CSR_SAN_PEM) + + def _init(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + return BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + + def test_init_downloads_directory(self): + uri = 'http://www.letsencrypt-demo.org/directory' + from acme.client import BackwardsCompatibleClientV2 + BackwardsCompatibleClientV2(net=self.net, + key=KEY, server=uri) + self.net.get.assert_called_once_with(uri) + + def test_init_acme_version(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.acme_version, 1) + + self.response.json.return_value = DIRECTORY_V2.to_json() + client = self._init() + self.assertEqual(client.acme_version, 2) + + def test_forwarding(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + client = self._init() + self.assertEqual(client.directory, client.client.directory) + self.assertEqual(client.key, KEY) + self.assertEqual(client.update_registration, client.client.update_registration) + self.assertRaises(AttributeError, client.__getattr__, 'nonexistent') + self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos') + self.assertRaises(AttributeError, client.__getattr__, 'new_account') + + def test_new_account_and_tos(self): + # v2 no tos + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().new_account.assert_called_with(self.new_reg) + + # v2 tos good + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + client.new_account_and_tos(self.new_reg, lambda x: True) + mock_client().new_account.assert_called_with( + self.new_reg.update(terms_of_service_agreed=True)) + + # v2 tos bad + with mock.patch('acme.client.ClientV2') as mock_client: + mock_client().directory.meta.__contains__.return_value = True + client = self._init() + def _tos_cb(tos): + raise errors.Error + self.assertRaises(errors.Error, client.new_account_and_tos, + self.new_reg, _tos_cb) + mock_client().new_account.assert_not_called() + + # v1 yes tos + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service="TOS") + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_called_once_with(regr) + + # v1 no tos + with mock.patch('acme.client.Client') as mock_client: + regr = mock.MagicMock(terms_of_service=None) + mock_client().register.return_value = regr + client = self._init() + client.new_account_and_tos(self.new_reg) + mock_client().register.assert_called_once_with(self.new_reg) + mock_client().agree_to_tos.assert_not_called() + + @mock.patch('OpenSSL.crypto.load_certificate_request') + @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names') + def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names, + unused_mock_load_certificate_request): + self.response.json.return_value = DIRECTORY_V1.to_json() + mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com'] + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.Client') as mock_client: + mock_client().request_domain_challenges.return_value = mock.sentinel.auth + client = self._init() + orderr = client.new_order(mock_csr_pem) + self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth]) + + def test_new_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_csr_pem = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.new_order(mock_csr_pem) + mock_client().new_order.assert_called_once_with(mock_csr_pem) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_success(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + mock_client().fetch_chain.assert_called_once_with(self.certr) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_fetch_chain_error(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + mock_client().fetch_chain.return_value = self.chain + mock_client().fetch_chain.side_effect = [errors.Error, self.chain] + + deadline = datetime.datetime(9999, 9, 9) + client = self._init() + result = client.finalize_order(self.orderr, deadline) + self.assertEqual(result.fullchain_pem, self.fullchain_pem) + self.assertEqual(mock_client().fetch_chain.call_count, 2) + + @mock.patch('acme.client.Client') + def test_finalize_order_v1_timeout(self, mock_client): + self.response.json.return_value = DIRECTORY_V1.to_json() + + mock_client().request_issuance.return_value = self.certr + + deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + client = self._init() + self.assertRaises(errors.TimeoutError, client.finalize_order, + self.orderr, deadline) + + def test_finalize_order_v2(self): + self.response.json.return_value = DIRECTORY_V2.to_json() + mock_orderr = mock.MagicMock() + mock_deadline = mock.MagicMock() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.finalize_order(mock_orderr, mock_deadline) + mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline) + + def test_revoke(self): + self.response.json.return_value = DIRECTORY_V1.to_json() + with mock.patch('acme.client.Client') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + self.response.json.return_value = DIRECTORY_V2.to_json() + with mock.patch('acme.client.ClientV2') as mock_client: + client = self._init() + client.revoke(messages_test.CERT, self.rsn) + mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn) + + +class ClientTest(ClientTestBase): + """Tests for acme.client.Client.""" + # pylint: disable=too-many-instance-attributes,too-many-public-methods + + def setUp(self): + super(ClientTest, self).setUp() + + self.directory = DIRECTORY_V1 + + # Registration + self.regr = self.regr.update( + terms_of_service='https://www.letsencrypt-demo.org/tos') + # Request issuance self.certr = messages.CertificateResource( body=messages_test.CERT, authzrs=(self.authzr,), uri='https://www.letsencrypt-demo.org/acme/cert/1', cert_chain_uri='https://www.letsencrypt-demo.org/ca') - # Reason code for revocation - self.rsn = 1 + from acme.client import Client + self.client = Client( + directory=self.directory, key=KEY, alg=jose.RS256, net=self.net) def test_init_downloads_directory(self): uri = 'http://www.letsencrypt-demo.org/directory' @@ -142,20 +350,23 @@ class ClientTest(unittest.TestCase): self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_deprecated_arg(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier, new_authzr_uri="hi") self.net.post.assert_called_once_with( self.directory.new_authz, - messages.NewAuthorization(identifier=self.identifier)) + messages.NewAuthorization(identifier=self.identifier), + acme_version=1) def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) self.net.post.assert_called_once_with( - 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY) + 'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY, + acme_version=1) def test_request_challenges_unexpected_update(self): self._prepare_response_for_request_challenges() @@ -165,6 +376,13 @@ class ClientTest(unittest.TestCase): errors.UnexpectedUpdate, self.client.request_challenges, self.identifier) + def test_request_challenges_wildcard(self): + wildcard_identifier = messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='*.example.org') + self.assertRaises( + errors.WildcardUnsupportedError, self.client.request_challenges, + wildcard_identifier) + def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() self.assertEqual( @@ -417,7 +635,7 @@ class ClientTest(unittest.TestCase): def test_revoke(self): self.client.revoke(self.certr.body, self.rsn) self.net.post.assert_called_once_with( - self.directory[messages.Revocation], mock.ANY, content_type=None) + self.directory[messages.Revocation], mock.ANY, acme_version=1) def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) @@ -432,9 +650,150 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class ClientV2Test(ClientTestBase): + """Tests for acme.client.ClientV2.""" + + def setUp(self): + super(ClientV2Test, self).setUp() + + self.directory = DIRECTORY_V2 + + from acme.client import ClientV2 + self.client = ClientV2(self.directory, self.net) + + self.new_reg = self.new_reg.update(terms_of_service_agreed=True) + + self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2' + self.authz2 = self.authz.update(identifier=messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value='www.example.com'), + status=messages.STATUS_PENDING) + self.authzr2 = messages.AuthorizationResource( + body=self.authz2, uri=self.authzr_uri2) + + self.order = messages.Order( + identifiers=(self.authz.identifier, self.authz2.identifier), + status=messages.STATUS_PENDING, + authorizations=(self.authzr.uri, self.authzr_uri2), + finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize') + self.orderr = messages.OrderResource( + body=self.order, + uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1', + authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM) + + def test_new_account(self): + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.assertEqual(self.regr, self.client.new_account(self.new_reg)) + + def test_new_order(self): + order_response = copy.deepcopy(self.response) + order_response.status_code = http_client.CREATED + order_response.json.return_value = self.order.to_json() + order_response.headers['Location'] = self.orderr.uri + self.net.post.return_value = order_response + + authz_response = copy.deepcopy(self.response) + authz_response.json.return_value = self.authz.to_json() + authz_response.headers['Location'] = self.authzr.uri + authz_response2 = self.response + authz_response2.json.return_value = self.authz2.to_json() + authz_response2.headers['Location'] = self.authzr2.uri + self.net.get.side_effect = (authz_response, authz_response2) + + self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr) + + @mock.patch('acme.client.datetime') + def test_poll_and_finalize(self, mock_datetime): + mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15) + mock_datetime.timedelta = datetime.timedelta + expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90) + + self.client.poll_authorizations = mock.Mock(return_value=self.orderr) + self.client.finalize_order = mock.Mock(return_value=self.orderr) + + self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr) + self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline) + self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline) + + @mock.patch('acme.client.datetime') + def test_poll_authorizations_timeout(self, mock_datetime): + now_side_effect = [datetime.datetime(2018, 2, 15), + datetime.datetime(2018, 2, 16), + datetime.datetime(2018, 2, 17)] + mock_datetime.datetime.now.side_effect = now_side_effect + self.response.json.side_effect = [ + self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()] + + self.assertRaises( + errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1]) + + def test_poll_authorizations_failure(self): + deadline = datetime.datetime(9999, 9, 9) + challb = self.challr.body.update(status=messages.STATUS_INVALID, + error=messages.Error.with_code('unauthorized')) + authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,)) + self.response.json.return_value = authz.to_json() + + self.assertRaises( + errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline) + + def test_poll_authorizations_success(self): + deadline = datetime.datetime(9999, 9, 9) + updated_authz2 = self.authz2.update(status=messages.STATUS_VALID) + updated_authzr2 = messages.AuthorizationResource( + body=updated_authz2, uri=self.authzr_uri2) + updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2]) + + self.response.json.side_effect = ( + self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json()) + self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr) + + def test_finalize_order_success(self): + updated_order = self.order.update( + certificate='https://www.letsencrypt-demo.org/acme/cert/') + updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM) + + self.response.json.return_value = updated_order.to_json() + self.response.text = CERT_SAN_PEM + + deadline = datetime.datetime(9999, 9, 9) + self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr) + + def test_finalize_order_error(self): + updated_order = self.order.update(error=messages.Error.with_code('unauthorized')) + self.response.json.return_value = updated_order.to_json() + + deadline = datetime.datetime(9999, 9, 9) + self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline) + + def test_finalize_order_timeout(self): + deadline = datetime.datetime.now() - datetime.timedelta(seconds=60) + self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline) + + def test_revoke(self): + self.client.revoke(messages_test.CERT, self.rsn) + self.net.post.assert_called_once_with( + self.directory["revokeCert"], mock.ANY, acme_version=2) + + +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +812,27 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=1) jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url", + acme_version=2) + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +1062,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1) def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index b8fba0348..2281196eb 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,9 +5,10 @@ import logging import os import re import socket -import sys import OpenSSL +import josepy as jose + from acme import errors @@ -130,8 +131,7 @@ def probe_sni(name, host, port=443, timeout=300, context = OpenSSL.SSL.Context(method) context.set_timeout(timeout) - socket_kwargs = {} if sys.version_info < (2, 7) else { - 'source_address': source_address} + socket_kwargs = {'source_address': source_address} host_protocol_agnostic = None if host == '::' or host == '0' else host @@ -186,6 +186,15 @@ def make_csr(private_key_pem, domains, must_staple=False): return OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr) +def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req): + common_name = loaded_cert_or_req.get_subject().CN + sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. @@ -271,3 +280,26 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_pubkey(key) cert.sign(key, "sha256") return cert + +def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): + """Dump certificate chain into a bundle. + + :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in + :class:`josepy.util.ComparableX509`). + + :returns: certificate chain bundle + :rtype: bytes + + """ + # XXX: returns empty string when no chain is available, which + # shuts up RenewableCert, but might not be the best solution... + + def _dump_cert(cert): + if isinstance(cert, jose.ComparableX509): + # pylint: disable=protected-access + cert = cert.wrapped + return OpenSSL.crypto.dump_certificate(filetype, cert) + + # assumes that OpenSSL.crypto.dump_certificate includes ending + # newline character + return b"".join(_dump_cert(cert) for cert in chain) diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..3874ba9d9 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -65,6 +65,30 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # self.assertRaises(errors.Error, self._probe, b'bar') +class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): + """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" + + @classmethod + def _call(cls, loader, name): + # pylint: disable=protected-access + from acme.crypto_util import _pyopenssl_cert_or_req_all_names + return _pyopenssl_cert_or_req_all_names(loader(name)) + + def _call_cert(self, name): + return self._call(test_util.load_cert, name) + + def test_cert_one_san_no_common(self): + self.assertEqual(self._call_cert('cert-nocn.der'), + ['no-common-name.badssl.com']) + + def test_cert_no_sans_yes_common(self): + self.assertEqual(self._call_cert('cert.pem'), ['example.com']) + + def test_cert_two_sans_yes_common(self): + self.assertEqual(self._call_cert('cert-san.pem'), + ['example.com', 'www.example.com']) + + class PyOpenSSLCertOrReqSANTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_san.""" @@ -170,9 +194,9 @@ class MakeCSRTest(unittest.TestCase): self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 1) self.assertEquals(csr.get_extensions()[0].get_data(), @@ -188,9 +212,9 @@ class MakeCSRTest(unittest.TestCase): csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) - # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr - # objects don't have a get_extensions() method, so we skip this test if - # the method isn't available. + # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't + # have a get_extensions() method, so we skip this test if the method + # isn't available. if hasattr(csr, 'get_extensions'): self.assertEquals(len(csr.get_extensions()), 2) # NOTE: Ideally we would filter by the TLS Feature OID, but @@ -201,5 +225,33 @@ class MakeCSRTest(unittest.TestCase): self.assertEqual(len(must_staple_exts), 1, "Expected exactly one Must Staple extension") + +class DumpPyopensslChainTest(unittest.TestCase): + """Test for dump_pyopenssl_chain.""" + + @classmethod + def _call(cls, loaded): + # pylint: disable=protected-access + from acme.crypto_util import dump_pyopenssl_chain + return dump_pyopenssl_chain(loaded) + + def test_dump_pyopenssl_chain(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + length = sum( + len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) + for cert in loaded) + self.assertEqual(len(self._call(loaded)), length) + + def test_dump_pyopenssl_chain_wrapped(self): + names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem'] + loaded = [test_util.load_cert(name) for name in names] + wrap_func = jose.ComparableX509 + wrapped = [wrap_func(cert) for cert in loaded] + dump_func = OpenSSL.crypto.dump_certificate + length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded) + self.assertEqual(len(self._call(wrapped)), length) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..97fa73614 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -83,6 +83,28 @@ class PollError(ClientError): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) +class ValidationError(Error): + """Error for authorization failures. Contains a list of authorization + resources, each of which is invalid and should have an error field. + """ + def __init__(self, failed_authzrs): + self.failed_authzrs = failed_authzrs + super(ValidationError, self).__init__() + +class TimeoutError(Error): + """Error for when polling an authorization or an order times out.""" + +class IssuanceError(Error): + """Error sent by the server after requesting issuance of a certificate.""" + + def __init__(self, error): + """Initialize. + + :param messages.Error error: The error provided by the server. + """ + self.error = error + super(IssuanceError, self).__init__() + class ConflictError(ClientError): """Error for when the server returns a 409 (Conflict) HTTP status. @@ -93,3 +115,6 @@ class ConflictError(ClientError): self.location = location super(ConflictError, self).__init__() + +class WildcardUnsupportedError(Error): + """Error for when a wildcard is requested but is unsupported by ACME CA.""" diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 6daf55094..23cd66c63 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -171,9 +171,30 @@ class Directory(jose.JSONDeSerializable): class Meta(jose.JSONObjectWithFields): """Directory Meta.""" - terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service = jose.Field('terms-of-service', omitempty=True) + _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True) website = jose.Field('website', omitempty=True) - caa_identities = jose.Field('caa-identities', omitempty=True) + caa_identities = jose.Field('caaIdentities', omitempty=True) + + def __init__(self, **kwargs): + kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items()) + # pylint: disable=star-args + super(Directory.Meta, self).__init__(**kwargs) + + @property + def terms_of_service(self): + """URL for the CA TOS""" + return self._terms_of_service or self._terms_of_service_v2 + + def __iter__(self): + # When iterating over fields, use the external name 'terms_of_service' instead of + # the internal '_terms_of_service'. + for name in super(Directory.Meta, self).__iter__(): + yield name[1:] if name == '_terms_of_service' else name + + def _internal_name(self, name): + return '_' + name if name == 'terms_of_service' else name + @classmethod def _canon_key(cls, key): @@ -251,6 +272,7 @@ class Registration(ResourceBody): contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) status = jose.Field('status', omitempty=True) + terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True) phone_prefix = 'tel:' email_prefix = 'mailto:' @@ -482,3 +504,50 @@ class Revocation(jose.JSONObjectWithFields): certificate = jose.Field( 'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert) reason = jose.Field('reason') + + +class Order(ResourceBody): + """Order Resource Body. + + :ivar list of .Identifier: List of identifiers for the certificate. + :ivar acme.messages.Status status: + :ivar list of str authorizations: URLs of authorizations. + :ivar str certificate: URL to download certificate as a fullchain PEM. + :ivar str finalize: URL to POST to to request issuance once all + authorizations have "valid" status. + :ivar datetime.datetime expires: When the order expires. + :ivar .Error error: Any error that occurred during finalization, if applicable. + """ + identifiers = jose.Field('identifiers', omitempty=True) + status = jose.Field('status', decoder=Status.from_json, + omitempty=True, default=STATUS_PENDING) + authorizations = jose.Field('authorizations', omitempty=True) + certificate = jose.Field('certificate', omitempty=True) + finalize = jose.Field('finalize', omitempty=True) + expires = fields.RFC3339Field('expires', omitempty=True) + error = jose.Field('error', omitempty=True, decoder=Error.from_json) + + @identifiers.decoder + def identifiers(value): # pylint: disable=missing-docstring,no-self-argument + return tuple(Identifier.from_json(identifier) for identifier in value) + +class OrderResource(ResourceWithURI): + """Order Resource. + + :ivar acme.messages.Order body: + :ivar str csr_pem: The CSR this Order will be finalized with. + :ivar list of acme.messages.AuthorizationResource authorizations: + Fully-fetched AuthorizationResource objects. + :ivar str fullchain_pem: The fetched contents of the certificate URL + produced once the order was finalized, if it's present. + """ + body = jose.Field('body', decoder=Order.from_json) + csr_pem = jose.Field('csr_pem', omitempty=True) + authorizations = jose.Field('authorizations') + fullchain_pem = jose.Field('fullchain_pem', omitempty=True) + +@Directory.register +class NewOrder(Order): + """New order.""" + resource_type = 'new-order' + resource = fields.Resource(resource_type) diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index fa885d3c2..64bc81efd 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -157,7 +157,7 @@ class DirectoryTest(unittest.TestCase): 'meta': { 'terms-of-service': 'https://example.com/acme/terms', 'website': 'https://www.example.com/', - 'caa-identities': ['example.com'], + 'caaIdentities': ['example.com'], }, }) @@ -165,6 +165,13 @@ class DirectoryTest(unittest.TestCase): from acme.messages import Directory Directory.from_json({'foo': 'bar'}) + def test_iter_meta(self): + result = False + for k in self.dir.meta: + if k == 'terms_of_service': + result = self.dir.meta[k] == 'https://example.com/acme/terms' + self.assertTrue(result) + class RegistrationTest(unittest.TestCase): """Tests for acme.messages.Registration.""" @@ -401,5 +408,21 @@ class RevocationTest(unittest.TestCase): hash(Revocation.from_json(self.rev.to_json())) +class OrderResourceTest(unittest.TestCase): + """Tests for acme.messages.OrderResource.""" + + def setUp(self): + from acme.messages import OrderResource + self.regr = OrderResource( + body=mock.sentinel.body, uri=mock.sentinel.uri) + + def test_to_partial_json(self): + self.assertEqual(self.regr.to_json(), { + 'body': mock.sentinel.body, + 'uri': mock.sentinel.uri, + 'authorizations': None, + }) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/testdata/cert-nocn.der b/acme/acme/testdata/cert-nocn.der new file mode 100644 index 000000000..59da83ccc Binary files /dev/null and b/acme/acme/testdata/cert-nocn.der differ diff --git a/acme/docs/api/other.rst b/acme/docs/api/other.rst deleted file mode 100644 index eb27a5d53..000000000 --- a/acme/docs/api/other.rst +++ /dev/null @@ -1,5 +0,0 @@ -Other ACME objects ------------------- - -.. automodule:: acme.other - :members: diff --git a/acme/setup.py b/acme/setup.py index ce426cf74..071b56ab3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -19,19 +19,10 @@ install_requires = [ 'pyrfc3339', 'pytz', 'requests[security]>=2.4.1', # security extras added in 2.4.1 - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ 'pytest', 'pytest-xdist', @@ -52,16 +43,15 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index e0982a5d6..711238eb6 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -5,6 +5,7 @@ import logging import os import pkg_resources import re +import six import socket import time @@ -152,6 +153,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = dict() # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) @@ -262,6 +266,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version, configurator=self) + def _wildcard_domain(self, domain): + """ + Checks if domain is a wildcard domain + + :param str domain: Domain to check + + :returns: If the domain is wildcard domain + :rtype: bool + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + return domain.startswith(wildcard_marker) + def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. @@ -280,9 +299,112 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): a lack of directives """ - # Choose vhost before (possible) enabling of mod_ssl, to keep the - # vhost choice namespace similar with the pre-validation one. - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def choose_vhosts(self, domain, create_if_no_ssl=True): + """ + Finds VirtualHosts that can be used with the provided domain + + :param str domain: Domain name to match VirtualHosts to + :param bool create_if_no_ssl: If found VirtualHost doesn't have a HTTPS + counterpart, should one get created + + :returns: List of VirtualHosts or None + :rtype: `list` of :class:`~certbot_apache.obj.VirtualHost` + """ + + if self._wildcard_domain(domain): + if domain in self._wildcard_vhosts: + # Vhosts for a wildcard domain were already selected + return self._wildcard_vhosts[domain] + # Ask user which VHosts to support. + # Returned objects are guaranteed to be ssl vhosts + return self._choose_vhosts_wildcard(domain, create_if_no_ssl) + else: + return [self.choose_vhost(domain)] + + def _vhosts_for_wildcard(self, domain): + """ + Get VHost objects for every VirtualHost that the user wants to handle + with the wildcard certificate. + """ + + # Collect all vhosts that match the name + matched = set() + for vhost in self.vhosts: + for name in vhost.get_names(): + if self._in_wildcard_scope(name, domain): + matched.add(vhost) + + return list(matched) + + def _in_wildcard_scope(self, name, domain): + """ + Helper method for _vhosts_for_wildcard() that makes sure that the domain + is in the scope of wildcard domain. + + eg. in scope: domain = *.wild.card, name = 1.wild.card + not in scope: domain = *.wild.card, name = 1.2.wild.card + """ + if len(name.split(".")) == len(domain.split(".")): + return fnmatch.fnmatch(name, domain) + + + def _choose_vhosts_wildcard(self, domain, create_ssl=True): + """Prompts user to choose vhosts to install a wildcard certificate for""" + + # Get all vhosts that are covered by the wildcard domain + vhosts = self._vhosts_for_wildcard(domain) + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL vhosts + filtered_vhosts = dict() + for vhost in vhosts: + for name in vhost.get_names(): + if vhost.ssl: + # Always prefer SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts and create_ssl: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + dialog_output = display_ops.select_vhost_multiple(list(dialog_input)) + + if not dialog_output: + logger.error( + "No vhost exists with servername or alias for domain %s. " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config.", + domain) + raise errors.PluginError("No vhost selected") + + # Make sure we create SSL vhosts for the ones that are HTTP only + # if requested. + return_vhosts = list() + for vhost in dialog_output: + if not vhost.ssl: + return_vhosts.append(self.make_vhost_ssl(vhost)) + else: + return_vhosts.append(vhost) + + self._wildcard_vhosts[domain] = return_vhosts + return return_vhosts + + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ + # This is done first so that ssl module is enabled and cert_path, # cert_key... can all be parsed appropriately @@ -311,7 +433,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -327,8 +449,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "version of Apache") else: if not fullchain_path: - raise errors.PluginError("Please provide the --fullchain-path\ - option pointing to your full chain file") + raise errors.PluginError("Please provide the --fullchain-path " + "option pointing to your full chain file") set_cert_path = fullchain_path self.aug.set(path["cert_path"][-1], fullchain_path) self.aug.set(path["cert_key"][-1], key_path) @@ -391,7 +513,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.error( "No vhost exists with servername or alias of %s. " "No vhost was selected. Please specify ServerName or ServerAlias " - "in the Apache config, or split vhosts into separate files.", + "in the Apache config.", target_name) raise errors.PluginError("No vhost selected") elif temp: @@ -1269,7 +1391,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_cert_file_path") self.parser.add_dir(vh_path, "SSLCertificateKeyFile", "insert_key_file_path") - self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + # Only include the TLS configuration if not already included + existing_inc = self.parser.find_dir("Include", self.mod_ssl_conf, vh_path) + if not existing_inc: + self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path @@ -1373,8 +1498,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): except KeyError: raise errors.PluginError( "Unsupported enhancement: {0}".format(enhancement)) + + vhosts = self.choose_vhosts(domain, create_if_no_ssl=False) try: - func(self.choose_vhost(domain), options) + for vhost in vhosts: + func(vhost, options) except errors.PluginError: logger.warning("Failed %s for %s", enhancement, domain) raise diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 9529c1ab3..097b84b96 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -13,10 +13,44 @@ import certbot.display.util as display_util logger = logging.getLogger(__name__) +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + + :param vhosts: Available Apache VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which VirtualHosts would you like to install the wildcard certificate for?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts + def select_vhost(domain, vhosts): """Select an appropriate Apache Vhost. - :param vhosts: Available Apache Virtual Hosts + :param vhosts: Available Apache VirtualHosts :type vhosts: :class:`list` of type `~obj.Vhost` :returns: VirtualHost or `None` @@ -25,13 +59,11 @@ def select_vhost(domain, vhosts): """ if not vhosts: return None - while True: - code, tag = _vhost_menu(domain, vhosts) - if code == display_util.OK: - return vhosts[tag] - else: - return None - + code, tag = _vhost_menu(domain, vhosts) + if code == display_util.OK: + return vhosts[tag] + else: + return None def _vhost_menu(domain, vhosts): """Select an appropriate Apache Vhost. diff --git a/certbot-apache/certbot_apache/entrypoint.py b/certbot-apache/certbot_apache/entrypoint.py index 4267398d5..6f1443507 100644 --- a/certbot-apache/certbot_apache/entrypoint.py +++ b/certbot-apache/certbot_apache/entrypoint.py @@ -17,6 +17,7 @@ OVERRIDE_CLASSES = { "centos": override_centos.CentOSConfigurator, "centos linux": override_centos.CentOSConfigurator, "fedora": override_centos.CentOSConfigurator, + "ol": override_centos.CentOSConfigurator, "red hat enterprise linux server": override_centos.CentOSConfigurator, "rhel": override_centos.CentOSConfigurator, "amazon": override_centos.CentOSConfigurator, diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py index e463f3880..cce93a646 100644 --- a/certbot-apache/certbot_apache/http_01.py +++ b/certbot-apache/certbot_apache/http_01.py @@ -11,30 +11,43 @@ logger = logging.getLogger(__name__) class ApacheHttp01(common.TLSSNI01): """Class that performs HTTP-01 challenges within the Apache configurator.""" - CONFIG_TEMPLATE22 = """\ + CONFIG_TEMPLATE22_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L] + """ + CONFIG_TEMPLATE22_POST = """\ Order Allow,Deny Allow from all + + Order Allow,Deny + Allow from all + """ - CONFIG_TEMPLATE24 = """\ + CONFIG_TEMPLATE24_PRE = """\ RewriteEngine on RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END] - + """ + CONFIG_TEMPLATE24_POST = """\ Require all granted + + Require all granted + """ def __init__(self, *args, **kwargs): super(ApacheHttp01, self).__init__(*args, **kwargs) - self.challenge_conf = os.path.join( + self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), - "le_http_01_challenge.conf") + "le_http_01_challenge_pre.conf") + self.challenge_conf_post = os.path.join( + self.configurator.conf("challenge-location"), + "le_http_01_challenge_post.conf") self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") @@ -79,24 +92,32 @@ class ApacheHttp01(common.TLSSNI01): chall.domain, filter_defaults=False, port=str(self.configurator.config.http01_port)) if vh: - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) else: for vh in self._relevant_vhosts(): - self._set_up_include_directive(vh) + self._set_up_include_directives(vh) self.configurator.reverter.register_file_creation( - True, self.challenge_conf) + True, self.challenge_conf_pre) + self.configurator.reverter.register_file_creation( + True, self.challenge_conf_post) if self.configurator.version < (2, 4): - config_template = self.CONFIG_TEMPLATE22 + config_template_pre = self.CONFIG_TEMPLATE22_PRE + config_template_post = self.CONFIG_TEMPLATE22_POST else: - config_template = self.CONFIG_TEMPLATE24 + config_template_pre = self.CONFIG_TEMPLATE24_PRE + config_template_post = self.CONFIG_TEMPLATE24_POST - config_text = config_template.format(self.challenge_dir) + config_text_pre = config_template_pre.format(self.challenge_dir) + config_text_post = config_template_post.format(self.challenge_dir) - logger.debug("writing a config file with text:\n %s", config_text) - with open(self.challenge_conf, "w") as new_conf: - new_conf.write(config_text) + logger.debug("writing a pre config file with text:\n %s", config_text_pre) + with open(self.challenge_conf_pre, "w") as new_conf: + new_conf.write(config_text_pre) + logger.debug("writing a post config file with text:\n %s", config_text_post) + with open(self.challenge_conf_post, "w") as new_conf: + new_conf.write(config_text_post) def _relevant_vhosts(self): http01_port = str(self.configurator.config.http01_port) @@ -137,14 +158,17 @@ class ApacheHttp01(common.TLSSNI01): return response - def _set_up_include_directive(self, vhost): - """Includes override configuration to the beginning of VirtualHost. - Note that this include isn't added to Augeas search tree""" + def _set_up_include_directives(self, vhost): + """Includes override configuration to the beginning and to the end of + VirtualHost. Note that this include isn't added to Augeas search tree""" if vhost not in self.moded_vhosts: logger.debug( "Adding a temporary challenge validation Include for name: %s " + "in: %s", vhost.name, vhost.filep) self.configurator.parser.add_dir_beginning( - vhost.path, "Include", self.challenge_conf) + vhost.path, "Include", self.challenge_conf_pre) + self.configurator.parser.add_dir( + vhost.path, "Include", self.challenge_conf_post) + self.moded_vhosts.add(vhost) diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 1e3579858..fcf3bfe08 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -167,6 +167,19 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods active="Yes" if self.enabled else "No", modmacro="Yes" if self.modmacro else "No")) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.get_names()), + https="Yes" if self.ssl else "No")) + + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and self.path == other.path and diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 530d75a92..c9bf9a63f 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -335,6 +335,33 @@ class MultipleVhostsTest(util.ApacheTest): "example/cert_chain.pem", "example/fullchain.pem") self.assertTrue(ssl_vhost.enabled) + def test_no_duplicate_include(self): + def mock_find_dir(directive, argument, _): + """Mock method for parser.find_dir""" + if directive == "Include" and argument.endswith("options-ssl-apache.conf"): + return ["/path/to/whatever"] + + mock_add = mock.MagicMock() + self.config.parser.add_dir = mock_add + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = False + for a in mock_add.call_args_list: + if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf: + tried_to_add = True + # Include should be added, find_dir is not patched, and returns falsy + self.assertTrue(tried_to_add) + + self.config.parser.find_dir = mock_find_dir + mock_add.reset_mock() + + self.config._add_dummy_ssl_directives(self.vh_truth[0]) # pylint: disable=protected-access + tried_to_add = [] + for a in mock_add.call_args_list: + tried_to_add.append(a[0][1] == "Include" and + a[0][2] == self.config.mod_ssl_conf) + # Include shouldn't be added, as patched find_dir "finds" existing one + self.assertFalse(any(tried_to_add)) + def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -474,7 +501,11 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 3) self.assertTrue(mock_add_dir.called) self.assertEqual(mock_add_dir.call_args[0][1], "Listen") - self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080']) + call_found = False + for mock_call in mock_add_dir.mock_calls: + if mock_call[1][2] == ['1.2.3.4:8080']: + call_found = True + self.assertTrue(call_found) def test_prepare_server_https(self): mock_enable = mock.Mock() @@ -1306,6 +1337,106 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enable_mod, "whatever") + def test_wildcard_domain(self): + # pylint: disable=protected-access + cases = {u"*.example.org": True, b"*.x.example.org": True, + u"a.example.org": False, b"a.x.example.org": False} + for key in cases.keys(): + self.assertEqual(self.config._wildcard_domain(key), cases[key]) + + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[3]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=True) + # Check that the dialog was called with one vh: certbot.demo + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].name == "certbot.demo") + self.assertTrue(vhs[0].ssl) + + self.assertFalse(vhs[0] == self.vh_truth[3]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl): + # pylint: disable=protected-access + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[1]] + vhs = self.config._choose_vhosts_wildcard("*.certbot.demo", + create_ssl=False) + self.assertFalse(mock_makessl.called) + self.assertEquals(vhs[0], self.vh_truth[1]) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard") + @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl") + def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w): + # pylint: disable=protected-access + # Already SSL vhost + mock_vh_for_w.return_value = [self.vh_truth[7]] + mock_path = "certbot_apache.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [self.vh_truth[7]] + vhs = self.config._choose_vhosts_wildcard("whatever", + create_ssl=True) + self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7]) + self.assertEquals(len(mock_select_vhs.call_args_list), 1) + # Ensure that make_vhost_ssl was not called, vhost.ssl == true + self.assertFalse(mock_makessl.called) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertTrue(vhs[0].ssl) + self.assertEquals(vhs[0], self.vh_truth[7]) + + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + mock_choose_vhosts.return_value = [self.vh_truth[7]] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.wildcard.example.org", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_apache.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_after_install(self, mock_choose): + # pylint: disable=protected-access + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertFalse(mock_choose.called) + + @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard") + def test_enhance_wildcard_no_install(self, mock_choose): + mock_choose.return_value = [self.vh_truth[3]] + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("headers_module") + self.config.enhance("*.certbot.demo", "ensure-http-header", + "Upgrade-Insecure-Requests") + self.assertTrue(mock_choose.called) + + class AugeasVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependent on augeas version.""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index e59d411bd..df5cdbac0 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util from certbot_apache import obj +from certbot_apache.display_ops import select_vhost_multiple from certbot_apache.tests import util +class SelectVhostMultiTest(unittest.TestCase): + """Tests for certbot_apache.display_ops.select_vhost_multiple.""" + + def setUp(self): + self.base_dir = "/example_path" + self.vhosts = util.get_vh_truth( + self.base_dir, "debian_apache_2_4/multiple_vhosts") + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + class SelectVhostTest(unittest.TestCase): """Tests for certbot_apache.display_ops.select_vhost.""" diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py index 64a76649a..9ed4ee509 100644 --- a/certbot-apache/certbot_apache/tests/http_01_test.py +++ b/certbot-apache/certbot_apache/tests/http_01_test.py @@ -158,23 +158,31 @@ class ApacheHttp01Test(util.ApacheTest): for vhost in vhosts: if not vhost.ssl: matches = self.config.parser.find_dir("Include", - self.http.challenge_conf, + self.http.challenge_conf_pre, + vhost.path) + self.assertEqual(len(matches), 1) + matches = self.config.parser.find_dir("Include", + self.http.challenge_conf_post, vhost.path) self.assertEqual(len(matches), 1) self.assertTrue(os.path.exists(challenge_dir)) def _test_challenge_conf(self): - with open(self.http.challenge_conf) as f: - conf_contents = f.read() + with open(self.http.challenge_conf_pre) as f: + pre_conf_contents = f.read() - self.assertTrue("RewriteEngine on" in conf_contents) - self.assertTrue("RewriteRule" in conf_contents) - self.assertTrue(self.http.challenge_dir in conf_contents) + with open(self.http.challenge_conf_post) as f: + post_conf_contents = f.read() + + self.assertTrue("RewriteEngine on" in pre_conf_contents) + self.assertTrue("RewriteRule" in pre_conf_contents) + + self.assertTrue(self.http.challenge_dir in post_conf_contents) if self.config.version < (2, 4): - self.assertTrue("Allow from all" in conf_contents) + self.assertTrue("Allow from all" in post_conf_contents) else: - self.assertTrue("Require all granted" in conf_contents) + self.assertTrue("Require all granted" in post_conf_contents) def _test_challenge_file(self, achall): name = os.path.join(self.http.challenge_dir, achall.chall.encode("token")) diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-apache/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 38f41e9f1..7608c0647 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-augeas', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] @@ -32,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-auto b/certbot-auto index 558c330b2..d3a5c23e5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8f9f897cf..861921ef7 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -34,16 +34,15 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile new file mode 100644 index 000000000..27dcc8751 --- /dev/null +++ b/certbot-dns-cloudflare/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudflare + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudflare/local-oldest-requirements.txt b/certbot-dns-cloudflare/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudflare/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudflare/readthedocs.org.requirements.txt b/certbot-dns-cloudflare/readthedocs.org.requirements.txt new file mode 100644 index 000000000..b18901111 --- /dev/null +++ b/certbot-dns-cloudflare/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudflare[docs] diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 612e7259f..4ed8e796d 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'cloudflare>=1.5.1', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile new file mode 100644 index 000000000..cc84ea65b --- /dev/null +++ b/certbot-dns-cloudxns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-cloudxns + +RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-cloudxns/readthedocs.org.requirements.txt b/certbot-dns-cloudxns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..ae2ff8165 --- /dev/null +++ b/certbot-dns-cloudxns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-cloudxns[docs] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3157400c6..7f973709c 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile new file mode 100644 index 000000000..8bdd0619f --- /dev/null +++ b/certbot-dns-digitalocean/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-digitalocean + +RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-digitalocean/local-oldest-requirements.txt b/certbot-dns-digitalocean/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-digitalocean/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-digitalocean/readthedocs.org.requirements.txt b/certbot-dns-digitalocean/readthedocs.org.requirements.txt new file mode 100644 index 000000000..08d973ab3 --- /dev/null +++ b/certbot-dns-digitalocean/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-digitalocean[docs] diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 1a68400fa..0ce91e64e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'mock', 'python-digitalocean>=1.11', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'six', 'zope.interface', ] @@ -32,6 +31,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +40,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile new file mode 100644 index 000000000..38d2be80e --- /dev/null +++ b/certbot-dns-dnsimple/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsimple + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsimple/readthedocs.org.requirements.txt b/certbot-dns-dnsimple/readthedocs.org.requirements.txt new file mode 100644 index 000000000..fef73916c --- /dev/null +++ b/certbot-dns-dnsimple/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsimple[docs] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 35de47308..d12b26d83 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile new file mode 100644 index 000000000..ff7936925 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-dnsmadeeasy + +RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt new file mode 100644 index 000000000..8f8c6c731 --- /dev/null +++ b/certbot-dns-dnsmadeeasy/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-dnsmadeeasy[docs] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index a946d00a4..856eaba0f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile new file mode 100644 index 000000000..4a258d0ee --- /dev/null +++ b/certbot-dns-google/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-google + +RUN pip install --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 7349a7696..f19266737 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -29,6 +29,8 @@ for an account with the following permissions: * ``dns.managedZones.list`` * ``dns.resourceRecordSets.create`` * ``dns.resourceRecordSets.delete`` +* ``dns.resourceRecordSets.list`` +* ``dns.resourceRecordSets.update`` Google provides instructions for `creating a service account `_ and diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 37fd6b0de..e2088b357 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -107,6 +107,17 @@ class _GoogleClient(object): zone_id = self._find_managed_zone_id(domain) + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = [] + add_records = record_contents[:] + + if "\""+record_content+"\"" in record_contents: + # The process was interrupted previously and validation token exists + return + + add_records.append(record_content) + data = { "kind": "dns#change", "additions": [ @@ -114,12 +125,24 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": add_records, "ttl": record_ttl, }, ], } + if record_contents: + # We need to remove old records in the same request + data["deletions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": record_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -154,6 +177,10 @@ class _GoogleClient(object): logger.warn('Error finding zone. Skipping cleanup.') return + record_contents = self.get_existing_txt_rrset(zone_id, record_name) + if record_contents is None: + record_contents = ["\"" + record_content + "\""] + data = { "kind": "dns#change", "deletions": [ @@ -161,12 +188,26 @@ class _GoogleClient(object): "kind": "dns#resourceRecordSet", "type": "TXT", "name": record_name + ".", - "rrdatas": [record_content, ], + "rrdatas": record_contents, "ttl": record_ttl, }, ], } + # Remove the record being deleted from the list + readd_contents = [r for r in record_contents if r != "\"" + record_content + "\""] + if readd_contents: + # We need to remove old records in the same request + data["additions"] = [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": readd_contents, + "ttl": record_ttl, + }, + ] + changes = self.dns.changes() # changes | pylint: disable=no-member try: @@ -175,6 +216,37 @@ class _GoogleClient(object): except googleapiclient_errors.Error as e: logger.warn('Encountered error deleting TXT record: %s', e) + def get_existing_txt_rrset(self, zone_id, record_name): + """ + Get existing TXT records from the RRset for the record name. + + If an error occurs while requesting the record set, it is suppressed + and None is returned. + + :param str zone_id: The ID of the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + + :returns: List of TXT record values or None + :rtype: `list` of `string` or `None` + + """ + rrs_request = self.dns.resourceRecordSets() # pylint: disable=no-member + request = rrs_request.list(managedZone=zone_id, project=self.project_id) + # Add dot as the API returns absolute domains + record_name += "." + try: + response = request.execute() + except googleapiclient_errors.Error: + logger.info("Unable to list existing records. If you're " + "requesting a wildcard certificate, this might not work.") + logger.debug("Error was:", exc_info=True) + else: + if response: + for rr in response["rrsets"]: + if rr["name"] == record_name and rr["type"] == "TXT": + return rr["rrdatas"] + return None + def _find_managed_zone_id(self, domain): """ Find the managed zone for a given domain. @@ -224,4 +296,7 @@ class _GoogleClient(object): if r.status != 200: raise ValueError("Invalid status code: {0}".format(r)) - return content + if isinstance(content, bytes): + return content.decode() + else: + return content diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py index 85649fc7f..72b8be8af 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google_test.py +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -74,10 +74,15 @@ class GoogleClientTest(unittest.TestCase): mock_mz = mock.MagicMock() mock_mz.list.return_value.execute.side_effect = zone_request_side_effect + mock_rrs = mock.MagicMock() + rrsets = {"rrsets": [{"name": "_acme-challenge.example.org.", "type": "TXT", + "rrdatas": ["\"example-txt-contents\""]}]} + mock_rrs.list.return_value.execute.return_value = rrsets mock_changes = mock.MagicMock() client.dns.managedZones = mock.MagicMock(return_value=mock_mz) client.dns.changes = mock.MagicMock(return_value=mock_changes) + client.dns.resourceRecordSets = mock.MagicMock(return_value=mock_rrs) return client, mock_changes @@ -137,6 +142,30 @@ class GoogleClientTest(unittest.TestCase): managedZone=self.zone, project=PROJECT_ID) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_delete_old(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["sample-txt-contents"] + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + self.assertTrue(changes.create.called) + self.assertTrue("sample-txt-contents" in + changes.create.call_args_list[0][1]["body"]["deletions"][0]["rrdatas"]) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_add_txt_record_noop(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + client.add_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) + self.assertFalse(changes.create.called) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') @mock.patch('certbot_dns_google.dns_google.open', mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) @@ -172,7 +201,12 @@ class GoogleClientTest(unittest.TestCase): def test_del_txt_record(self, unused_credential_mock): client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) - client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + mock_get_rrs = "certbot_dns_google.dns_google._GoogleClient.get_existing_txt_rrset" + with mock.patch(mock_get_rrs) as mock_rrs: + mock_rrs.return_value = ["\"sample-txt-contents\"", + "\"example-txt-contents\""] + client.del_txt_record(DOMAIN, "_acme-challenge.example.org", + "example-txt-contents", self.record_ttl) expected_body = { "kind": "dns#change", @@ -180,8 +214,17 @@ class GoogleClientTest(unittest.TestCase): { "kind": "dns#resourceRecordSet", "type": "TXT", - "name": self.record_name + ".", - "rrdatas": [self.record_content, ], + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", "\"example-txt-contents\""], + "ttl": self.record_ttl, + }, + ], + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": "_acme-challenge.example.org.", + "rrdatas": ["\"sample-txt-contents\"", ], "ttl": self.record_ttl, }, ], @@ -217,15 +260,44 @@ class GoogleClientTest(unittest.TestCase): client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # Record name mocked in setUp + found = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertEquals(found, ["\"example-txt-contents\""]) + not_found = client.get_existing_txt_rrset(self.zone, "nonexistent.tld") + self.assertEquals(not_found, None) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}'), create=True) + def test_get_existing_fallback(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock( + [{'managedZones': [{'id': self.zone}]}]) + # pylint: disable=no-member + mock_execute = client.dns.resourceRecordSets.return_value.list.return_value.execute + mock_execute.side_effect = API_ERROR + + rrset = client.get_existing_txt_rrset(self.zone, "_acme-challenge.example.org") + self.assertFalse(rrset) + def test_get_project_id(self): from certbot_dns_google.dns_google import _GoogleClient response = DummyResponse() response.status = 200 - with mock.patch('httplib2.Http.request', return_value=(response, 1234)): + with mock.patch('httplib2.Http.request', return_value=(response, 'test-test-1')): project_id = _GoogleClient.get_project_id() - self.assertEqual(project_id, 1234) + self.assertEqual(project_id, 'test-test-1') + + with mock.patch('httplib2.Http.request', return_value=(response, b'test-test-1')): + project_id = _GoogleClient.get_project_id() + self.assertEqual(project_id, 'test-test-1') failed_response = DummyResponse() failed_response.status = 404 diff --git a/certbot-dns-google/local-oldest-requirements.txt b/certbot-dns-google/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-google/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-google/readthedocs.org.requirements.txt b/certbot-dns-google/readthedocs.org.requirements.txt new file mode 100644 index 000000000..6ea393f86 --- /dev/null +++ b/certbot-dns-google/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-google[docs] diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8585fc848..0dfff0402 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,18 +6,17 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', # 1.5 is the first version that supports oauth2client>=2.0 'google-api-python-client>=1.5', 'mock', # for oauth2client.service_account.ServiceAccountCredentials 'oauth2client>=2.0', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', # already a dependency of google-api-python-client, but added for consistency 'httplib2' @@ -36,6 +35,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -44,10 +44,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile new file mode 100644 index 000000000..6efb4d777 --- /dev/null +++ b/certbot-dns-luadns/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-luadns + +RUN pip install --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-luadns/readthedocs.org.requirements.txt b/certbot-dns-luadns/readthedocs.org.requirements.txt new file mode 100644 index 000000000..acb51e4ef --- /dev/null +++ b/certbot-dns-luadns/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-luadns[docs] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4fec37e29..b255691dc 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile new file mode 100644 index 000000000..88fc13c57 --- /dev/null +++ b/certbot-dns-nsone/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-nsone + +RUN pip install --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-nsone/readthedocs.org.requirements.txt b/certbot-dns-nsone/readthedocs.org.requirements.txt new file mode 100644 index 000000000..dbdee4480 --- /dev/null +++ b/certbot-dns-nsone/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-nsone[docs] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index dca9ebf27..68d8f6cdb 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dns-lexicon', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -41,7 +41,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile new file mode 100644 index 000000000..1b8feb2f8 --- /dev/null +++ b/certbot-dns-rfc2136/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-rfc2136 + +RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-rfc2136/local-oldest-requirements.txt b/certbot-dns-rfc2136/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-rfc2136/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-rfc2136/readthedocs.org.requirements.txt b/certbot-dns-rfc2136/readthedocs.org.requirements.txt new file mode 100644 index 000000000..df89018ce --- /dev/null +++ b/certbot-dns-rfc2136/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-rfc2136[docs] diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index bfa72b50b..3d6b3799b 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,15 +6,14 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'dnspython', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -31,6 +30,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -39,10 +39,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile new file mode 100644 index 000000000..a1b8d6caf --- /dev/null +++ b/certbot-dns-route53/Dockerfile @@ -0,0 +1,5 @@ +FROM certbot/certbot + +COPY . src/certbot-dns-route53 + +RUN pip install --no-cache-dir --editable src/certbot-dns-route53 diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/dns_route53.py index 67462e369..08b1d03f0 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53.py @@ -1,4 +1,5 @@ """Certbot Route53 authenticator plugin.""" +import collections import logging import time @@ -33,6 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") + self._resource_records = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-docstring,no-self-use return "Solve a DNS01 challenge using AWS Route53" @@ -88,6 +90,20 @@ class Authenticator(dns_common.DNSAuthenticator): def _change_txt_record(self, action, validation_domain_name, validation): zone_id = self._find_zone_id_for_domain(validation_domain_name) + rrecords = self._resource_records[validation_domain_name] + challenge = {"Value": '"{0}"'.format(validation)} + if action == "DELETE": + # Remove the record being deleted from the list of tracked records + rrecords.remove(challenge) + if rrecords: + # Need to update instead, as we're not deleting the rrset + action = "UPSERT" + else: + # Create a new list containing the record to use with DELETE + rrecords = [challenge] + else: + rrecords.append(challenge) + response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ @@ -99,11 +115,7 @@ class Authenticator(dns_common.DNSAuthenticator): "Name": validation_domain_name, "Type": "TXT", "TTL": self.ttl, - "ResourceRecords": [ - # For some reason TXT records need to be - # manually quoted. - {"Value": '"{0}"'.format(validation)} - ], + "ResourceRecords": rrecords, } } ] diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index d5f1b2816..7534e132c 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -186,6 +186,48 @@ class ClientTest(unittest.TestCase): call_count = self.client.r53.change_resource_record_sets.call_count self.assertEqual(call_count, 1) + def test_change_txt_record_delete(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + validation = "some-value" + validation_record = {"Value": '"{0}"'.format(validation)} + self.client._resource_records[DOMAIN] = [validation_record] + + self.client._change_txt_record("DELETE", DOMAIN, validation) + + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "DELETE") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [validation_record]) + + def test_change_txt_record_multirecord(self): + self.client._find_zone_id_for_domain = mock.MagicMock() + self.client._get_validation_rrset = mock.MagicMock() + self.client._resource_records[DOMAIN] = [ + {"Value": "\"pre-existing-value\""}, + {"Value": "\"pre-existing-value-two\""}, + ] + self.client.r53.change_resource_record_sets = mock.MagicMock( + return_value={"ChangeInfo": {"Id": 1}}) + + self.client._change_txt_record("DELETE", DOMAIN, "pre-existing-value") + + call_count = self.client.r53.change_resource_record_sets.call_count + call_args = self.client.r53.change_resource_record_sets.call_args_list[0][1] + call_args_batch = call_args["ChangeBatch"]["Changes"][0] + self.assertEqual(call_args_batch["Action"], "UPSERT") + self.assertEqual( + call_args_batch["ResourceRecordSet"]["ResourceRecords"], + [{"Value": "\"pre-existing-value-two\""}]) + + self.assertEqual(call_count, 1) + def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( side_effect=[{"ChangeInfo": {"Status": "PENDING"}}, diff --git a/certbot-dns-route53/local-oldest-requirements.txt b/certbot-dns-route53/local-oldest-requirements.txt new file mode 100644 index 000000000..8368d266e --- /dev/null +++ b/certbot-dns-route53/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +acme[dev]==0.21.1 +certbot[dev]==0.21.1 diff --git a/certbot-dns-route53/readthedocs.org.requirements.txt b/certbot-dns-route53/readthedocs.org.requirements.txt new file mode 100644 index 000000000..660a90d0e --- /dev/null +++ b/certbot-dns-route53/readthedocs.org.requirements.txt @@ -0,0 +1,12 @@ +# readthedocs.org gives no way to change the install command to "pip +# install -e .[docs]" (that would in turn install documentation +# dependencies), but it allows to specify a requirements.txt file at +# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259) + +# Although ReadTheDocs certainly doesn't need to install the project +# in --editable mode (-e), just "pip install .[docs]" does not work as +# expected and "pip install -e .[docs]" must be used instead + +-e acme +-e . +-e certbot-dns-route53[docs] diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8df687972..ad20725b5 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -5,14 +5,14 @@ from setuptools import find_packages version = '0.22.0.dev0' +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + 'acme>=0.21.1', + 'certbot>=0.21.1', 'boto3', 'mock', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -32,10 +33,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index bb2933a39..83e308bac 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,6 +23,7 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants +from certbot_nginx import display_ops from certbot_nginx import nginxparser from certbot_nginx import parser from certbot_nginx import tls_sni_01 @@ -31,16 +32,6 @@ from certbot_nginx import http_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [ - ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - ['\n'] -] - -REDIRECT_COMMENT_BLOCK = [ - ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' return 301 https://$host$request_uri;'], - ['\n'] -] @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) @@ -102,6 +93,11 @@ class NginxConfigurator(common.Installer): # For creating new vhosts if no names match self.new_vhost = None + # List of vhosts configured per wildcard domain on this run. + # used by deploy_cert() and enhance() + self._wildcard_vhosts = {} + self._wildcard_redirect_vhosts = {} + # Add number of outstanding challenges self._chall_out = 0 @@ -156,6 +152,7 @@ class NginxConfigurator(common.Installer): raise errors.PluginError( 'Unable to lock %s', self.conf('server-root')) + # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): @@ -176,14 +173,24 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, create_if_no_match=True) + vhosts = self.choose_vhosts(domain, create_if_no_match=True) + for vhost in vhosts: + self._deploy_cert(vhost, cert_path, key_path, chain_path, fullchain_path) + + def _deploy_cert(self, vhost, cert_path, key_path, chain_path, fullchain_path): + # pylint: disable=unused-argument + """ + Helper function for deploy_cert() that handles the actual deployment + this exists because we might want to do multiple deployments per + domain originally passed for deploy_cert(). This is especially true + with wildcard certificates + """ cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) - logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -191,10 +198,61 @@ class NginxConfigurator(common.Installer): self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path + def _choose_vhosts_wildcard(self, domain, prefer_ssl, no_ssl_filter_port=None): + """Prompts user to choose vhosts to install a wildcard certificate for""" + if prefer_ssl: + vhosts_cache = self._wildcard_vhosts + preference_test = lambda x: x.ssl + else: + vhosts_cache = self._wildcard_redirect_vhosts + preference_test = lambda x: not x.ssl + + # Caching! + if domain in vhosts_cache: + # Vhosts for a wildcard domain were already selected + return vhosts_cache[domain] + + # Get all vhosts whether or not they are covered by the wildcard domain + vhosts = self.parser.get_vhosts() + + # Go through the vhosts, making sure that we cover all the names + # present, but preferring the SSL or non-SSL vhosts + filtered_vhosts = {} + for vhost in vhosts: + # Ensure we're listening non-sslishly on no_ssl_filter_port + if no_ssl_filter_port is not None: + if not self._vhost_listening_on_port_no_ssl(vhost, no_ssl_filter_port): + continue + for name in vhost.names: + if preference_test(vhost): + # Prefer either SSL or non-SSL vhosts + filtered_vhosts[name] = vhost + elif name not in filtered_vhosts: + # Add if not in list previously + filtered_vhosts[name] = vhost + + # Only unique VHost objects + dialog_input = set([vhost for vhost in filtered_vhosts.values()]) + + # Ask the user which of names to enable, expect list of names back + return_vhosts = display_ops.select_vhost_multiple(list(dialog_input)) + + for vhost in return_vhosts: + if domain not in vhosts_cache: + vhosts_cache[domain] = [] + vhosts_cache[domain].append(vhost) + + return return_vhosts + ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, create_if_no_match=False): + def _choose_vhost_single(self, target_name): + matches = self._get_ranked_matches(target_name) + vhosts = [x for x in [self._select_best_name_match(matches)] if x is not None] + return vhosts + + def choose_vhosts(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -212,17 +270,19 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: ssl vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: ssl vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - vhost = None - - matches = self._get_ranked_matches(target_name) - vhost = self._select_best_name_match(matches) - if not vhost: + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to support. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=True) + else: + vhosts = self._choose_vhost_single(target_name) + if not vhosts: if create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name) + # result will not be [None] because it errors on failure + vhosts = [self._vhost_from_duplicated_default(target_name)] else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( @@ -232,10 +292,11 @@ class NginxConfigurator(common.Installer): "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + for vhost in vhosts: + if not vhost.ssl: + self._make_server_ssl(vhost) - return vhost + return vhosts def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) @@ -250,6 +311,9 @@ class NginxConfigurator(common.Installer): configuration, and existence of ipv6only directive for specified port :rtype: tuple of type (bool, bool) """ + # port should be a string, but it's easy to mess up, so let's + # make sure it is one + port = str(port) vhosts = self.parser.get_vhosts() ipv6_active = False ipv6only_present = False @@ -369,7 +433,7 @@ class NginxConfigurator(common.Installer): return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhost(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -387,15 +451,20 @@ class NginxConfigurator(common.Installer): when there is no match found. If we can't choose a default, raise a MisconfigurationError. - :returns: vhost associated with name - :rtype: :class:`~certbot_nginx.obj.VirtualHost` + :returns: vhosts associated with name + :rtype: list of :class:`~certbot_nginx.obj.VirtualHost` """ - matches = self._get_redirect_ranked_matches(target_name, port) - vhost = self._select_best_name_match(matches) - if not vhost and create_if_no_match: - vhost = self._vhost_from_duplicated_default(target_name, port=port) - return vhost + if util.is_wildcard_domain(target_name): + # Ask user which VHosts to enhance. + vhosts = self._choose_vhosts_wildcard(target_name, prefer_ssl=False, + no_ssl_filter_port=port) + else: + matches = self._get_redirect_ranked_matches(target_name, port) + vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] + if not vhosts and create_if_no_match: + vhosts = [self._vhost_from_duplicated_default(target_name, port=port)] + return vhosts def _port_matches(self, test_port, matching_port): # test_port is a number, matching is a number or "" or None @@ -405,6 +474,23 @@ class NginxConfigurator(common.Installer): else: return test_port == matching_port + def _vhost_listening_on_port_no_ssl(self, vhost, port): + found_matching_port = False + if len(vhost.addrs) == 0: + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + found_matching_port = (port == self.DEFAULT_LISTEN_PORT) + else: + for addr in vhost.addrs: + if self._port_matches(port, addr.get_port()) and addr.ssl == False: + found_matching_port = True + + if found_matching_port: + # make sure we don't have an 'ssl on' directive + return not self.parser.has_ssl_on_directive(vhost) + else: + return False + def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name @@ -421,21 +507,7 @@ class NginxConfigurator(common.Installer): all_vhosts = self.parser.get_vhosts() def _vhost_matches(vhost, port): - found_matching_port = False - if len(vhost.addrs) == 0: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and addr.ssl == False: - found_matching_port = True - - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - else: - return False + return self._vhost_listening_on_port_no_ssl(vhost, port) matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)] @@ -571,24 +643,17 @@ class NginxConfigurator(common.Installer): logger.warning("Failed %s for %s", enhancement, domain) raise - def _has_certbot_redirect(self, vhost): - test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + def _has_certbot_redirect(self, vhost, domain): + test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain)) return vhost.contains_list(test_redirect_block) - def _has_certbot_redirect_comment(self, vhost): - test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) - return vhost.contains_list(test_redirect_comment_block) - - def _add_redirect_block(self, vhost, active=True): + def _add_redirect_block(self, vhost, domain): """Add redirect directive to vhost """ - if active: - redirect_block = REDIRECT_BLOCK - else: - redirect_block = REDIRECT_COMMENT_BLOCK + redirect_block = _redirect_block_for_domain(domain) self.parser.add_server_directives( - vhost, redirect_block, replace=False) + vhost, redirect_block, replace=False, insert_at_top=True) def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -604,17 +669,32 @@ class NginxConfigurator(common.Installer): """ port = self.DEFAULT_LISTEN_PORT - vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. - vhost = self.choose_redirect_vhost(domain, port) + vhosts = self.choose_redirect_vhosts(domain, port) - if vhost is None: + if not vhosts: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) return + for vhost in vhosts: + self._enable_redirect_single(domain, vhost) + + def _enable_redirect_single(self, domain, vhost): + """Redirect all equivalent HTTP traffic to ssl_vhost. + + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. + + .. note:: This function saves the configuration + + :param str domain: domain to enable redirect for + :param `~obj.Vhost` vhost: vhost to enable redirect for + """ + + new_vhost = None if vhost.ssl: new_vhost = self.parser.duplicate_vhost(vhost, only_directives=['listen', 'server_name']) @@ -631,20 +711,18 @@ class NginxConfigurator(common.Installer): # remove all non-ssl addresses from the existing block self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + # Add this at the bottom to get the right order of directives + return_404_directive = [['\n ', 'return', ' ', '404']] + self.parser.add_server_directives(new_vhost, return_404_directive, replace=False) + vhost = new_vhost - if self._has_certbot_redirect(vhost): + if self._has_certbot_redirect(vhost, domain): logger.info("Traffic on port %s already redirecting to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) else: # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) + self._add_redirect_block(vhost, domain) logger.info("Redirecting all traffic on port %s to ssl in %s", self.DEFAULT_LISTEN_PORT, vhost.filep) @@ -656,7 +734,18 @@ class NginxConfigurator(common.Installer): :type chain_path: `str` or `None` """ - vhost = self.choose_vhost(domain) + vhosts = self.choose_vhosts(domain) + for vhost in vhosts: + self._enable_ocsp_stapling_single(vhost, chain_path) + + def _enable_ocsp_stapling_single(self, vhost, chain_path): + """Include OCSP response in TLS handshake + + :param str vhost: vhost to enable OCSP response for + :param chain_path: chain file path + :type chain_path: `str` or `None` + + """ if self.version < (1, 3, 7): raise errors.PluginError("Version 1.3.7 or greater of nginx " "is needed to enable OCSP stapling") @@ -907,6 +996,23 @@ def _test_block_from_block(block): parser.comment_directive(test_block, 0) return test_block[:-1] + +def _redirect_block_for_domain(domain): + updated_domain = domain + match_symbol = '=' + if util.is_wildcard_domain(domain): + match_symbol = '~' + updated_domain = updated_domain.replace('.', r'\.') + updated_domain = updated_domain.replace('*', '[^.]+') + updated_domain = '^' + updated_domain + '$' + redirect_block = [[ + ['\n ', 'if', ' ', '($host', ' ', match_symbol, ' ', '%s)' % updated_domain, ' '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + '\n ']], + ['\n']] + return redirect_block + + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/display_ops.py b/certbot-nginx/certbot_nginx/display_ops.py new file mode 100644 index 000000000..5d6bda6b0 --- /dev/null +++ b/certbot-nginx/certbot_nginx/display_ops.py @@ -0,0 +1,44 @@ +"""Contains UI methods for Nginx operations.""" +import logging + +import zope.component + +from certbot import interfaces + +import certbot.display.util as display_util + + +logger = logging.getLogger(__name__) + + +def select_vhost_multiple(vhosts): + """Select multiple Vhosts to install the certificate for + :param vhosts: Available Nginx VirtualHosts + :type vhosts: :class:`list` of type `~obj.Vhost` + :returns: List of VirtualHosts + :rtype: :class:`list`of type `~obj.Vhost` + """ + if not vhosts: + return list() + tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] + # Remove the extra newline from the last entry + if len(tags_list): + tags_list[-1] = tags_list[-1][:-1] + code, names = zope.component.getUtility(interfaces.IDisplay).checklist( + "Which server blocks would you like to modify?", + tags=tags_list, force_interactive=True) + if code == display_util.OK: + return_vhosts = _reversemap_vhosts(names, vhosts) + return return_vhosts + return [] + +def _reversemap_vhosts(names, vhosts): + """Helper function for select_vhost_multiple for mapping string + representations back to actual vhost objects""" + return_vhosts = list() + + for selection in names: + for vhost in vhosts: + if vhost.display_repr().strip() == selection.strip(): + return_vhosts.append(vhost) + return return_vhosts diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py index c0dec061a..0b1b2bfe0 100644 --- a/certbot-nginx/certbot_nginx/http_01.py +++ b/certbot-nginx/certbot_nginx/http_01.py @@ -179,13 +179,17 @@ class NginxHttp01(common.ChallengePerformer): """ try: - vhost = self.configurator.choose_redirect_vhost(achall.domain, + vhosts = self.configurator.choose_redirect_vhosts(achall.domain, '%i' % self.configurator.config.http01_port, create_if_no_match=True) except errors.MisconfigurationError: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block return self._make_server_block(achall) + # len is max 1 because Nginx doesn't authenticate wildcards + # if len were or vhosts None, we would have errored + vhost = vhosts[0] + # Modify existing server block validation = achall.validation(achall.account_key) validation_path = self._get_validation_path(achall) diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index f5ac5c2e3..3625a95b9 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -193,14 +193,10 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods return False - def has_redirect(self): - """Determine if this vhost has a redirecting statement - """ - for directive_name in REDIRECT_DIRECTIVES: - found = _find_directive(self.raw, directive_name) - if found is not None: - return True - return False + def __hash__(self): + return hash((self.filep, tuple(self.path), + tuple(self.addrs), tuple(self.names), + self.ssl, self.enabled)) def contains_list(self, test): """Determine if raw server block contains test list at top level @@ -226,14 +222,14 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods if not a.ipv6: return True -def _find_directive(directives, directive_name): - """Find a directive of type directive_name in directives - """ - if not directives or isinstance(directives, six.string_types) or len(directives) == 0: - return None - - if directives[0] == directive_name: - return directives - - matches = (_find_directive(line, directive_name) for line in directives) - return next((m for m in matches if m is not None), None) + def display_repr(self): + """Return a representation of VHost to be used in dialog""" + return ( + "File: {filename}\n" + "Addresses: {addrs}\n" + "Names: {names}\n" + "HTTPS: {https}\n".format( + filename=self.filep, + addrs=", ".join(str(addr) for addr in self.addrs), + names=", ".join(self.names), + https="Yes" if self.ssl else "No")) diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 7475df40c..bffaef5e4 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser +from certbot_nginx.configurator import _redirect_block_for_domain +from certbot_nginx.nginxparser import UnspacedList from certbot_nginx.tests import util @@ -126,7 +128,7 @@ class NginxConfiguratorTest(util.NginxTest): ['#', parser.COMMENT]]]], parsed[0]) - def test_choose_vhost(self): + def test_choose_vhosts(self): localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.']) server_conf = set(['somename', 'another.alias', 'alias']) example_conf = set(['.example.com', 'example.*']) @@ -157,7 +159,7 @@ class NginxConfiguratorTest(util.NginxTest): '69.255.225.155'] for name in results: - vhost = self.config.choose_vhost(name) + vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) self.assertEqual(results[name], vhost.names) @@ -171,7 +173,7 @@ class NginxConfiguratorTest(util.NginxTest): for name in bad_results: self.assertRaises(errors.MisconfigurationError, - self.config.choose_vhost, name) + self.config.choose_vhosts, name) def test_ipv6only(self): # ipv6_info: (ipv6_active, ipv6only_present) @@ -179,6 +181,18 @@ class NginxConfiguratorTest(util.NginxTest): # Port 443 has ipv6only=on because of ipv6ssl.com vhost self.assertEquals((True, True), self.config.ipv6_info("443")) + def test_ipv6only_detection(self): + self.config.version = (1, 3, 1) + + self.config.deploy_cert( + "ipv6.com", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + for addr in self.config.choose_vhosts("ipv6.com")[0].addrs: + self.assertFalse(addr.ipv6only) def test_more_info(self): self.assertTrue('nginx.conf' in self.config.more_info()) @@ -447,7 +461,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -460,6 +474,8 @@ class NginxConfiguratorTest(util.NginxTest): migration_conf = self.config.parser.abs_path('sites-enabled/migration.com') self.config.enhance("migration.com", "redirect") + expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0] + generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -484,101 +500,27 @@ class NginxConfiguratorTest(util.NginxTest): ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], [], []]], [['server'], [ + [['if', '($host', '=', 'www.example.com)'], [ + ['return', '301', 'https://$host$request_uri']]], + ['#', ' managed by Certbot'], [], ['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], - [], []]]], + ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]], generated_conf) @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): + def test_certbot_redirect_exists(self, mock_contains_list): # Test that we add no redirect statement if there is already a # redirect in the block that is managed by certbot # Has a certbot redirect - mock_has_redirect.return_value = True mock_contains_list.return_value = True with mock.patch("certbot_nginx.configurator.logger") as mock_logger: self.config.enhance("www.example.com", "redirect") self.assertEqual(mock_logger.info.call_args[0][0], "Traffic on port %s already redirecting to ssl in %s") - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): - # Test that we add a redirect as a comment if there is already a - # redirect-class statement in the block that isn't managed by certbot - example_conf = self.config.parser.abs_path('sites-enabled/example.com') - - self.config.deploy_cert( - "example.org", - "example/cert.pem", - "example/key.pem", - "example/chain.pem", - "example/fullchain.pem") - - # Has a non-Certbot redirect, and has no existing comment - mock_contains_list.return_value = False - mock_has_redirect.return_value = True - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertEqual(mock_logger.info.call_args[0][0], - "The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.") - generated_conf = self.config.parser.parsed[example_conf] - expected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' return 301 https://$host$request_uri;'], - ] - for line in expected: - self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) - - @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') - @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment') - @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block') - def test_redirect_comment_exists(self, mock_add_redirect_block, - mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list): - # Test that we add nothing if there is a non-Certbot redirect and a - # preexisting comment - # Has a non-Certbot redirect and a comment - mock_has_redirect.return_value = True - mock_contains_list.return_value = False # self._has_certbot_redirect(vhost): - mock_has_cb_redirect_comment.return_value = True - - # assert _add_redirect_block not called - with mock.patch("certbot_nginx.configurator.logger") as mock_logger: - self.config.enhance("www.example.com", "redirect") - self.assertFalse(mock_add_redirect_block.called) - self.assertTrue(mock_logger.info.called) - def test_redirect_dont_enhance(self): # Test that we don't accidentally add redirect to ssl-only block with mock.patch("certbot_nginx.configurator.logger") as mock_logger: @@ -586,22 +528,18 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_logger.info.call_args[0][0], 'No matching insecure server blocks listening on port %s found.') - def test_no_double_redirect(self): - # Test that we don't also add the commented redirect if we've just added - # a redirect to that vhost this run + def test_double_redirect(self): + # Test that we add one redirect for each domain example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("example.com", "redirect") self.config.enhance("example.org", "redirect") - unexpected = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] - ] + expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0] + expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0] + generated_conf = self.config.parser.parsed[example_conf] - for line in unexpected: - self.assertFalse(util.contains_at_depth(generated_conf, line, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2)) + self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2)) def test_staple_ocsp_bad_version(self): self.config.version = (1, 3, 1) @@ -763,7 +701,7 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = ['return', '301', 'https://$host$request_uri'] + expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) @@ -776,6 +714,100 @@ class NginxConfiguratorTest(util.NginxTest): self.config.rollback_checkpoints() self.assertTrue(mock_parser_load.call_count == 3) + def test_choose_vhosts_wildcard(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=True) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_choose_vhosts_wildcard_redirect(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_select_vhs.return_value = [vhost] + vhs = self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False) + # Check that the dialog was called with migration.com + self.assertTrue(vhost in mock_select_vhs.call_args[0][0]) + + # And the actual returned values + self.assertEquals(len(vhs), 1) + self.assertEqual(vhs[0], vhost) + + def test_deploy_cert_wildcard(self): + # pylint: disable=protected-access + mock_choose_vhosts = mock.MagicMock() + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + mock_choose_vhosts.return_value = [vhost] + self.config._choose_vhosts_wildcard = mock_choose_vhosts + mock_d = "certbot_nginx.configurator.NginxConfigurator._deploy_cert" + with mock.patch(mock_d) as mock_dep: + self.config.deploy_cert("*.com", "/tmp/path", + "/tmp/path", "/tmp/path", "/tmp/path") + self.assertTrue(mock_dep.called) + self.assertEquals(len(mock_dep.call_args_list), 1) + self.assertEqual(vhost, mock_dep.call_args_list[0][0][0]) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog): + # pylint: disable=protected-access + mock_dialog.return_value = [] + self.assertRaises(errors.PluginError, + self.config.deploy_cert, + "*.wild.cat", "/tmp/path", "/tmp/path", + "/tmp/path", "/tmp/path") + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_ocsp_after_install(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'geese.com' in x.names][0] + self.config._wildcard_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertFalse(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_redirect_or_ocsp_no_install(self, mock_dialog): + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + mock_dialog.return_value = [vhost] + self.config.enhance("*.com", "staple-ocsp", "example/chain.pem") + self.assertTrue(mock_dialog.called) + + @mock.patch("certbot_nginx.display_ops.select_vhost_multiple") + def test_enhance_wildcard_double_redirect(self, mock_dialog): + # pylint: disable=protected-access + vhost = [x for x in self.config.parser.get_vhosts() + if 'summer.com' in x.names][0] + self.config._wildcard_redirect_vhosts["*.com"] = [vhost] + self.config.enhance("*.com", "redirect") + self.assertFalse(mock_dialog.called) + + def test_choose_vhosts_wildcard_no_ssl_filter_port(self): + # pylint: disable=protected-access + mock_path = "certbot_nginx.display_ops.select_vhost_multiple" + with mock.patch(mock_path) as mock_select_vhs: + mock_select_vhs.return_value = [] + self.config._choose_vhosts_wildcard("*.com", + prefer_ssl=False, + no_ssl_filter_port='80') + # Check that the dialog was called with only port 80 vhosts + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 4) + + class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/display_ops_test.py b/certbot-nginx/certbot_nginx/tests/display_ops_test.py new file mode 100644 index 000000000..e3c6fb66b --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/display_ops_test.py @@ -0,0 +1,45 @@ +"""Test certbot_apache.display_ops.""" +import unittest + +from certbot.display import util as display_util + +from certbot.tests import util as certbot_util + +from certbot_nginx import parser + +from certbot_nginx.display_ops import select_vhost_multiple +from certbot_nginx.tests import util + + +class SelectVhostMultiTest(util.NginxTest): + """Tests for certbot_nginx.display_ops.select_vhost_multiple.""" + + def setUp(self): + super(SelectVhostMultiTest, self).setUp() + nparser = parser.NginxParser(self.config_path) + self.vhosts = nparser.get_vhosts() + + def test_select_no_input(self): + self.assertFalse(select_vhost_multiple([])) + + @certbot_util.patch_get_utility() + def test_select_correct(self, mock_util): + mock_util().checklist.return_value = ( + display_util.OK, [self.vhosts[3].display_repr(), + self.vhosts[2].display_repr()]) + vhs = select_vhost_multiple([self.vhosts[3], + self.vhosts[2], + self.vhosts[1]]) + self.assertTrue(self.vhosts[2] in vhs) + self.assertTrue(self.vhosts[3] in vhs) + self.assertFalse(self.vhosts[1] in vhs) + + @certbot_util.patch_get_utility() + def test_select_cancel(self, mock_util): + mock_util().checklist.return_value = (display_util.CANCEL, "whatever") + vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]]) + self.assertFalse(vhs) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index 92cb0e086..b30338b5b 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -162,17 +162,15 @@ class VirtualHostTest(unittest.TestCase): 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) - def test_has_redirect(self): - self.assertTrue(self.vhost1.has_redirect()) - self.assertTrue(self.vhost2.has_redirect()) - self.assertTrue(self.vhost3.has_redirect()) - self.assertFalse(self.vhost4.has_redirect()) - def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block - test_needle = _test_block_from_block(REDIRECT_BLOCK) + from certbot_nginx.configurator import _test_block_from_block + test_block = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] + ] + test_needle = _test_block_from_block(test_block) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com index d8f7eff12..875a9ee1b 100644 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/sites-enabled/ipv6ssl.com @@ -1,5 +1,7 @@ server { listen 443 ssl; listen [::]:443 ssl ipv6only=on; + listen 5001 ssl; + listen [::]:5001 ssl ipv6only=on; server_name ipv6ssl.com; } diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 61ee293fa..72b65911c 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -61,10 +61,10 @@ class TlsSniPerformTest(util.NginxTest): shutil.rmtree(self.work_dir) @mock.patch("certbot_nginx.configurator" - ".NginxConfigurator.choose_vhost") + ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): self.sni.add_chall(self.achalls[1]) - mock_choose.return_value = None + mock_choose.return_value = [] result = self.sni.perform() self.assertFalse(result is None) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index eca198bfe..0fd37e0cb 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,10 +55,11 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) + vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True) - if vhost is not None and vhost.addrs: - addresses.append(list(vhost.addrs)) + # len is max 1 because Nginx doesn't authenticate wildcards + if vhosts and vhosts[0].addrs: + addresses.append(list(vhosts[0].addrs)) else: if ipv6: # If IPv6 is active in Nginx configuration diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-nginx/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 152f77de8..bb71cf19a 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,16 +6,18 @@ from setuptools import find_packages version = '0.22.0.dev0' -# Please update tox.ini when modifying dependency version requirements +# Remember to update local-oldest-requirements.txt when changing the minimum +# acme/certbot version. install_requires = [ - 'acme=={0}'.format(version), - 'certbot=={0}'.format(version), + # This plugin works with an older version of acme, but Certbot does not. + # 0.22.0 is specified here to work around + # https://github.com/pypa/pip/issues/988. + 'acme>0.21.1', + 'certbot>0.21.1', 'mock', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.interface', ] @@ -32,6 +34,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Plugins', @@ -40,10 +43,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/certbot/account.py b/certbot/account.py index 41e980097..70d9a7fc3 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -223,12 +223,17 @@ class AccountFileStorage(interfaces.AccountStorage): try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr = account.regr - with_uri = RegistrationResourceWithNewAuthzrURI( - new_authzr_uri=acme.directory.new_authz, - body=regr.body, - uri=regr.uri, - terms_of_service=regr.terms_of_service) - regr_file.write(with_uri.json_dumps()) + # If we have a value for new-authz, save it for forwards + # compatibility with older versions of Certbot. If we don't + # have a value for new-authz, this is an ACMEv2 directory where + # an older version of Certbot won't work anyway. + if hasattr(acme.directory, "new-authz"): + regr = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(regr.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 5f520cbcb..51cdf09ee 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,4 +1,5 @@ """ACME AuthHandler.""" +import collections import logging import time @@ -17,6 +18,10 @@ from certbot import interfaces logger = logging.getLogger(__name__) +AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) +"""Stores an authorization resource and its active annotated challenges.""" + + class AuthHandler(object): """ACME Authorization Handler for a client. @@ -24,15 +29,11 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.Client acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` - :ivar dict authzr: ACME Authorization Resource dict where keys are domains - and values are :class:`acme.messages.AuthorizationResource` - :ivar list achalls: DV challenges in the form of - :class:`certbot.achallenges.AnnotatedChallenge` :ivar list pref_challs: sorted user specified preferred challenges type strings with the most preferred challenge listed first @@ -42,18 +43,15 @@ class AuthHandler(object): self.acme = acme self.account = account - self.authzr = dict() self.pref_challs = pref_challs - # List must be used to keep responses straight. - self.achalls = [] - - def get_authorizations(self, domains, best_effort=False): + def handle_authorizations(self, orderr, best_effort=False): """Retrieve all authorizations for challenges. - :param list domains: Domains for authorization + :param acme.messages.OrderResource orderr: must have + authorizations filled in :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) + required (this is useful in renewal) :returns: List of authorization resources :rtype: list @@ -62,30 +60,30 @@ class AuthHandler(object): authorizations """ - for domain in domains: - self.authzr[domain] = self.acme.request_domain_challenges(domain) + aauthzrs = [AnnotatedAuthzr(authzr, []) + for authzr in orderr.authorizations] - self._choose_challenges(domains) + self._choose_challenges(aauthzrs) config = zope.component.getUtility(interfaces.IConfig) notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... - while self.achalls: - resp = self._solve_challenges() + while self._has_challenges(aauthzrs): + resp = self._solve_challenges(aauthzrs) logger.info("Waiting for verification...") if config.debug_challenges: notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls - self._respond(resp, best_effort) + self._respond(aauthzrs, resp, best_effort) # Just make sure all decisions are complete. - self.verify_authzr_complete() + self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [authzr for authzr in self.authzr.values() - if authzr.body.status == messages.STATUS_VALID] + retVal = [aauthzr.authzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] if not retVal: raise errors.AuthorizationError( @@ -93,36 +91,55 @@ class AuthHandler(object): return retVal - def _choose_challenges(self, domains): + def _choose_challenges(self, aauthzrs): """Retrieve necessary challenges to satisfy server.""" logger.info("Performing the following challenges:") - for dom in domains: + for aauthzr in aauthzrs: + aauthzr_challenges = aauthzr.authzr.body.challenges + if self.acme.acme_version == 1: + combinations = aauthzr.authzr.body.combinations + else: + combinations = tuple((i,) for i in range(len(aauthzr_challenges))) + path = gen_challenge_path( - self.authzr[dom].body.challenges, - self._get_chall_pref(dom), - self.authzr[dom].body.combinations) + aauthzr_challenges, + self._get_chall_pref(aauthzr.authzr.body.identifier.value), + combinations) - dom_achalls = self._challenge_factory( - dom, path) - self.achalls.extend(dom_achalls) + aauthzr_achalls = self._challenge_factory( + aauthzr.authzr, path) + aauthzr.achalls.extend(aauthzr_achalls) - def _solve_challenges(self): + def _has_challenges(self, aauthzrs): + """Do we have any challenges to perform?""" + return any(aauthzr.achalls for aauthzr in aauthzrs) + + def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" resp = [] - with error_handler.ErrorHandler(self._cleanup_challenges): + all_achalls = self._get_all_achalls(aauthzrs) + with error_handler.ErrorHandler(self._cleanup_challenges, all_achalls): try: - if self.achalls: - resp = self.auth.perform(self.achalls) + if all_achalls: + resp = self.auth.perform(all_achalls) except errors.AuthorizationError: logger.critical("Failure in setting up challenges.") logger.info("Attempting to clean up outstanding challenges...") raise - assert len(resp) == len(self.achalls) + assert len(resp) == len(all_achalls) return resp - def _respond(self, resp, best_effort): + def _get_all_achalls(self, aauthzrs): + """Return all active challenges.""" + all_achalls = [] + for aauthzr in aauthzrs: + all_achalls.extend(aauthzr.achalls) + + return all_achalls + + def _respond(self, aauthzrs, resp, best_effort): """Send/Receive confirmation of all challenges. .. note:: This method also cleans up the auth_handler state. @@ -130,69 +147,70 @@ class AuthHandler(object): """ # TODO: chall_update is a dirty hack to get around acme-spec #105 chall_update = dict() - active_achalls = self._send_responses(self.achalls, - resp, chall_update) + active_achalls = self._send_responses(aauthzrs, resp, chall_update) # Check for updated status... try: - self._poll_challenges(chall_update, best_effort) + self._poll_challenges(aauthzrs, chall_update, best_effort) finally: - # This removes challenges from self.achalls - self._cleanup_challenges(active_achalls) + self._cleanup_challenges(aauthzrs, active_achalls) - def _send_responses(self, achalls, resps, chall_update): + def _send_responses(self, aauthzrs, resps, chall_update): """Send responses and make sure errors are handled. + :param aauthzrs: authorizations and the selected annotated challenges + to try and perform + :type aauthzrs: `list` of `AnnotatedAuthzr` :param dict chall_update: parameter that is updated to hold - authzr -> list of outstanding solved annotated challenges + aauthzr index to list of outstanding solved annotated challenges """ active_achalls = [] - for achall, resp in six.moves.zip(achalls, resps): - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) + resps_iter = iter(resps) + for i, aauthzr in enumerate(aauthzrs): + for achall in aauthzr.achalls: + # This line needs to be outside of the if block below to + # ensure failed challenges are cleaned up correctly + active_achalls.append(achall) - # Don't send challenges for None and False authenticator responses - if resp is not None and resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - if achall.domain in chall_update: - chall_update[achall.domain].append(achall) - else: - chall_update[achall.domain] = [achall] + resp = next(resps_iter) + # Don't send challenges for None and False authenticator responses + if resp: + self.acme.answer_challenge(achall.challb, resp) + # TODO: answer_challenge returns challr, with URI, + # that can be used in _find_updated_challr + # comparisons... + chall_update.setdefault(i, []).append(achall) return active_achalls - def _poll_challenges( - self, chall_update, best_effort, min_sleep=3, max_rounds=15): + def _poll_challenges(self, aauthzrs, chall_update, + best_effort, min_sleep=3, max_rounds=15): """Wait for all challenge results to be determined.""" - dom_to_check = set(chall_update.keys()) - comp_domains = set() + indices_to_check = set(chall_update.keys()) + comp_indices = set() rounds = 0 - while dom_to_check and rounds < max_rounds: + while indices_to_check and rounds < max_rounds: # TODO: Use retry-after... time.sleep(min_sleep) all_failed_achalls = set() - for domain in dom_to_check: + for index in indices_to_check: comp_achalls, failed_achalls = self._handle_check( - domain, chall_update[domain]) + aauthzrs, index, chall_update[index]) - if len(comp_achalls) == len(chall_update[domain]): - comp_domains.add(domain) + if len(comp_achalls) == len(chall_update[index]): + comp_indices.add(index) elif not failed_achalls: for achall, _ in comp_achalls: - chall_update[domain].remove(achall) + chall_update[index].remove(achall) # We failed some challenges... damage control else: if best_effort: - comp_domains.add(domain) + comp_indices.add(index) logger.warning( "Challenge failed for domain %s", - domain) + aauthzrs[index].authzr.body.identifier.value) else: all_failed_achalls.update( updated for _, updated in failed_achalls) @@ -201,24 +219,26 @@ class AuthHandler(object): _report_failed_challs(all_failed_achalls) raise errors.FailedChallenges(all_failed_achalls) - dom_to_check -= comp_domains - comp_domains.clear() + indices_to_check -= comp_indices + comp_indices.clear() rounds += 1 - def _handle_check(self, domain, achalls): + def _handle_check(self, aauthzrs, index, achalls): """Returns tuple of ('completed', 'failed').""" completed = [] failed = [] - self.authzr[domain], _ = self.acme.poll(self.authzr[domain]) - if self.authzr[domain].body.status == messages.STATUS_VALID: + original_aauthzr = aauthzrs[index] + updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) + aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) + if updated_authzr.body.status == messages.STATUS_VALID: return achalls, [] # Note: if the whole authorization is invalid, the individual failed # challenges will be determined here... for achall in achalls: updated_achall = achall.update(challb=self._find_updated_challb( - self.authzr[domain], achall)) + updated_authzr, achall)) # This does nothing for challenges that have yet to be decided yet. if updated_achall.status == messages.STATUS_VALID: @@ -267,7 +287,7 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, achall_list=None): + def _cleanup_challenges(self, aauthzrs, achall_list=None): """Cleanup challenges. If achall_list is not provided, cleanup all achallenges. @@ -276,31 +296,39 @@ class AuthHandler(object): logger.info("Cleaning up challenges") if achall_list is None: - achalls = self.achalls + achalls = self._get_all_achalls(aauthzrs) else: achalls = achall_list if achalls: self.auth.cleanup(achalls) for achall in achalls: - self.achalls.remove(achall) + for aauthzr in aauthzrs: + if achall in aauthzr.achalls: + aauthzr.achalls.remove(achall) + break - def verify_authzr_complete(self): + def verify_authzr_complete(self, aauthzrs): """Verifies that all authorizations have been decided. + :param aauthzrs: authorizations and their selected annotated + challenges + :type aauthzrs: `list` of `AnnotatedAuthzr` + :returns: Whether all authzr are complete :rtype: bool """ - for authzr in self.authzr.values(): + for aauthzr in aauthzrs: + authzr = aauthzr.authzr if (authzr.body.status != messages.STATUS_VALID and authzr.body.status != messages.STATUS_INVALID): raise errors.AuthorizationError("Incomplete authorizations") - def _challenge_factory(self, domain, path): + def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges - :param str domain: domain of the enrollee + :param messages.AuthorizationResource authzr: authorization :param list path: List of indices from `challenges`. @@ -314,8 +342,9 @@ class AuthHandler(object): achalls = [] for index in path: - challb = self.authzr[domain].body.challenges[index] - achalls.append(challb_to_achall(challb, self.account.key, domain)) + challb = authzr.body.challenges[index] + achalls.append(challb_to_achall( + challb, self.account.key, authzr.body.identifier.value)) return achalls @@ -408,7 +437,7 @@ def _find_smart_path(challbs, preferences, combinations): combo_total = 0 if not best_combo: - _report_no_chall_path() + _report_no_chall_path(challbs) return best_combo @@ -429,15 +458,23 @@ def _find_dumb_path(challbs, preferences): if supported: path.append(i) else: - _report_no_chall_path() + _report_no_chall_path(challbs) return path -def _report_no_chall_path(): - """Logs and raises an error that no satisfiable chall path exists.""" +def _report_no_chall_path(challbs): + """Logs and raises an error that no satisfiable chall path exists. + + :param challbs: challenges from the authorization that can't be satisfied + + """ msg = ("Client with the currently selected authenticator does not support " "any combination of challenges that will satisfy the CA.") + if len(challbs) == 1 and isinstance(challbs[0].chall, challenges.DNS01): + msg += ( + " You may need to use an authenticator " + "plugin that can do challenges over DNS.") logger.fatal(msg) raise errors.AuthorizationError(msg) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index da85ed783..4240a0523 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -121,7 +121,28 @@ def domains_for_certname(config, certname): return lineage.names() if lineage else None def find_duplicative_certs(config, domains): - """Find existing certs that duplicate the request.""" + """Find existing certs that match the given domain names. + + This function searches for certificates whose domains are equal to + the `domains` parameter and certificates whose domains are a subset + of the domains in the `domains` parameter. If multiple certificates + are found whose names are a subset of `domains`, the one whose names + are the largest subset of `domains` is returned. + + If multiple certificates' domains are an exact match or equally + sized subsets, which matching certificates are returned is + undefined. + + :param config: Configuration. + :type config: :class:`certbot.configuration.NamespaceConfig` + :param domains: List of domain names + :type domains: `list` of `str` + + :returns: lineages representing the identically matching cert and the + largest subset if they exist + :rtype: `tuple` of `storage.RenewableCert` or `None` + + """ def update_certs_for_domain_matches(candidate_lineage, rv): """Return cert as identical_names_cert if it matches, or subset_names_cert if it matches as subset diff --git a/certbot/cli.py b/certbot/cli.py index 27a2af3d4..06ba41124 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -207,13 +207,15 @@ def set_by_cli(var): # propagate plugin requests: eg --standalone modifies config.authenticator detector.authenticator, detector.installer = ( plugin_selection.cli_plugin_requests(detector)) - logger.debug("Default Detector is %r", detector) if not isinstance(getattr(detector, var), _Default): + logger.debug("Var %s=%s (set by user).", var, getattr(detector, var)) return True for modifier in VAR_MODIFIERS.get(var, []): if set_by_cli(modifier): + logger.debug("Var %s=%s (set by user).", + var, VAR_MODIFIERS.get(var, [])) return True return False @@ -597,6 +599,11 @@ class HelpfulArgumentParser(object): if parsed_args.validate_hooks: hooks.validate_hooks(parsed_args) + if parsed_args.allow_subset_of_names: + if any(util.is_wildcard_domain(d) for d in parsed_args.domains): + raise errors.Error("Using --allow-subset-of-names with a" + " wildcard domain is not supported.") + possible_deprecation_warning(parsed_args) return parsed_args @@ -1295,14 +1302,13 @@ def _paths_parser(helpful): elif verb == "revoke": add(section, "--cert-path", type=read_file, required=True, help=cph) else: - add(section, "--cert-path", type=os.path.abspath, - help=cph, required=(verb == "install")) + add(section, "--cert-path", type=os.path.abspath, help=cph) section = "paths" if verb in ("install", "revoke"): section = verb # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", required=(verb == "install"), + add(section, "--key-path", type=((verb == "revoke" and read_file) or os.path.abspath), help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/client.py b/certbot/client.py index b735421f5..2992c0cec 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -1,4 +1,5 @@ """Certbot client API.""" +import datetime import logging import os import platform @@ -37,12 +38,12 @@ from certbot.plugins import selection as plugin_selection logger = logging.getLogger(__name__) -def acme_from_config_key(config, key): +def acme_from_config_key(config, key, regr=None): "Wrangle ACME client construction" # TODO: Allow for other alg types besides RS256 - net = acme_client.ClientNetwork(key, verify_ssl=(not config.no_verify_ssl), + net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl), user_agent=determine_user_agent(config)) - return acme_client.Client(config.server, key=key, net=net) + return acme_client.BackwardsCompatibleClientV2(net, key, config.server) def determine_user_agent(config): @@ -162,14 +163,7 @@ def register(config, account_storage, tos_cb=None): backend=default_backend()))) acme = acme_from_config_key(config, key) # TODO: add phone? - regr = perform_registration(acme, config) - - if regr.terms_of_service is not None: - if tos_cb is not None and not tos_cb(regr): - raise errors.Error( - "Registration cannot proceed without accepting " - "Terms of Service.") - regr = acme.agree_to_tos(regr) + regr = perform_registration(acme, config, tos_cb) acc = account.Account(regr, key) account.report_new_account(config) @@ -180,7 +174,7 @@ def register(config, account_storage, tos_cb=None): return acc, acme -def perform_registration(acme, config): +def perform_registration(acme, config, tos_cb): """ Actually register new account, trying repeatedly if there are email problems @@ -192,7 +186,8 @@ def perform_registration(acme, config): :rtype: `acme.messages.RegistrationResource` """ try: - return acme.register(messages.NewRegistration.from_data(email=config.email)) + return acme.new_account_and_tos(messages.NewRegistration.from_data(email=config.email), + tos_cb) except messages.Error as e: if e.code == "invalidEmail" or e.code == "invalidContact": if config.noninteractive_mode: @@ -202,7 +197,7 @@ def perform_registration(acme, config): raise errors.Error(msg) else: config.email = display_ops.get_email(invalid=True) - return perform_registration(acme, config) + return perform_registration(acme, config, tos_cb) else: raise @@ -218,8 +213,8 @@ class Client(object): :ivar .IAuthenticator auth: Prepared (`.IAuthenticator.prepare`) authenticator that can solve ACME challenges. :ivar .IInstaller installer: Installer. - :ivar acme.client.Client acme: Optional ACME client API handle. - You might already have one from `register`. + :ivar acme.client.BackwardsCompatibleClientV2 acme: Optional ACME + client API handle. You might already have one from `register`. """ @@ -232,7 +227,7 @@ class Client(object): # Initialize ACME if account is provided if acme is None and self.account is not None: - acme = acme_from_config_key(config, self.account.key) + acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme if auth is not None: @@ -241,21 +236,15 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, authzr=None): + def obtain_certificate_from_csr(self, csr, orderr=None): """Obtain certificate. - Internal function with precondition that `domains` are - consistent with identifiers present in the `csr`. - - :param list domains: Domain names. :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. - :param list authzr: List of - :class:`acme.messages.AuthorizationResource` + :param acme.messages.OrderResource orderr: contains authzrs - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). + :returns: certificate and chain as PEM byte strings :rtype: tuple """ @@ -267,37 +256,15 @@ class Client(object): if self.account.regr is None: raise errors.Error("Please register with the ACME server first.") - logger.debug("CSR: %s, domains: %s", csr, domains) + logger.debug("CSR: %s", csr) - if authzr is None: - authzr = self.auth_handler.get_authorizations(domains) + if orderr is None: + orderr = self._get_order_and_authorizations(csr.data, best_effort=False) - certr = self.acme.request_issuance( - jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), - authzr) - - notify = zope.component.getUtility(interfaces.IDisplay).notification - retries = 0 - chain = None - - while retries <= 1: - if retries: - notify('Failed to fetch chain, please check your network ' - 'and continue', pause=True) - try: - chain = self.acme.fetch_chain(certr) - break - except acme_errors.Error: - logger.debug('Failed to fetch chain', exc_info=True) - retries += 1 - - if chain is None: - raise acme_errors.Error( - 'Failed to fetch chain. You should not deploy the generated ' - 'certificate, please rerun the command for a new one.') - - return certr, chain + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = self.acme.finalize_order(orderr, deadline) + cert, chain = crypto_util.cert_and_chain_from_fullchain(orderr.fullchain_pem) + return cert.encode(), chain.encode() def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -306,20 +273,12 @@ class Client(object): :param list domains: domains to get a certificate - :returns: `.CertificateResource`, certificate chain (as - returned by `.fetch_chain`), and newly generated private key - (`.util.Key`) and DER-encoded Certificate Signing Request - (`.util.CSR`). + :returns: certificate as PEM string, chain as PEM string, + newly generated private key (`.util.Key`), and DER-encoded + Certificate Signing Request (`.util.CSR`). :rtype: tuple """ - authzr = self.auth_handler.get_authorizations( - domains, - self.config.allow_subset_of_names) - - auth_domains = set(a.body.identifier.value for a in authzr) - domains = [d for d in domains if d in auth_domains] - # Create CSR from names if self.config.dry_run: key = util.Key(file=None, @@ -332,10 +291,44 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - certr, chain = self.obtain_certificate_from_csr( - domains, csr, authzr=authzr) + orderr = self._get_order_and_authorizations(csr.data, self.config.allow_subset_of_names) + authzr = orderr.authorizations + auth_domains = set(a.body.identifier.value for a in authzr) + successful_domains = [d for d in domains if d in auth_domains] - return certr, chain, key, csr + # allow_subset_of_names is currently disabled for wildcard + # certificates. The reason for this and checking allow_subset_of_names + # below is because successful_domains == domains is never true if + # domains contains a wildcard because the ACME spec forbids identifiers + # in authzs from containing a wildcard character. + if self.config.allow_subset_of_names and successful_domains != domains: + if not self.config.dry_run: + os.remove(key.file) + os.remove(csr.file) + return self.obtain_certificate(successful_domains) + else: + cert, chain = self.obtain_certificate_from_csr(csr, orderr) + + return cert, chain, key, csr + + def _get_order_and_authorizations(self, csr_pem, best_effort): + """Request a new order and complete its authorizations. + + :param str csr_pem: A CSR in PEM format. + :param bool best_effort: True if failing to complete all + authorizations should not raise an exception + + :returns: order resource containing its completed authorizations + :rtype: acme.messages.OrderResource + + """ + try: + orderr = self.acme.new_order(csr_pem) + except acme_errors.WildcardUnsupportedError: + raise errors.Error("The currently selected ACME CA endpoint does" + " not support issuing wildcard certificates.") + authzr = self.auth_handler.handle_authorizations(orderr, best_effort) + return orderr.update(authorizations=authzr) # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): @@ -354,7 +347,7 @@ class Client(object): be obtained, or None if doing a successful dry run. """ - certr, chain, key, _ = self.obtain_certificate(domains) + cert, chain, key, _ = self.obtain_certificate(domains) if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): @@ -362,26 +355,30 @@ class Client(object): "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - new_name = certname if certname else domains[0] + if certname: + new_name = certname + elif util.is_wildcard_domain(domains[0]): + # Don't make files and directories starting with *. + new_name = domains[0][2:] + else: + new_name = domains[0] + if self.config.dry_run: logger.debug("Dry run: Skipping creating new lineage for %s", new_name) return None else: return storage.RenewableCert.new_lineage( - new_name, OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), + new_name, cert, + key.pem, chain, self.config) - def save_certificate(self, certr, chain_cert, + def save_certificate(self, cert_pem, chain_pem, cert_path, chain_path, fullchain_path): """Saves the certificate received from the ACME server. - :param certr: ACME "certificate" resource. - :type certr: :class:`acme.messages.Certificate` - - :param list chain_cert: + :param str cert_pem: + :param str chain_pem: :param str cert_path: Candidate path to a certificate. :param str chain_path: Candidate path to a certificate chain. :param str fullchain_path: Candidate path to a full cert chain. @@ -398,8 +395,6 @@ class Client(object): os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) - cert_pem = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) @@ -410,20 +405,15 @@ class Client(object): logger.info("Server issued certificate; certificate written to %s", abs_cert_path) - if not chain_cert: - return abs_cert_path, None, None - else: - chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) - chain_file, abs_chain_path =\ - _open_pem_file('chain_path', chain_path) - fullchain_file, abs_fullchain_path =\ - _open_pem_file('fullchain_path', fullchain_path) + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) - _save_chain(chain_pem, chain_file) - _save_chain(cert_pem + chain_pem, fullchain_file) - - return abs_cert_path, abs_chain_path, abs_fullchain_path + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): @@ -556,11 +546,11 @@ class Client(object): self.installer.rollback_checkpoints() self.installer.restart() except: - # TODO: suggest letshelp-letsencrypt here reporter.add_message( "An error occurred and we failed to restore your config and " - "restart your server. Please submit a bug report to " - "https://github.com/letsencrypt/letsencrypt", + "restart your server. Please post to " + "https://community.letsencrypt.org/c/server-config " + "with details about your configuration and this error you received.", reporter.HIGH_PRIORITY) raise reporter.add_message(success_msg, reporter.HIGH_PRIORITY) diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..37118c591 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,7 +14,6 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util @@ -340,14 +339,8 @@ def _get_names_from_cert_or_req(cert_or_req, load_func, typ): def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): - common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access - sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) - - if common_name is None: - return sans - else: - return [common_name] + [d for d in sans if d != common_name] + return acme_crypto_util._pyopenssl_cert_or_req_all_names(loaded_cert_or_req) def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): @@ -373,16 +366,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """ # XXX: returns empty string when no chain is available, which # shuts up RenewableCert, but might not be the best solution... - - def _dump_cert(cert): - if isinstance(cert, jose.ComparableX509): - # pylint: disable=protected-access - cert = cert.wrapped - return OpenSSL.crypto.dump_certificate(filetype, cert) - - # assumes that OpenSSL.crypto.dump_certificate includes ending - # newline character - return b"".join(_dump_cert(cert) for cert in chain) + return acme_crypto_util.dump_pyopenssl_chain(chain, filetype) def notBefore(cert_path): @@ -449,3 +433,17 @@ def sha256sum(filename): with open(filename, 'rb') as f: sha256.update(f.read()) return sha256.hexdigest() + +def cert_and_chain_from_fullchain(fullchain_pem): + """Split fullchain_pem into cert_pem and chain_pem + + :param str fullchain_pem: concatenated cert + chain + + :returns: tuple of string cert_pem and chain_pem + :rtype: tuple + + """ + cert = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)).decode() + chain = fullchain_pem[len(cert):] + return (cert, chain) diff --git a/certbot/log.py b/certbot/log.py index f7c7b126c..e0d2e8f11 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -165,12 +165,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(ColoredStreamHandler, self).__init__(stream) + super(ColoredStreamHandler, self).__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -184,9 +179,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(ColoredStreamHandler, self).format(record)) + out = super(ColoredStreamHandler, self).format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: @@ -203,23 +196,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): def __init__(self, target=None): # capacity doesn't matter because should_flush() is overridden capacity = float('inf') - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.__init__( - self, capacity, target=target) - else: - super(MemoryHandler, self).__init__(capacity, target=target) + super(MemoryHandler, self).__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = self.target - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.close(self) - else: - super(MemoryHandler, self).close() + super(MemoryHandler, self).close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -233,10 +217,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - if sys.version_info < (2, 7): # pragma: no cover - logging.handlers.MemoryHandler.flush(self) - else: - super(MemoryHandler, self).flush() + super(MemoryHandler, self).flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -262,12 +243,7 @@ class TempHandler(logging.StreamHandler): """ def __init__(self): stream = tempfile.NamedTemporaryFile('w', delete=False) - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.__init__(self, stream) - else: - super(TempHandler, self).__init__(stream) + super(TempHandler, self).__init__(stream) self.path = stream.name self._delete = True @@ -278,12 +254,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - # logging handlers use old style classes in Python 2.6 so - # super() cannot be used - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.emit(self, record) - else: - super(TempHandler, self).emit(record) + super(TempHandler, self).emit(record) def close(self): """Close the handler and the temporary log file. @@ -299,10 +270,7 @@ class TempHandler(logging.StreamHandler): if self._delete: os.remove(self.path) self._delete = False - if sys.version_info < (2, 7): # pragma: no cover - logging.StreamHandler.close(self) - else: - super(TempHandler, self).close() + super(TempHandler, self).close() finally: self.release() diff --git a/certbot/main.py b/certbot/main.py index 1decb1a3d..47402ba80 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,10 +1,10 @@ """Certbot main entry point.""" +# pylint: disable=too-many-lines from __future__ import print_function import functools import logging.handlers import os import sys -import warnings import configobj import josepy as jose @@ -496,17 +496,20 @@ def _determine_account(config): if config.email is None and not config.register_unsafely_without_email: config.email = display_ops.get_email() - def _tos_cb(regr): + def _tos_cb(terms_of_service): if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( - regr.terms_of_service, config.server)) + terms_of_service, config.server)) obj = zope.component.getUtility(interfaces.IDisplay) - return obj.yesno(msg, "Agree", "Cancel", + result = obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos", force_interactive=True) - + if not result: + raise errors.Error( + "Registration cannot proceed without accepting " + "Terms of Service.") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) @@ -780,11 +783,45 @@ def install(config, plugins): except errors.PluginSelectionError as e: return str(e) - domains, _ = _find_domains_or_certname(config, installer) - le_client = _init_le_client(config, authenticator=None, installer=installer) - _install_cert(config, le_client, domains) + # If cert-path is defined, populate missing (ie. not overridden) values. + # Unfortunately this can't be done in argument parser, as certificate + # manager needs the access to renewal directory paths + if config.certname: + config = _populate_from_certname(config) + if config.key_path and config.cert_path: + _check_certificate_and_key(config) + domains, _ = _find_domains_or_certname(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + _install_cert(config, le_client, domains) + else: + raise errors.ConfigurationError("Path to certificate or key was not defined. " + "If your certificate is managed by Certbot, please use --cert-name " + "to define which certificate you would like to install.") +def _populate_from_certname(config): + """Helper function for install to populate missing config values from lineage + defined by --cert-name.""" + lineage = cert_manager.lineage_for_certname(config, config.certname) + if not lineage: + return config + if not config.key_path: + config.namespace.key_path = lineage.key_path + if not config.cert_path: + config.namespace.cert_path = lineage.cert_path + if not config.chain_path: + config.namespace.chain_path = lineage.chain_path + if not config.fullchain_path: + config.namespace.fullchain_path = lineage.fullchain_path + return config + +def _check_certificate_and_key(config): + if not os.path.isfile(os.path.realpath(config.cert_path)): + raise errors.ConfigurationError("Error while reading certificate from path " + "{0}".format(config.cert_path)) + if not os.path.isfile(os.path.realpath(config.key_path)): + raise errors.ConfigurationError("Error while reading private key from path " + "{0}".format(config.key_path)) def plugins_cmd(config, plugins): """List server software plugins. @@ -946,11 +983,11 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config config.cert_path[0], config.key_path[0]) crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) key = jose.JWK.load(config.key_path[1]) + acme = client.acme_from_config_key(config, key) else: # revocation by account key logger.debug("Revoking %s using Account Key", config.cert_path[0]) acc, _ = _determine_account(config) - key = acc.key - acme = client.acme_from_config_key(config, key) + acme = client.acme_from_config_key(config, acc.key, acc.regr) cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] logger.debug("Reason code for revocation: %s", config.reason) @@ -1027,13 +1064,13 @@ def _csr_get_and_save_cert(config, le_client): """ csr, _ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) + cert, chain = le_client.obtain_certificate_from_csr(csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) return None, None cert_path, _, fullchain_path = le_client.save_certificate( - certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + cert, chain, config.cert_path, config.chain_path, config.fullchain_path) return cert_path, fullchain_path def renew_cert(config, plugins, lineage): @@ -1220,17 +1257,6 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise - deprecation_fmt = ( - "Python %s.%s support will be dropped in the next " - "release of Certbot - please upgrade your Python version.") - # We use the warnings system for Python 2.6 and logging for Python 3 - # because DeprecationWarnings are only reported by default in Python <= 2.6 - # and warnings can be disabled by the user. - if sys.version_info[:2] == (2, 6): - warning = deprecation_fmt % sys.version_info[:2] - warnings.warn(warning, DeprecationWarning) - elif sys.version_info[:2] == (3, 3): - logger.warning(deprecation_fmt, *sys.version_info[:2]) set_displayer(config) diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 37baf98f7..062c11650 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -5,6 +5,8 @@ import logging import pkg_resources import six +from collections import OrderedDict + import zope.interface import zope.interface.verify @@ -12,12 +14,6 @@ from certbot import constants from certbot import errors from certbot import interfaces -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) @@ -194,6 +190,7 @@ class PluginsRegistry(collections.Mapping): def find_all(cls): """Find plugins using setuptools entry points.""" plugins = {} + # pylint: disable=not-callable entry_points = itertools.chain( pkg_resources.iter_entry_points( constants.SETUPTOOLS_PLUGINS_ENTRY_POINT), diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 07371ad34..614449d34 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -189,7 +189,7 @@ when it receives a TLS ClientHello with the SNI extension set to os.environ.update(env) _, out = hooks.execute(self.conf('auth-hook')) env['CERTBOT_AUTH_OUTPUT'] = out.strip() - self.env[achall.domain] = env + self.env[achall] = env def _perform_achall_manually(self, achall): validation = achall.validation(achall.account_key) @@ -215,7 +215,7 @@ when it receives a TLS ClientHello with the SNI extension set to def cleanup(self, achalls): # pylint: disable=missing-docstring if self.conf('cleanup-hook'): for achall in achalls: - env = self.env.pop(achall.domain) + env = self.env.pop(achall) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) diff --git a/certbot/plugins/manual_test.py b/certbot/plugins/manual_test.py index ac528e81c..e5c22b377 100644 --- a/certbot/plugins/manual_test.py +++ b/certbot/plugins/manual_test.py @@ -93,10 +93,10 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.perform(self.achalls), [achall.response(achall.account_key) for achall in self.achalls]) self.assertEqual( - self.auth.env[self.dns_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.dns_achall]['CERTBOT_AUTH_OUTPUT'], dns_expected) self.assertEqual( - self.auth.env[self.http_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.http_achall]['CERTBOT_AUTH_OUTPUT'], http_expected) # tls_sni_01 challenge must be perform()ed above before we can # get the cert_path and key_path. @@ -107,7 +107,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.auth.tls_sni_01.get_z_domain(self.tls_sni_achall), 'novalidation') self.assertEqual( - self.auth.env[self.tls_sni_achall.domain]['CERTBOT_AUTH_OUTPUT'], + self.auth.env[self.tls_sni_achall]['CERTBOT_AUTH_OUTPUT'], tls_sni_expected) @test_util.patch_get_utility() diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index f0e2f4c5b..ad2257e1d 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -6,6 +6,24 @@ from certbot import util logger = logging.getLogger(__name__) +def get_prefixes(path): + """Retrieves all possible path prefixes of a path, in descending order + of length. For instance, + /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'] + :param str path: the path to break into prefixes + + :returns: all possible path prefixes of given path in descending order + :rtype: `list` of `str` + """ + prefix = path + prefixes = [] + while len(prefix) > 0: + prefixes.append(prefix) + prefix, _ = os.path.split(prefix) + # break once we hit '/' + if prefix == prefixes[-1]: + break + return prefixes def path_surgery(cmd): """Attempt to perform PATH surgery to find cmd diff --git a/certbot/plugins/util_test.py b/certbot/plugins/util_test.py index 947f24697..2c0e476ae 100644 --- a/certbot/plugins/util_test.py +++ b/certbot/plugins/util_test.py @@ -5,6 +5,14 @@ import unittest import mock +class GetPrefixTest(unittest.TestCase): + """Tests for certbot.plugins.get_prefixes.""" + def test_get_prefix(self): + from certbot.plugins.util import get_prefixes + self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']) + self.assertEqual(get_prefixes("/"), ["/"]) + self.assertEqual(get_prefixes("a"), ["a"]) + class PathSurgeryTest(unittest.TestCase): """Tests for certbot.plugins.path_surgery.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index 714d83cce..6328b16ef 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -18,6 +18,7 @@ from certbot import interfaces from certbot.display import util as display_util from certbot.display import ops from certbot.plugins import common +from certbot.plugins import util logger = logging.getLogger(__name__) @@ -65,6 +66,8 @@ to serve all files under specified web root ({0}).""" super(Authenticator, self).__init__(*args, **kwargs) self.full_roots = {} self.performed = collections.defaultdict(set) + # stack of dirs successfully created by this authenticator + self._created_dirs = [] def prepare(self): # pylint: disable=missing-docstring pass @@ -161,27 +164,26 @@ to serve all files under specified web root ({0}).""" # Umask is used instead of chmod to ensure the client can also # run as non-root (GH #1795) old_umask = os.umask(0o022) - try: - # This is coupled with the "umask" call above because - # os.makedirs's "mode" parameter may not always work: - # https://stackoverflow.com/questions/5231901/permission-problems-when-creating-a-dir-with-os-makedirs-python - os.makedirs(self.full_roots[name], 0o0755) - - # Set owner as parent directory if possible - try: - stat_path = os.stat(path) - os.chown(self.full_roots[name], stat_path.st_uid, - stat_path.st_gid) - except OSError as exception: - logger.info("Unable to change owner and uid of webroot directory") - logger.debug("Error was: %s", exception) - - except OSError as exception: - if exception.errno != errno.EEXIST: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + stat_path = os.stat(path) + for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len): + try: + # This is coupled with the "umask" call above because + # os.mkdir's "mode" parameter may not always work: + # https://docs.python.org/3/library/os.html#os.mkdir + os.mkdir(prefix, 0o0755) + self._created_dirs.append(prefix) + # Set owner as parent directory if possible + try: + os.chown(prefix, stat_path.st_uid, stat_path.st_gid) + except OSError as exception: + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for {0} http-01 " + "challenge responses: {1}".format(name, exception)) finally: os.umask(old_umask) @@ -217,16 +219,17 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - for root_path, achalls in six.iteritems(self.performed): - if not achalls: - try: - os.rmdir(root_path) - logger.debug("All challenges cleaned up, removing %s", - root_path) - except OSError as exc: - logger.info( - "Unable to clean up challenge directory %s", root_path) - logger.debug("Error was: %s", exc) + not_removed = [] + while len(self._created_dirs) > 0: + path = self._created_dirs.pop() + try: + os.rmdir(path) + except OSError as exc: + not_removed.insert(0, path) + logger.info("Challenge directory %s was not empty, didn't remove", path) + logger.debug("Error was: %s", exc) + self._created_dirs = not_removed + logger.debug("All challenges cleaned up") class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..59133f0aa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -36,6 +36,8 @@ class AuthenticatorTest(unittest.TestCase): def setUp(self): from certbot.plugins.webroot import Authenticator self.path = tempfile.mkdtemp() + self.partial_root_challenge_path = os.path.join( + self.path, ".well-known") self.root_challenge_path = os.path.join( self.path, ".well-known", "acme-challenge") self.validation_path = os.path.join( @@ -199,6 +201,35 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup([self.achall]) self.assertFalse(os.path.exists(self.validation_path)) self.assertFalse(os.path.exists(self.root_challenge_path)) + self.assertFalse(os.path.exists(self.partial_root_challenge_path)) + + def test_perform_cleanup_existing_dirs(self): + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([self.achall]) + self.auth.cleanup([self.achall]) + + # Ensure we don't "clean up" directories that previously existed + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) + + def test_perform_cleanup_multiple_challenges(self): + bingo_achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"bingo"), "pending"), + domain="thing.com", account_key=KEY) + + bingo_validation_path = "YmluZ28" + os.mkdir(self.partial_root_challenge_path) + self.auth.prepare() + self.auth.perform([bingo_achall, self.achall]) + + self.auth.cleanup([self.achall]) + self.assertFalse(os.path.exists(bingo_validation_path)) + self.assertTrue(os.path.exists(self.root_challenge_path)) + self.auth.cleanup([bingo_achall]) + self.assertFalse(os.path.exists(self.validation_path)) + self.assertFalse(os.path.exists(self.root_challenge_path)) def test_cleanup_leftovers(self): self.auth.prepare() diff --git a/certbot/renewal.py b/certbot/renewal.py index b73fb71f0..57985a8ed 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -295,15 +295,12 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) if not domains: domains = lineage.names() - new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) + new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() - new_cert = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped) - new_chain = crypto_util.dump_pyopenssl_chain(new_chain) # TODO: Check return value of save_successor lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, config) lineage.update_all_links_to(lineage.latest_common_version()) @@ -426,10 +423,14 @@ def handle_renewal_request(config): main.renew_cert(lineage_config, plugins, renewal_candidate) renew_successes.append(renewal_candidate.fullchain) else: - renew_skipped.append(renewal_candidate.fullchain) + expiry = crypto_util.notAfter(renewal_candidate.version( + "cert", renewal_candidate.latest_common_version())) + renew_skipped.append("%s expires on %s" % (renewal_candidate.fullchain, + expiry.strftime("%Y-%m-%d"))) # Run updater interface methods updater.run_renewal_updaters(lineage_config, plugins, renewal_candidate) + except Exception as e: # pylint: disable=broad-except # obtain_cert (presumably) encountered an unanticipated problem. logger.warning("Attempting to renew cert (%s) from %s produced an " diff --git a/certbot/storage.py b/certbot/storage.py index 67d2155ae..ed3922c58 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -417,7 +417,7 @@ class RenewableCert(object): conf_version = self.configuration.get("version") if (conf_version is not None and util.get_strict_version(conf_version) > CURRENT_VERSION): - logger.warning( + logger.info( "Attempting to parse the version %s renewal configuration " "file found at %s with version %s of Certbot. This might not " "work.", conf_version, config_filename, certbot.__version__) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 32c4c0d3b..54e284d9e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -29,36 +29,35 @@ class ChallengeFactoryTest(unittest.TestCase): # Account is mocked... self.handler = AuthHandler(None, None, mock.Mock(key="mock_key"), []) - self.dom = "test" - self.handler.authzr[self.dom] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.dom, acme_util.CHALLENGES, + self.authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", acme_util.CHALLENGES, [messages.STATUS_PENDING] * 6, False) def test_all(self): achalls = self.handler._challenge_factory( - self.dom, range(0, len(acme_util.CHALLENGES))) + self.authzr, range(0, len(acme_util.CHALLENGES))) self.assertEqual( [achall.chall for achall in achalls], acme_util.CHALLENGES) def test_one_tls_sni(self): - achalls = self.handler._challenge_factory(self.dom, [1]) + achalls = self.handler._challenge_factory(self.authzr, [1]) self.assertEqual( [achall.chall for achall in achalls], [acme_util.TLSSNI01]) def test_unrecognized(self): - self.handler.authzr["failure.com"] = acme_util.gen_authzr( - messages.STATUS_PENDING, "failure.com", - [mock.Mock(chall="chall", typ="unrecognized")], - [messages.STATUS_PENDING]) + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "test", + [mock.Mock(chall="chall", typ="unrecognized")], + [messages.STATUS_PENDING]) self.assertRaises( - errors.Error, self.handler._challenge_factory, "failure.com", [0]) + errors.Error, self.handler._challenge_factory, authzr, [0]) -class GetAuthorizationsTest(unittest.TestCase): - """get_authorizations test. +class HandleAuthorizationsTest(unittest.TestCase): + """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -81,6 +80,7 @@ class GetAuthorizationsTest(unittest.TestCase): self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) + self.mock_net.acme_version = 1 self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -90,20 +90,19 @@ class GetAuthorizationsTest(unittest.TestCase): def tearDown(self): logging.disable(logging.NOTSET) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def _test_name1_tls_sni_01_1_common(self, combos): + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) + mock_order = mock.MagicMock(authorizations=[authzr]) - mock_poll.side_effect = self._validate_all - - authzr = self.handler.get_authorizations(["0"]) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + chall_update = mock_poll.call_args[0][1] + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -113,22 +112,28 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES, combos=False) + def test_name1_tls_sni_01_1_acme_1(self): + self._test_name1_tls_sni_01_1_common(combos=True) + def test_name1_tls_sni_01_1_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name1_tls_sni_01_1_common(combos=False) + + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] - self.assertEqual(list(six.iterkeys(chall_update)), ["0"]) + chall_update = mock_poll.call_args[0][1] + self.assertEqual(list(six.iterkeys(chall_update)), [0]) self.assertEqual(len(chall_update.values()), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -140,89 +145,149 @@ class GetAuthorizationsTest(unittest.TestCase): self.assertEqual(len(authzr), 1) @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name3_tls_sni_01_3(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) - + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + self.mock_net.acme_version = 2 mock_poll.side_effect = self._validate_all + self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) + self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) - authzr = self.handler.get_authorizations(["0", "1", "2"]) + authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=False) + mock_order = mock.MagicMock(authorizations=[authzr]) + authzr = self.handler.handle_authorizations(mock_order) + + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + + self.assertEqual(mock_poll.call_count, 1) + chall_update = mock_poll.call_args[0][1] + self.assertEqual(list(six.iterkeys(chall_update)), [0]) + self.assertEqual(len(chall_update.values()), 1) + + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] + self.assertEqual(len(cleaned_up_achalls), 1) + self.assertEqual(cleaned_up_achalls[0].typ, "tls-sni-01") + + # Length of authorizations list + self.assertEqual(len(authzr), 1) + + def _test_name3_tls_sni_01_3_common(self, combos): + self.mock_net.request_domain_challenges.side_effect = functools.partial( + gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) + + + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) # Check poll call self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][0] + chall_update = mock_poll.call_args[0][1] self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue("0" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["0"]), 1) - self.assertTrue("1" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["1"]), 1) - self.assertTrue("2" in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update["2"]), 1) + self.assertTrue(0 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[0]), 1) + self.assertTrue(1 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[1]), 1) + self.assertTrue(2 in list(six.iterkeys(chall_update))) + self.assertEqual(len(chall_update[2]), 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual(len(authzr), 3) + def test_name3_tls_sni_01_3_common_acme_1(self): + self._test_name3_tls_sni_01_3_common(combos=True) + + def test_name3_tls_sni_01_3_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_name3_tls_sni_01_3_common(combos=False) + @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") def test_debug_challenges(self, mock_poll): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) mock_poll.side_effect = self._validate_all - self.handler.get_authorizations(["0"]) + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 1) self.assertEqual(self.mock_display.notification.call_count, 1) def test_perform_failure(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + self.mock_auth.perform.side_effect = errors.AuthorizationError self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) def test_no_domains(self): - self.assertRaises(errors.AuthorizationError, self.handler.get_authorizations, []) + mock_order = mock.MagicMock(authorizations=[]) + self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_preferred_challenge_choice(self, mock_poll): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def _test_preferred_challenge_choice_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] + mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - self.handler.get_authorizations(["0"]) + with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: + mock_poll.side_effect = self._validate_all + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") - def test_preferred_challenges_not_supported(self): - self.mock_net.request_domain_challenges.side_effect = functools.partial( - gen_dom_authzr, challs=acme_util.CHALLENGES) + def test_preferred_challenge_choice_common_acme_1(self): + self._test_preferred_challenge_choice_common(combos=True) + + def test_preferred_challenge_choice_common_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenge_choice_common(combos=False) + + def _test_preferred_challenges_not_supported_common(self, combos): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos)] + mock_order = mock.MagicMock(authorizations=authzrs) self.handler.pref_challs.append(challenges.HTTP01.typ) self.assertRaises( - errors.AuthorizationError, self.handler.get_authorizations, ["0"]) + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) - def _validate_all(self, unused_1, unused_2): - for dom in six.iterkeys(self.handler.authzr): - azr = self.handler.authzr[dom] - self.handler.authzr[dom] = acme_util.gen_authzr( + def test_preferred_challenges_not_supported_acme_1(self): + self._test_preferred_challenges_not_supported_common(combos=True) + + def test_preferred_challenges_not_supported_acme_2(self): + self.mock_net.acme_version = 2 + self._test_preferred_challenges_not_supported_common(combos=False) + + def test_dns_only_challenge_not_supported(self): + authzrs = [gen_dom_authzr(domain="0", challs=[acme_util.DNS01])] + mock_order = mock.MagicMock(authorizations=authzrs) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + def _validate_all(self, aauthzrs, unused_1, unused_2): + for i, aauthzr in enumerate(aauthzrs): + azr = aauthzr.authzr + updated_azr = acme_util.gen_authzr( messages.STATUS_VALID, - dom, + azr.body.identifier.value, [challb.chall for challb in azr.body.challenges], [messages.STATUS_VALID] * len(azr.body.challenges), azr.body.combinations) + aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) class PollChallengesTest(unittest.TestCase): @@ -231,7 +296,7 @@ class PollChallengesTest(unittest.TestCase): def setUp(self): from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler + from certbot.auth_handler import AuthHandler, AnnotatedAuthzr # Account and network are mocked... self.mock_net = mock.MagicMock() @@ -239,40 +304,40 @@ class PollChallengesTest(unittest.TestCase): None, self.mock_net, mock.Mock(key="mock_key"), []) self.doms = ["0", "1", "2"] - self.handler.authzr[self.doms[0]] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False) - - self.handler.authzr[self.doms[1]] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) - - self.handler.authzr[self.doms[2]] = acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False) + self.aauthzrs = [ + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[0], + [acme_util.HTTP01, acme_util.TLSSNI01], + [messages.STATUS_PENDING] * 2, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[1], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), + AnnotatedAuthzr(acme_util.gen_authzr( + messages.STATUS_PENDING, self.doms[2], + acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) + ] self.chall_update = {} - for dom in self.doms: - self.chall_update[dom] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), dom) - for challb in self.handler.authzr[dom].body.challenges] + for i, aauthzr in enumerate(self.aauthzrs): + self.chall_update[i] = [ + challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) + for challb in aauthzr.authzr.body.challenges] @mock.patch("certbot.auth_handler.time") def test_poll_challenges(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.chall_update, False) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_VALID) + for aauthzr in self.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) @mock.patch("certbot.auth_handler.time") def test_poll_challenges_failure_best_effort(self, unused_mock_time): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.chall_update, True) + self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - for authzr in self.handler.authzr.values(): - self.assertEqual(authzr.body.status, messages.STATUS_PENDING) + for aauthzr in self.aauthzrs: + self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) @mock.patch("certbot.auth_handler.time") @test_util.patch_get_utility() @@ -280,21 +345,21 @@ class PollChallengesTest(unittest.TestCase): self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) @mock.patch("certbot.auth_handler.time") def test_unable_to_find_challenge_status(self, unused_mock_time): from certbot.auth_handler import challb_to_achall self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[self.doms[0]].append( + self.chall_update[0].append( challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) self.assertRaises( errors.AuthorizationError, self.handler._poll_challenges, - self.chall_update, False) + self.aauthzrs, self.chall_update, False) def test_verify_authzr_failure(self): - self.assertRaises( - errors.AuthorizationError, self.handler.verify_authzr_complete) + self.assertRaises(errors.AuthorizationError, + self.handler.verify_authzr_complete, self.aauthzrs) def _mock_poll_solve_one_valid(self, authzr): # Pending here because my dummy script won't change the full status. diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..1bba6991a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -426,6 +426,10 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods namespace = self.parse(["--no-delete-after-revoke"]) self.assertFalse(namespace.delete_after_revoke) + def test_allow_subset_with_wildcard(self): + self.assertRaises(errors.Error, self.parse, + "--allow-subset-of-names -d *.example.org".split()) + class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..0f2c58161 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,12 +4,8 @@ import shutil import tempfile import unittest -import josepy as jose -import OpenSSL import mock -from acme import errors as acme_errors - from certbot import account from certbot import errors from certbot import util @@ -30,31 +26,27 @@ class RegisterTest(test_util.ConfigTestCase): self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" self.account_storage = account.AccountMemoryStorage() - self.tos_cb = mock.MagicMock() def _call(self): from certbot.client import register - return register(self.config, self.account_storage, self.tos_cb) + tos_cb = mock.MagicMock() + return register(self.config, self.account_storage, tos_cb) def test_no_tos(self): - with mock.patch("certbot.client.acme_client.Client") as mock_client: - mock_client.register().terms_of_service = "http://tos" + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: + mock_client.new_account_and_tos().terms_of_service = "http://tos" with mock.patch("certbot.eff.handle_subscription") as mock_handle: with mock.patch("certbot.account.report_new_account"): - self.tos_cb.return_value = False + mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) self.assertFalse(mock_handle.called) - self.tos_cb.return_value = True + mock_client().new_account_and_tos.side_effect = None self._call() self.assertTrue(mock_handle.called) - self.tos_cb = None - self._call() - self.assertEqual(mock_handle.call_count, 2) - def test_it(self): - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): with mock.patch("certbot.eff.handle_subscription"): self._call() @@ -66,9 +58,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = False msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) self.assertTrue(mock_handle.called) @@ -79,9 +71,9 @@ class RegisterTest(test_util.ConfigTestCase): self.config.noninteractive_mode = True msg = "DNS problem: NXDOMAIN looking up MX for example.com" mx_err = messages.Error.with_code('invalidContact', detail=msg) - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription"): - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(errors.Error, self._call) def test_needs_email(self): @@ -91,7 +83,7 @@ class RegisterTest(test_util.ConfigTestCase): @mock.patch("certbot.client.logger") def test_without_email(self, mock_logger): with mock.patch("certbot.eff.handle_subscription") as mock_handle: - with mock.patch("certbot.client.acme_client.Client"): + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2"): with mock.patch("certbot.account.report_new_account"): self.config.email = None self.config.register_unsafely_without_email = True @@ -104,9 +96,9 @@ class RegisterTest(test_util.ConfigTestCase): from acme import messages msg = "Test" mx_err = messages.Error(detail=msg, typ="malformed", title="title") - with mock.patch("certbot.client.acme_client.Client") as mock_client: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as mock_client: with mock.patch("certbot.eff.handle_subscription") as mock_handle: - mock_client().register.side_effect = [mx_err, mock.MagicMock()] + mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) self.assertFalse(mock_handle.called) @@ -122,7 +114,7 @@ class ClientTestCommon(test_util.ConfigTestCase): self.account = mock.MagicMock(**{"key.pem": KEY}) from certbot.client import Client - with mock.patch("certbot.client.acme_client.Client") as acme: + with mock.patch("certbot.client.acme_client.BackwardsCompatibleClientV2") as acme: self.acme_client = acme self.acme = acme.return_value = mock.MagicMock() self.client = Client( @@ -138,106 +130,74 @@ class ClientTest(ClientTestCommon): self.config.allow_subset_of_names = False self.config.dry_run = False self.eg_domains = ["example.com", "www.example.com"] + self.eg_order = mock.MagicMock( + authorizations=[None], + csr_pem=mock.sentinel.csr_pem) def test_init_acme_verify_ssl(self): - net = self.acme_client.call_args[1]["net"] + net = self.acme_client.call_args[0][0] self.assertTrue(net.verify_ssl) def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() - self.client.auth_handler.get_authorizations.return_value = [None] - self.acme.request_issuance.return_value = mock.sentinel.certr - self.acme.fetch_chain.return_value = mock.sentinel.chain + self.client.auth_handler.handle_authorizations.return_value = [None] + self.acme.finalize_order.return_value = self.eg_order + self.acme.new_order.return_value = self.eg_order + self.eg_order.update.return_value = self.eg_order - def _check_obtain_certificate(self): - self.client.auth_handler.get_authorizations.assert_called_once_with( - self.eg_domains, - self.config.allow_subset_of_names) + def _check_obtain_certificate(self, auth_count=1): + if auth_count == 1: + self.client.auth_handler.handle_authorizations.assert_called_once_with( + self.eg_order, + self.config.allow_subset_of_names) + else: + self.assertEqual(self.client.auth_handler.handle_authorizations.call_count, auth_count) - authzr = self.client.auth_handler.get_authorizations() - - self.acme.request_issuance.assert_called_once_with( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_PEM, CSR_SAN)), - authzr) - - self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) + self.acme.finalize_order.assert_called_once_with( + self.eg_order, mock.ANY) + @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.logger") @test_util.patch_get_utility() def test_obtain_certificate_from_csr(self, unused_mock_get_utility, - mock_logger): + mock_logger, mock_crypto_util): self._mock_obtain_certificate() test_csr = util.CSR(form="pem", file=None, data=CSR_SAN) auth_handler = self.client.auth_handler + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) - authzr = auth_handler.get_authorizations(self.eg_domains, False) + orderr = self.acme.new_order(test_csr.data) + auth_handler.handle_authorizations(orderr, False) self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=authzr)) + orderr=orderr)) # and that the cert was obtained correctly self._check_obtain_certificate() - # Test for authzr=None + # Test for orderr=None self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), + (mock.sentinel.cert, mock.sentinel.chain), self.client.obtain_certificate_from_csr( - self.eg_domains, test_csr, - authzr=None)) - auth_handler.get_authorizations.assert_called_with(self.eg_domains) + orderr=None)) + auth_handler.handle_authorizations.assert_called_with(self.eg_order, False) # Test for no auth_handler self.client.auth_handler = None self.assertRaises( errors.Error, self.client.obtain_certificate_from_csr, - self.eg_domains, test_csr) mock_logger.warning.assert_called_once_with(mock.ANY) - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_succeeded( - self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = [acme_errors.Error, - mock.sentinel.chain] - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler - - authzr = auth_handler.get_authorizations(self.eg_domains, False) - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) - self.assertEqual(1, mock_get_utility().notification.call_count) - - @test_util.patch_get_utility() - def test_obtain_certificate_from_csr_retry_failed(self, mock_get_utility): - self._mock_obtain_certificate() - self.acme.fetch_chain.side_effect = acme_errors.Error - test_csr = util.CSR(form="der", file=None, data=CSR_SAN) - auth_handler = self.client.auth_handler - - authzr = auth_handler.get_authorizations(self.eg_domains, False) - self.assertRaises( - acme_errors.Error, - self.client.obtain_certificate_from_csr, - self.eg_domains, - test_csr, - authzr=authzr) - self.assertEqual(1, mock_get_utility().notification.call_count) - @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): csr = util.CSR(form="pem", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) self._test_obtain_certificate_common(mock.sentinel.key, csr) @@ -245,6 +205,26 @@ class ClientTest(ClientTestCommon): self.config.rsa_key_size, self.config.key_dir) mock_crypto_util.init_save_csr.assert_called_once_with( mock.sentinel.key, self.eg_domains, self.config.csr_dir) + mock_crypto_util.cert_and_chain_from_fullchain.assert_called_once_with( + self.eg_order.fullchain_pem) + + @mock.patch("certbot.client.crypto_util") + @mock.patch("os.remove") + def test_obtain_certificate_partial_success(self, mock_remove, mock_crypto_util): + csr = util.CSR(form="pem", file=mock.sentinel.csr_file, data=CSR_SAN) + key = util.CSR(form="pem", file=mock.sentinel.key_file, data=CSR_SAN) + mock_crypto_util.init_save_csr.return_value = csr + mock_crypto_util.init_save_key.return_value = key + self._set_mock_from_fullchain(mock_crypto_util.cert_and_chain_from_fullchain) + + authzr = self._authzr_from_domains(["example.com"]) + self.config.allow_subset_of_names = True + self._test_obtain_certificate_common(key, csr, authzr_ret=authzr, auth_count=2) + + self.assertEqual(mock_crypto_util.init_save_key.call_count, 2) + self.assertEqual(mock_crypto_util.init_save_csr.call_count, 2) + self.assertEqual(mock_remove.call_count, 2) + self.assertEqual(mock_crypto_util.cert_and_chain_from_fullchain.call_count, 1) @mock.patch("certbot.client.crypto_util") @mock.patch("certbot.client.acme_crypto_util") @@ -253,6 +233,7 @@ class ClientTest(ClientTestCommon): mock_acme_crypto.make_csr.return_value = CSR_SAN mock_crypto.make_key.return_value = mock.sentinel.key_pem key = util.Key(file=None, pem=mock.sentinel.key_pem) + self._set_mock_from_fullchain(mock_crypto.cert_and_chain_from_fullchain) self.client.config.dry_run = True self._test_obtain_certificate_common(key, csr) @@ -262,40 +243,51 @@ class ClientTest(ClientTestCommon): mock.sentinel.key_pem, self.eg_domains, self.config.must_staple) mock_crypto.init_save_key.assert_not_called() mock_crypto.init_save_csr.assert_not_called() + self.assertEqual(mock_crypto.cert_and_chain_from_fullchain.call_count, 1) - def _test_obtain_certificate_common(self, key, csr): - self._mock_obtain_certificate() - - # return_value is essentially set to (None, None) in - # _mock_obtain_certificate(), which breaks this test. - # Thus fixed by the next line. + def _set_mock_from_fullchain(self, mock_from_fullchain): + mock_cert = mock.Mock() + mock_cert.encode.return_value = mock.sentinel.cert + mock_chain = mock.Mock() + mock_chain.encode.return_value = mock.sentinel.chain + mock_from_fullchain.return_value = (mock_cert, mock_chain) + def _authzr_from_domains(self, domains): authzr = [] # domain ordering should not be affected by authorization order - for domain in reversed(self.eg_domains): + for domain in reversed(domains): authzr.append( mock.MagicMock( body=mock.MagicMock( identifier=mock.MagicMock( value=domain)))) + return authzr - self.client.auth_handler.get_authorizations.return_value = authzr + def _test_obtain_certificate_common(self, key, csr, authzr_ret=None, auth_count=1): + self._mock_obtain_certificate() + + # return_value is essentially set to (None, None) in + # _mock_obtain_certificate(), which breaks this test. + # Thus fixed by the next line. + authzr = authzr_ret or self._authzr_from_domains(self.eg_domains) + + self.eg_order.authorizations = authzr + self.client.auth_handler.handle_authorizations.return_value = authzr with test_util.patch_get_utility(): result = self.client.obtain_certificate(self.eg_domains) self.assertEqual( result, - (mock.sentinel.certr, mock.sentinel.chain, key, csr)) - self._check_obtain_certificate() + (mock.sentinel.cert, mock.sentinel.chain, key, csr)) + self._check_obtain_certificate(auth_count) @mock.patch('certbot.client.Client.obtain_certificate') @mock.patch('certbot.storage.RenewableCert.new_lineage') - @mock.patch('OpenSSL.crypto.dump_certificate') - def test_obtain_and_enroll_certificate(self, mock_dump_certificate, + def test_obtain_and_enroll_certificate(self, mock_storage, mock_obtain_certificate): - domains = ["example.com", "www.example.com"] + domains = ["*.example.com", "example.com"] mock_obtain_certificate.return_value = (mock.MagicMock(), mock.MagicMock(), mock.MagicMock(), None) @@ -303,13 +295,14 @@ class ClientTest(ClientTestCommon): self.assertTrue(self.client.obtain_and_enroll_certificate(domains, "example_cert")) self.assertTrue(self.client.obtain_and_enroll_certificate(domains, None)) + self.assertTrue(self.client.obtain_and_enroll_certificate(domains[1:], None)) self.client.config.dry_run = True self.assertFalse(self.client.obtain_and_enroll_certificate(domains, None)) - self.assertTrue(mock_storage.call_count == 2) - self.assertTrue(mock_dump_certificate.call_count == 2) + names = [call[0][0] for call in mock_storage.call_args_list] + self.assertEqual(names, ["example_cert", "example.com", "example.com"]) @mock.patch("certbot.cli.helpful_parser") def test_save_certificate(self, mock_parser): @@ -318,9 +311,8 @@ class ClientTest(ClientTestCommon): tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? - certr = mock.MagicMock(body=test_util.load_comparable_cert(certs[0])) - chain_cert = [test_util.load_comparable_cert(certs[0]), - test_util.load_comparable_cert(certs[1])] + cert_pem = test_util.load_vector(certs[0]) + chain_pem = (test_util.load_vector(certs[0]) + test_util.load_vector(certs[1])) candidate_cert_path = os.path.join(tmp_path, "certs", "cert_512.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") @@ -330,7 +322,7 @@ class ClientTest(ClientTestCommon): "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( - certr, chain_cert, candidate_cert_path, candidate_chain_path, + cert_pem, chain_pem, candidate_cert_path, candidate_chain_path, candidate_fullchain_path) self.assertEqual(os.path.dirname(cert_path), diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index f0e2c017e..480139378 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -373,5 +373,18 @@ class Sha256sumTest(unittest.TestCase): '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') +class CertAndChainFromFullchainTest(unittest.TestCase): + """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + + def test_cert_and_chain_from_fullchain(self): + cert_pem = CERT.decode() + chain_pem = cert_pem + SS_CERT.decode() + fullchain_pem = cert_pem + chain_pem + from certbot.crypto_util import cert_and_chain_from_fullchain + cert_out, chain_out = cert_and_chain_from_fullchain(fullchain_pem) + self.assertEqual(cert_out, cert_pem) + self.assertEqual(chain_out, chain_pem) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..c4f58ba7c 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -300,8 +300,8 @@ class ChooseNamesTest(unittest.TestCase): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", "also.example.com", "under_score.example.com", - "justtld"] - all_invalid = ["öóòps.net", "*.wildcard.com", "uniçodé.com"] + "justtld", "*.wildcard.com"] + all_invalid = ["öóòps.net", "uniçodé.com"] two_valid = ["example.com", "úniçøde.com", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 2f5d55a51..7fa85b30c 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,3 +1,4 @@ +# coding=utf-8 """Tests for certbot.main.""" # pylint: disable=too-many-lines from __future__ import print_function @@ -226,7 +227,7 @@ class RevokeTest(test_util.TempDirTestCase): 'cert_512.pem')) self.patches = [ - mock.patch('acme.client.Client', autospec=True), + mock.patch('acme.client.BackwardsCompatibleClientV2'), mock.patch('certbot.client.Client'), mock.patch('certbot.main._determine_account'), mock.patch('certbot.main.display_ops.success_revocation') @@ -268,7 +269,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_with_reason(self, mock_acme_client, mock_delete_if_appropriate): mock_delete_if_appropriate.return_value = False - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke expected = [] for reason, code in constants.REVOCATION_REASONS.items(): self._call("--reason " + reason) @@ -593,11 +594,30 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met super(MainTest, self).tearDown() - def _call(self, args, stdout=None): - "Run the cli with output streams and actual client mocked out" - with mock.patch('certbot.main.client') as client: - ret, stdout, stderr = self._call_no_clientmock(args, stdout) - return ret, stdout, stderr, client + def _call(self, args, stdout=None, mockisfile=False): + """Run the cli with output streams, actual client and optionally + os.path.isfile() mocked out""" + + if mockisfile: + orig_open = os.path.isfile + def mock_isfile(fn, *args, **kwargs): + """Mock os.path.isfile()""" + if (fn.endswith("cert") or + fn.endswith("chain") or + fn.endswith("privkey")): + return True + else: + return orig_open(fn, *args, **kwargs) + + with mock.patch("os.path.isfile") as mock_if: + mock_if.side_effect = mock_isfile + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client + else: + with mock.patch('certbot.main.client') as client: + ret, stdout, stderr = self._call_no_clientmock(args, stdout) + return ret, stdout, stderr, client def _call_no_clientmock(self, args, stdout=None): "Run the client with output streams mocked out" @@ -643,10 +663,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._cli_missing_flag(args, "specify a plugin") args.extend(['--standalone', '-d', 'eg.is']) self._cli_missing_flag(args, "register before running") - with mock.patch('certbot.main._get_and_save_cert'): - with mock.patch('certbot.main.client.acme_from_config_key'): - args.extend(['--email', 'io@io.is']) - self._cli_missing_flag(args, "--agree-tos") @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.main.client.acme_client.Client') @@ -675,15 +691,75 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met ua = "bandersnatch" args += ["--user-agent", ua] self._call_no_clientmock(args) - acme_net.assert_called_once_with(mock.ANY, verify_ssl=True, user_agent=ua) + acme_net.assert_called_once_with(mock.ANY, account=mock.ANY, verify_ssl=True, + user_agent=ua) @mock.patch('certbot.main.plug_sel.record_chosen_plugins') @mock.patch('certbot.main.plug_sel.pick_installer') def test_installer_selection(self, mock_pick_installer, _rec): self._call(['install', '--domains', 'foo.bar', '--cert-path', 'cert', - '--key-path', 'key', '--chain-path', 'chain']) + '--key-path', 'privkey', '--chain-path', 'chain'], mockisfile=True) self.assertEqual(mock_pick_installer.call_count, 1) + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_certname(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main._install_cert') + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_override(self, _inst, _rec, mock_install): + mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain", + fullchain_path="/tmp/chain", + key_path="/tmp/privkey") + with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin: + mock_getlin.return_value = mock_lineage + self._call(['install', '--cert-name', 'whatever', + '--key-path', '/tmp/overriding_privkey'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.chain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/overriding_privkey") + + mock_install.reset() + + self._call(['install', '--cert-name', 'whatever', + '--cert-path', '/tmp/overriding_cert'], mockisfile=True) + call_config = mock_install.call_args[0][0] + self.assertEqual(call_config.cert_path, "/tmp/overriding_cert") + self.assertEqual(call_config.fullchain_path, "/tmp/chain") + self.assertEqual(call_config.key_path, "/tmp/privkey") + + @mock.patch('certbot.main.plug_sel.record_chosen_plugins') + @mock.patch('certbot.main.plug_sel.pick_installer') + def test_installer_param_error(self, _inst, _rec): + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--key-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-path', '/tmp/key_path']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install']) + self.assertRaises(errors.ConfigurationError, + self._call, + ['install', '--cert-name', 'notfound', + '--key-path', 'invalid']) + @mock.patch('certbot.main._report_new_cert') @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists, unused_report): @@ -865,11 +941,6 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self.assertRaises(errors.ConfigurationError, self._call, ['-d', (('a' * 50) + '.') * 10]) - # Wildcard - self.assertRaises(errors.ConfigurationError, - self._call, - ['-d', '*.wildcard.tld']) - # Bare IP address (this is actually a different error message now) self.assertRaises(errors.ConfigurationError, self._call, @@ -952,7 +1023,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, - quiet_mode=False): + quiet_mode=False, expiry_date=datetime.datetime.now()): # pylint: disable=too-many-locals,too-many-arguments cert_path = test_util.vector_path('cert_512.pem') chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' @@ -985,7 +1056,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met mock_latest = mock.MagicMock() mock_latest.get_issuer.return_value = "Fake fake" mock_ssl.crypto.load_certificate.return_value = mock_latest - with mock.patch('certbot.main.renewal.crypto_util'): + with mock.patch('certbot.main.renewal.crypto_util') as mock_crypto_util: + mock_crypto_util.notAfter.return_value = expiry_date if not args: args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly'] if extra_args: @@ -1053,6 +1125,16 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) + @mock.patch('certbot.renewal.should_renew') + def test_renew_skips_recent_certs(self, should_renew): + should_renew.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + expiry = datetime.datetime.now() + datetime.timedelta(days=90) + _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, + args=['renew'], expiry_date=expiry) + self.assertTrue('No renewals were attempted.' in stdout.getvalue()) + self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue()) + def test_quiet_renew(self): test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] @@ -1147,7 +1229,7 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met def test_renew_with_bad_domain(self): renewalparams = {'authenticator': 'webroot'} - names = ['*.example.com'] + names = ['uniçodé.com'] self._test_renew_common(renewalparams=renewalparams, error_expected=True, names=names, assert_oc_called=False) @@ -1266,11 +1348,11 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met self._call_no_clientmock(['--cert-path', SS_CERT_PATH, '--key-path', RSA2048_KEY_PATH, '--server', server, 'revoke']) with open(RSA2048_KEY_PATH, 'rb') as f: - mock_acme_client.Client.assert_called_once_with( - server, key=jose.JWK.load(f.read()), net=mock.ANY) + mock_acme_client.BackwardsCompatibleClientV2.assert_called_once_with( + mock.ANY, jose.JWK.load(f.read()), server) with open(SS_CERT_PATH, 'rb') as f: cert = crypto_util.pyopenssl_load_certificate(f.read())[0] - mock_revoke = mock_acme_client.Client().revoke + mock_revoke = mock_acme_client.BackwardsCompatibleClientV2().revoke mock_revoke.assert_called_once_with( jose.ComparableX509(cert), mock.ANY) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6c8f775e2..6c0970e72 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -158,8 +158,8 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertTrue(mock_logger.warning.called) - self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + self.assertTrue(mock_logger.info.called) + self.assertTrue("version" in mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=too-many-statements,protected-access diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..8434d11de 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -57,11 +57,6 @@ def load_cert(*names): return OpenSSL.crypto.load_certificate(loader, load_vector(*names)) -def load_comparable_cert(*names): - """Load ComparableX509 cert.""" - return jose.ComparableX509(load_cert(*names)) - - def load_csr(*names): """Load certificate request.""" loader = _guess_loader( @@ -340,7 +335,7 @@ class ConfigTestCase(TempDirTestCase): self.config.cert_path = constants.CLI_DEFAULTS['auth_cert_path'] self.config.fullchain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] - self.config.server = "example.com" + self.config.server = "https://example.com" def lock_and_call(func, lock_path): """Grab a lock for lock_path and call func. diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 50d323ffd..0e280f3ab 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -487,6 +487,26 @@ class EnforceDomainSanityTest(unittest.TestCase): self._call('this.is.xn--ls8h.tld') +class IsWildcardDomainTest(unittest.TestCase): + """Tests for is_wildcard_domain.""" + + def setUp(self): + self.wildcard = u"*.example.org" + self.no_wildcard = u"example.org" + + def _call(self, domain): + from certbot.util import is_wildcard_domain + return is_wildcard_domain(domain) + + def test_no_wildcard(self): + self.assertFalse(self._call(self.no_wildcard)) + self.assertFalse(self._call(self.no_wildcard.encode())) + + def test_wildcard(self): + self.assertTrue(self._call(self.wildcard)) + self.assertTrue(self._call(self.wildcard.encode())) + + class OsInfoTest(unittest.TestCase): """Test OS / distribution detection""" diff --git a/certbot/util.py b/certbot/util.py index b7e60a225..f7ce6a3bc 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -16,18 +16,14 @@ import stat import subprocess import sys +from collections import OrderedDict + import configargparse from certbot import constants from certbot import errors from certbot import lock -try: - from collections import OrderedDict -except ImportError: # pragma: no cover - # OrderedDict was added in Python 2.7 - from ordereddict import OrderedDict # pylint: disable=import-error - logger = logging.getLogger(__name__) @@ -552,16 +548,6 @@ def enforce_domain_sanity(domain): :returns: The domain cast to `str`, with ASCII-only contents :rtype: str """ - if isinstance(domain, six.text_type): - wildcard_marker = u"*." - else: - wildcard_marker = b"*." - - # Check if there's a wildcard domain - if domain.startswith(wildcard_marker): - raise errors.ConfigurationError( - "Wildcard domains are not supported: {0}".format(domain)) - # Unicode try: if isinstance(domain, six.binary_type): @@ -615,6 +601,24 @@ def enforce_domain_sanity(domain): return domain +def is_wildcard_domain(domain): + """"Is domain a wildcard domain? + + :param damain: domain to check + :type domain: `bytes` or `str` or `unicode` + + :returns: True if domain is a wildcard, otherwise, False + :rtype: bool + + """ + if isinstance(domain, six.text_type): + wildcard_marker = u"*." + else: + wildcard_marker = b"*." + + return domain.startswith(wildcard_marker) + + def get_strict_version(normalized): """Converts a normalized version to a strict version. diff --git a/docker-compose.yml b/docker-compose.yml index 00d3d4c72..75a5b9aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: context: . dockerfile: Dockerfile-dev ports: + - "80:80" - "443:443" volumes: - .:/opt/certbot/src diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 000000000..8fd0f127d --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,52 @@ +
+ {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} + +
+ +
+

+ + © Copyright 2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license. + +
+
+ + Let's Encrypt Status + + + {%- if build_id and build_url %} + {% trans build_url=build_url, build_id=build_id %} + + Build + {{ build_id }}. + + {% endtrans %} + {%- elif commit %} + {% trans commit=commit %} + + Revision {{ commit }}. + + {% endtrans %} + {%- elif last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} + {%- endif %} + +

+
+ + {%- if show_sphinx %} + {% trans %}Built with Sphinx using a theme provided by Read the Docs{% endtrans %}. + {%- endif %} + + {%- block extrafooter %} {% endblock %} + +
diff --git a/docs/cli-help.txt b/docs/cli-help.txt index f7318f0b3..abebdb9c9 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,9 +107,9 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.21.0 (certbot; - Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY - (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + "". (default: CertbotACMEClient/0.21.1 (certbot; + darwin 10.13.3) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags encoded in the user agent are: --duplicate, --force- renew, --allow-subset-of-names, -n, and whether any hooks are set. @@ -448,11 +448,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary. (default: - a2enmod) + Path to the Apache 'a2enmod' binary. (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary. (default: - a2dismod) + Path to the Apache 'a2dismod' binary. (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension. (default: -le- ssl.conf) @@ -466,13 +464,13 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration. (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for - you.(Only Ubuntu/Debian currently) (default: True) + you.(Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you.(Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 diff --git a/docs/conf.py b/docs/conf.py index 73df47dbd..09bb44285 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,8 @@ master_doc = 'index' # General information about the project. project = u'Certbot' -copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' +# this is now overridden by the footer.html template +#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/contributing.rst b/docs/contributing.rst index c144a4f74..654528e3d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -32,10 +32,13 @@ a new plugin is introduced. .. code-block:: shell cd certbot - ./certbot-auto --os-packages-only + sudo ./certbot-auto --os-packages-only ./tools/venv.sh -Then in each shell where you're working on the client, do: +You can now run the copy of Certbot from git either by executing +``venv/bin/certbot``, or by activating the virtual environment. If you're +actively modifying and testing the code, you may want to run commands like this in +each shell where you're working: .. code-block:: shell @@ -43,7 +46,8 @@ Then in each shell where you're working on the client, do: export SERVER=https://acme-staging.api.letsencrypt.org/directory source tests/integration/_common.sh -After that, your shell will be using the virtual environment, and you run the +After that, your shell will be using the virtual environment, your copy of +Certbot will default to requesting test (staging) certificates, and you run the client by typing `certbot` or `certbot_test`. The latter is an alias that includes several flags useful for testing. For instance, it sets various output directories to point to /tmp/, and uses non-privileged ports for challenges, so @@ -418,7 +422,7 @@ OS-level dependencies can be installed like so: In general... * ``sudo`` is required as a suggested way of running privileged process -* `Python`_ 2.6/2.7 is required +* `Python`_ 2.7 is required * `Augeas`_ is required for the Python bindings * ``virtualenv`` and ``pip`` are used for managing other python library dependencies diff --git a/docs/install.rst b/docs/install.rst index a914586ff..aec885b62 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,7 +22,7 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +Certbot currently requires Python 2.7, or 3.4+. By default, it requires root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver @@ -116,7 +116,7 @@ certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, -from the server that the domain you're requesting a cert for resolves +from the server that the domain you're requesting a certficate for resolves to, `install Docker`_, then issue the following command: .. code-block:: shell diff --git a/docs/using.rst b/docs/using.rst index ab4670052..e8f84e2d7 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -557,8 +557,8 @@ apologize for any inconvenience you encounter in integrating these commands into your individual environment. .. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. - This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. - If you write a custom script and expect to run a command only after a cert was actually renewed + This means ``certbot renew`` exit status will be 0 if no certificate needs to be updated. + If you write a custom script and expect to run a command only after a certificate was actually renewed you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal and when renewal is not necessary. diff --git a/letsencrypt-auto b/letsencrypt-auto index 558c330b2..d3a5c23e5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.21.0" +LE_AUTO_VERSION="0.21.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -761,13 +761,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +858,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 9df8013a0..f28fd9893 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v2 -iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n -e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW -AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL -RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn -IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf -yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo= -=d/hS +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X +dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3 +JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc +a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3 +0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR +zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR +F/j/631OC6yLRasr+xcL1gc+SCryfA== +=tkZT -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9c8ccf344..f97dc078d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -761,13 +764,8 @@ BootstrapMageiaCommon() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -863,6 +861,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -1190,24 +1199,24 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -1231,6 +1240,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -1264,14 +1274,14 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) @@ -1279,18 +1289,14 @@ maybe_argparse = ( PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -1311,12 +1317,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -1326,8 +1333,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -1340,6 +1348,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -1347,11 +1373,13 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: @@ -1422,6 +1450,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 2e52f03fd..8dd683775 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 96e5c2db0..9d0f27009 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed. --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit + --install-only install certbot, upgrade if needed, and exit -v, --verbose provide more output -q, --quiet provide only update/error output; implies --non-interactive @@ -60,6 +61,8 @@ for arg in "$@" ; do DEBUG=1;; --os-packages-only) OS_PACKAGES_ONLY=1;; + --install-only) + INSTALL_ONLY=1;; --no-self-upgrade) # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) @@ -246,7 +249,7 @@ DeprecationBootstrap() { fi } -MIN_PYTHON_VERSION="2.6" +MIN_PYTHON_VERSION="2.7" MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//') # Sets LE_PYTHON to Python version string and PYVER to the first two # digits of the python version @@ -300,13 +303,8 @@ DeterminePythonVersion() { # Set Bootstrap to the function that installs OS dependencies on this system # and BOOTSTRAP_VERSION to the unique identifier for the current version of # that function. If Bootstrap is set to a function that doesn't install any -# packages (either because --no-bootstrap was included on the command line or -# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } -elif [ -f /etc/debian_version ]; then +# packages BOOTSTRAP_VERSION is not set. +if [ -f /etc/debian_version ]; then Bootstrap() { BootstrapMessage "Debian-based OSes" BootstrapDebCommon @@ -402,6 +400,17 @@ else } fi +# We handle this case after determining the normal bootstrap version to allow +# variables like USE_PYTHON_3 to be properly set. As described above, if the +# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not +# be set so we unset it here. +if [ "$NO_BOOTSTRAP" = 1 ]; then + Bootstrap() { + : + } + unset BOOTSTRAP_VERSION +fi + # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set # if it is unknown how OS dependencies were installed on this system. @@ -581,6 +590,12 @@ UNLIKELY_EOF say "Installation succeeded." fi + + if [ "$INSTALL_ONLY" = 1 ]; then + say "Certbot is installed." + exit 0 + fi + "$VENV_BIN/letsencrypt" "$@" else diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 406086fb5..0a5994afc 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.21.0 \ - --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \ - --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857 -acme==0.21.0 \ - --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \ - --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d -certbot-apache==0.21.0 \ - --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \ - --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b -certbot-nginx==0.21.0 \ - --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \ - --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642 +certbot==0.21.1 \ + --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \ + --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea +acme==0.21.1 \ + --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \ + --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b +certbot-apache==0.21.1 \ + --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \ + --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5 +certbot-nginx==0.21.1 \ + --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \ + --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 78491b5e3..d55d5bceb 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 +"""A small script that can act as a trust root for installing pip >=8 Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, @@ -23,6 +23,7 @@ anything goes wrong, it will exit with a non-zero status code. from __future__ import print_function from distutils.version import StrictVersion from hashlib import sha256 +from os import environ from os.path import join from pipes import quote from shutil import rmtree @@ -56,14 +57,14 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 3, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' +DEFAULT_INDEX_BASE = 'https://pypi.python.org' # wheel has a conditional dependency on argparse: maybe_argparse = ( - [('https://pypi.python.org/packages/18/dd/' - 'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' + [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] if version_info < (2, 7, 0) else []) @@ -71,18 +72,14 @@ maybe_argparse = ( PACKAGES = maybe_argparse + [ # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/11/b6/' - 'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz' - .format(PIP_VERSION), + ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' + 'pip-{0}.tar.gz'.format(PIP_VERSION), '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/69/65/' - '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/' - 'setuptools-20.2.2.tar.gz', - '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), - ('https://pypi.python.org/packages/c9/1d/' - 'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' + ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/' + 'setuptools-29.0.1.tar.gz', + 'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'), + ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' 'wheel-0.29.0.tar.gz', '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') ] @@ -103,12 +100,13 @@ def hashed_download(url, temp, digest): # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert # authenticity has only privacy (not arbitrary code execution) # implications, since we're checking hashes. - def opener(): + def opener(using_https=True): opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) + if using_https: + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) return opener def read_chunks(response, chunk_size): @@ -118,8 +116,9 @@ def hashed_download(url, temp, digest): break yield chunk - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) + parsed_url = urlparse(url) + response = opener(using_https=parsed_url.scheme == 'https').open(url) + path = join(temp, parsed_url.path.split('/')[-1]) actual_hash = sha256() with open(path, 'wb') as file: for chunk in read_chunks(response, 4096): @@ -132,6 +131,24 @@ def hashed_download(url, temp, digest): return path +def get_index_base(): + """Return the URL to the dir containing the "packages" folder. + + Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the + end if it's there; that is likely to give us the right dir. + + """ + env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') + if env_var: + SIMPLE = '/simple' + if env_var.endswith(SIMPLE): + return env_var[:-len(SIMPLE)] + else: + return env_var + else: + return DEFAULT_INDEX_BASE + + def main(): pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) @@ -139,11 +156,13 @@ def main(): if pip_version >= min_pip_version: return 0 has_pip_cache = pip_version >= StrictVersion('6.0') - + index_base = get_index_base() temp = mkdtemp(prefix='pipstrap-') try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] + downloads = [hashed_download(index_base + '/packages/' + path, + temp, + digest) + for path, digest in PACKAGES] check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh index a0e96edf8..2c6dcf734 100644 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ b/letsencrypt-auto-source/tests/centos6_tests.sh @@ -69,5 +69,13 @@ fi echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." echo "" +export VENV_PATH=$(mktemp -d) +"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1 +if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then + echo "Python 3 wasn't used with --no-bootstrap!" + exit 1 +fi +unset VENV_PATH + # test using python3 pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index b13057ca5..f77a6a1b0 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -5,7 +5,6 @@ from __future__ import print_function import argparse import atexit -import contextlib import os import re import shutil @@ -302,8 +301,7 @@ def main(): make_and_verify_selection(args.server_root, tempdir) tarpath = os.path.join(tempdir, "config.tar.gz") - # contextlib.closing used for py26 support - with contextlib.closing(tarfile.open(tarpath, mode="w:gz")) as tar: + with tarfile.open(tarpath, mode="w:gz") as tar: tar.add(tempdir, arcname=".") # TODO: Submit tarpath diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 3ce442b3e..b5be07a59 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -24,6 +24,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: System Administrators', @@ -31,10 +32,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/local-oldest-requirements.txt b/local-oldest-requirements.txt new file mode 100644 index 000000000..2346300a3 --- /dev/null +++ b/local-oldest-requirements.txt @@ -0,0 +1 @@ +-e acme[dev] diff --git a/setup.py b/setup.py index ce505a62e..3667a6976 100644 --- a/setup.py +++ b/setup.py @@ -34,31 +34,25 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme=={0}'.format(version), + # Remember to update local-oldest-requirements.txt when changing the + # minimum acme version. + 'acme>0.21.1', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=1.2', # load_pem_x509_certificate + 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', - # For pkg_resources. >=1.0 so pip resolves it to a version cryptography - # will tolerate; see #2599: - 'setuptools>=1.0', + 'setuptools', 'zope.component', 'zope.interface', ] -# env markers cause problems with older pip and setuptools -if sys.version_info < (2, 7): - install_requires.extend([ - 'argparse', - 'ordereddict', - ]) - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', @@ -89,6 +83,7 @@ setup( author="Certbot Project", author_email='client-dev@letsencrypt.org', license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', @@ -98,10 +93,8 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 08eb736c2..fc9cbaae7 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -16,6 +16,13 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1} [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml + +# If we're testing against ACMEv2, we need to use a newer boulder config for +# now. See https://github.com/letsencrypt/boulder#quickstart. +if [ "$BOULDER_INTEGRATION" = "v2" ]; then + sed -i 's/BOULDER_CONFIG_DIR: .*/BOULDER_CONFIG_DIR: test\/config-next/' docker-compose.yml +fi + docker-compose up -d set +x # reduce verbosity while waiting for boulder diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..9748befa3 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -233,6 +233,7 @@ certname="dns.le.wtf" common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ --pre-hook 'echo wtf2.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf2.post >> "$HOOK_TEST"' \ --renew-hook 'echo deploy >> "$HOOK_TEST"' @@ -251,7 +252,7 @@ openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ - --key-path "${root}/csr/key.pem" + --key-path "${root}/key.pem" CheckCertCount() { CERTCOUNT=`ls "${root}/conf/archive/$1/cert"* | wc -l` @@ -326,6 +327,19 @@ CheckDirHooks 1 common renew --cert-name le2.wtf CheckDirHooks 1 +# manual-dns-auth.sh will skip completing the challenge for domains that begin +# with fail. +common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ + --allow-subset-of-names \ + --preferred-challenges dns,tls-sni \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh + +if common certificates | grep "fail\.dns1\.le\.wtf"; then + echo "certificate should not have been issued for domain!" >&2 + exit 1 +fi + # ECDSA openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ @@ -430,6 +444,13 @@ for path in $archive $conf $live; do fi done +# Test ACMEv2-only features +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then + common -a manual -d '*.le4.wtf,le4.wtf' --preferred-challenges dns \ + --manual-auth-hook ./tests/manual-dns-auth.sh \ + --manual-cleanup-hook ./tests/manual-dns-cleanup.sh +fi + # Most CI systems set this variable to true. # If the tests are running as part of CI, Nginx should be available. if ${CI:-false} || type nginx; @@ -437,4 +458,4 @@ then . ./certbot-nginx/tests/boulder-integration.sh fi -coverage report --fail-under 64 -m +coverage report --fail-under 67 -m diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index d151bdc3f..a8d35ed89 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -16,9 +16,14 @@ certbot_test () { "$@" } +# Use local ACMEv2 endpoint if requested and SERVER isn't already set. +if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then + SERVER="http://localhost:4001/directory" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" - omit_patterns="$omit_patterns,*_test.py,*_test_*," + omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" omit_patterns="$omit_patterns,certbot-compatibility-test/*,certbot-dns*/" coverage run \ --append \ diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index d9491939c..17740cde8 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -38,6 +38,7 @@ from multiprocessing import Manager import urllib2 import yaml import boto3 +from botocore.exceptions import ClientError import fabric from fabric.api import run, execute, local, env, sudo, cd, lcd from fabric.operations import get, put @@ -141,7 +142,7 @@ def make_instance(instance_name, # give instance a name try: new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) - except botocore.exceptions.ClientError as e: + except ClientError as e: if "InvalidInstanceID.NotFound" in str(e): # This seems to be ephemeral... retry time.sleep(1) diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 51472f2e6..355fead2e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -65,6 +65,7 @@ iQIDAQAB " if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + RUN_PYTHON3_TESTS=1 if command -v python3; then echo "Didn't expect Python 3 to be installed!" exit 1 @@ -85,11 +86,25 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; exit 1 fi unset VENV_PATH - EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) - if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then - echo "Certbot didn't upgrade as expected!" - exit 1 - fi +fi + +if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then + echo "Had problems checking for updates!" + exit 1 +fi + +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2) +if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then + echo upgrade appeared to fail + exit 1 +fi + +if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then + echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ + exit 1 +fi + +if [ "$RUN_PYTHON3_TESTS" = 1 ]; then if ! command -v python3; then echo "Python3 wasn't properly installed" exit 1 @@ -98,11 +113,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; echo "Python3 wasn't used in venv!" exit 1 fi -elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo upgrade appeared to fail - exit 1 fi - echo upgrade appeared to be successful if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index 4bed2dd3a..e6ab836b8 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,11 +1,13 @@ #!/bin/sh -xe +LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" VENV_NAME=venv # *-auto respects VENV_PATH -letsencrypt/certbot-auto --debug --os-packages-only --non-interactive -LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version +$LE_AUTO --os-packages-only +LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate # change to an empty directory to ensure CWD doesn't affect tests diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 4c2eb429e..84e4bcd22 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -15,10 +15,4 @@ VENV_BIN=${VENV_PATH}/bin cd letsencrypt ./tools/venv.sh -PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - -if [ $PYVER -eq 26 ] ; then - venv/bin/tox -e py26 -else - venv/bin/tox -e py27 -fi +venv/bin/tox -e py27 diff --git a/tests/manual-dns-auth.sh b/tests/manual-dns-auth.sh index 9b9a1a5eb..febecf455 100755 --- a/tests/manual-dns-auth.sh +++ b/tests/manual-dns-auth.sh @@ -1,4 +1,8 @@ -#!/bin/sh -curl -X POST 'http://localhost:8055/set-txt' -d \ - "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ - \"value\": \"$CERTBOT_VALIDATION\"}" +#!/bin/bash + +# If domain begins with fail, fail the challenge by not completing it. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/set-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\", \ + \"value\": \"$CERTBOT_VALIDATION\"}" +fi diff --git a/tests/manual-dns-cleanup.sh b/tests/manual-dns-cleanup.sh new file mode 100755 index 000000000..1c09e892c --- /dev/null +++ b/tests/manual-dns-cleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# If domain begins with fail, we didn't complete the challenge so there is +# nothing to clean up. +if [[ "$CERTBOT_DOMAIN" != fail* ]]; then + curl -X POST 'http://localhost:8055/clear-txt' -d \ + "{\"host\": \"_acme-challenge.$CERTBOT_DOMAIN.\"}" +fi diff --git a/tests/run_http_server.py b/tests/run_http_server.py index fd1163816..0e4f8ac79 100644 --- a/tests/run_http_server.py +++ b/tests/run_http_server.py @@ -3,7 +3,7 @@ import sys # Run Python's built-in HTTP server # Usage: python ./tests/run_http_server.py port_num -# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ +# NOTE: This script should be compatible with 2.7, 3.4+ # sys.argv (port number) is passed as-is to the HTTP server module runpy.run_module( diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..f0385470b 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -18,10 +18,6 @@ for requirement in "$@" ; do pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] if [ $pkg = "." ]; then pkg="certbot" - else - # Work around a bug in pytest/importlib for the deprecated Python 3.3. - # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. - pkg=$(echo "$pkg" | tr - _) fi "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43..b385c5482 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,18 +1,30 @@ #!/bin/bash -e # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. # get the root of the Certbot repo tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" +all_constraints=$(mktemp) test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT +trap "rm -f $all_constraints $test_constraints" EXIT if [ "$CERTBOT_OLDEST" = 1 ]; then + if [ "$1" != "-e" -o "$#" -ne "2" ]; then + echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." + exit 1 + fi + pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] + requirements="$pkg_dir/local-oldest-requirements.txt" + # packages like acme don't have any local oldest requirements + if [ ! -f "$requirements" ]; then + unset requirements + fi cp "$tools_dir/oldest_constraints.txt" "$test_constraints" else repo_root=$(dirname "$tools_dir") @@ -20,7 +32,13 @@ else sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" fi +"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ + "$test_constraints" > "$all_constraints" + set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +if [ -n "$requirements" ]; then + pip install -q --constraint "$all_constraints" --requirement "$requirements" +fi +pip install -q --constraint "$all_constraints" "$@" diff --git a/tox.ini b/tox.ini index 20f5cda32..049220bbb 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{26,33,34,35,36},cover,lint +envlist = modification,py{34,35,36},cover,lint [base] # pip installs the requested packages in editable mode @@ -14,25 +14,24 @@ pip_install = {toxinidir}/tools/pip_install_editable.sh # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh -py26_packages = +dns_packages = + certbot-dns-cloudflare \ + certbot-dns-cloudxns \ + certbot-dns-digitalocean \ + certbot-dns-dnsimple \ + certbot-dns-dnsmadeeasy \ + certbot-dns-google \ + certbot-dns-luadns \ + certbot-dns-nsone \ + certbot-dns-rfc2136 \ + certbot-dns-route53 +all_packages = acme[dev] \ .[dev] \ certbot-apache \ - certbot-dns-cloudflare \ - certbot-dns-digitalocean \ - certbot-dns-google \ - certbot-dns-rfc2136 \ - certbot-dns-route53 \ + {[base]dns_packages} \ certbot-nginx \ letshelp-certbot -non_py26_packages = - certbot-dns-cloudxns \ - certbot-dns-dnsimple \ - certbot-dns-dnsmadeeasy \ - certbot-dns-luadns \ - certbot-dns-nsone -all_packages = - {[base]py26_packages} {[base]non_py26_packages} install_packages = {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} source_paths = @@ -54,32 +53,15 @@ source_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py -[testenv:py26] -commands = - {[base]install_and_test} {[base]py26_packages} - python tests/lock_test.py -deps = - setuptools==36.8.0 - wheel==0.29.0 -passenv = TRAVIS - [testenv] commands = - {[testenv:py26]commands} - {[base]install_and_test} {[base]non_py26_packages} + {[base]install_and_test} {[base]all_packages} + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 passenv = - {[testenv:py26]passenv} - -[testenv:py33] -commands = - {[testenv]commands} -deps = - wheel==0.29.0 -passenv = - {[testenv]passenv} + TRAVIS [testenv:py27-oldest] commands = @@ -90,6 +72,47 @@ setenv = passenv = {[testenv]passenv} +[testenv:py27-acme-oldest] +commands = + {[base]install_and_test} acme[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-apache-oldest] +commands = + {[base]install_and_test} certbot-apache +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-certbot-oldest] +commands = + {[base]install_and_test} .[dev] +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-dns-oldest] +commands = + {[base]install_and_test} {[base]dns_packages} +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + +[testenv:py27-nginx-oldest] +commands = + {[base]install_and_test} certbot-nginx + python tests/lock_test.py +setenv = + {[testenv:py27-oldest]setenv} +passenv = + {[testenv:py27-oldest]passenv} + [testenv:py27_install] basepython = python2.7 commands = @@ -104,7 +127,6 @@ passenv = {[testenv]passenv} [testenv:lint] -# recent versions of pylint do not support Python 2.6 (#97, #187) basepython = python2.7 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will