diff --git a/.gitignore b/.gitignore index b63e40d1c..a01d2e1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ tests/letstest/*.pem tests/letstest/venv/ .venv + +# pytest cache +.cache diff --git a/.travis.yml b/.travis.yml index 359801622..3d41bfa4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,19 +13,32 @@ before_script: matrix: include: - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" - - python: "2.7" - env: TOXENV=lint - - python: "2.7" - env: TOXENV=py27-oldest + env: TOXENV=py27_install BOULDER_INTEGRATION=1 sudo: required services: docker + - python: "2.7" + env: TOXENV=cover FYI="this also tests py27" + - sudo: required + env: TOXENV=nginx_compat + services: docker + before_install: + addons: + - python: "2.7" + env: TOXENV=lint - python: "2.6" env: TOXENV=py26 sudo: required services: docker - python: "2.7" - env: TOXENV=py27_install BOULDER_INTEGRATION=1 + env: TOXENV=py27-oldest + sudo: required + services: docker + - python: "3.3" + env: TOXENV=py33 + sudo: required + services: docker + - python: "3.6" + env: TOXENV=py36 sudo: required services: docker - sudo: required @@ -33,55 +46,14 @@ matrix: services: docker before_install: addons: - - sudo: required - env: TOXENV=nginx_compat - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=le_auto_precise - services: docker - before_install: - addons: - sudo: required env: TOXENV=le_auto_trusty services: docker before_install: addons: - - sudo: required - env: TOXENV=le_auto_wheezy - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=le_auto_centos6 - services: docker - before_install: - addons: - - sudo: required - env: TOXENV=docker_dev - services: docker - before_install: - addons: - python: "2.7" env: TOXENV=apacheconftest sudo: required - - python: "3.3" - env: TOXENV=py33 - sudo: required - services: docker - - python: "3.4" - env: TOXENV=py34 - sudo: required - services: docker - - python: "3.5" - env: TOXENV=py35 - sudo: required - services: docker - - python: "3.6" - env: TOXENV=py36 - sudo: required - services: docker - python: "2.7" env: TOXENV=nginxroundtrip diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aaef4af1..4acfc0401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,42 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.20.0 - 2017-12-06 + +### Added + +* Certbot's ACME library now recognizes URL fields in challenge objects in + preparation for Let's Encrypt's new ACME endpoint. The value is still + accessible in our ACME library through the name "uri". + +### Changed + +* The Apache plugin now parses some distro specific Apache configuration files + on non-Debian systems allowing it to get a clearer picture on the running + configuration. Internally, these changes were structured so that external + contributors can easily write patches to make the plugin work in new Apache + configurations. +* Certbot better reports network failures by removing information about + connection retries from the error output. +* An unnecessary question when using Certbot's webroot plugin interactively has + been removed. + +### Fixed + +* Certbot's NGINX plugin no longer sometimes incorrectly reports that it was + unable to deploy a HTTP->HTTPS redirect when requesting Certbot to enable a + redirect for multiple domains. +* Problems where the Apache plugin was failing to find directives and + duplicating existing directives on openSUSE have been resolved. +* An issue running the test shipped with Certbot and some our DNS plugins with + older versions of mock have been resolved. +* On some systems, users reported strangely interleaved output depending on + when stdout and stderr were flushed. This problem was resolved by having + Certbot regularly flush these streams. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/44?closed=1 + ## 0.19.0 - 2017-10-04 ### Added diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index e8a0b16a8..618dda200 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -10,3 +10,12 @@ supported version: `draft-ietf-acme-01`_. https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 """ +import sys +import warnings + +if sys.version_info[:2] == (3, 3): + warnings.warn( + "Python 3.3 support will be dropped in the next release of " + "acme. Please upgrade your Python version.", + PendingDeprecationWarning, + ) #pragma: no cover diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 14641af10..96997297b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -6,13 +6,13 @@ import logging import socket from cryptography.hazmat.primitives import hashes # type: ignore +import josepy as jose import OpenSSL import requests from acme import errors from acme import crypto_util from acme import fields -from acme import jose logger = logging.getLogger(__name__) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 49e790102..834d569aa 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,6 +1,7 @@ """Tests for acme.challenges.""" import unittest +import josepy as jose import mock import OpenSSL import requests @@ -8,7 +9,6 @@ import requests from six.moves.urllib import parse as urllib_parse # pylint: disable=import-error from acme import errors -from acme import jose from acme import test_util CERT = test_util.load_comparable_cert('cert.pem') diff --git a/acme/acme/client.py b/acme/acme/client.py index 5785b806c..877560767 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -10,13 +10,13 @@ import time import six from six.moves import http_client # pylint: disable=import-error +import josepy as jose import OpenSSL import re import requests import sys from acme import errors -from acme import jose from acme import jws from acme import messages @@ -39,39 +39,21 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class Client(object): # pylint: disable=too-many-instance-attributes - """ACME client. - - .. todo:: - Clean up raised error types hierarchy, document, and handle (wrap) - instances of `.DeserializationError` raised in `from_json()`. +class ClientBase(object): # pylint: disable=too-many-instance-attributes + """ACME client base object. :ivar messages.Directory directory: - :ivar key: `.JWK` (private) - :ivar alg: `.JWASignature` - :ivar bool verify_ssl: Verify SSL certificates? - :ivar .ClientNetwork net: Client network. Useful for testing. If not - supplied, it will be initialized using `key`, `alg` and - `verify_ssl`. - + :ivar .ClientNetwork net: Client network. """ - def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True, - net=None): + def __init__(self, directory, net=None): """Initialize. - :param directory: Directory Resource (`.messages.Directory`) or - URI from which the resource will be downloaded. - + :param .messages.Directory directory: Directory Resource + :param .ClientNetwork net: Client network. """ - self.key = key - self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net - - if isinstance(directory, six.string_types): - self.directory = messages.Directory.from_json( - self.net.get(directory).json()) - else: - self.directory = directory + self.directory = directory + self.net = net @classmethod def _regr_from_response(cls, response, uri=None, terms_of_service=None): @@ -83,26 +65,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes uri=response.headers.get('Location', uri), terms_of_service=terms_of_service) - def register(self, new_reg=None): - """Register. - - :param .NewRegistration new_reg: - - :returns: Registration Resource. - :rtype: `.RegistrationResource` - - """ - new_reg = messages.NewRegistration() if new_reg is None else new_reg - assert isinstance(new_reg, messages.NewRegistration) - - response = self.net.post(self.directory[new_reg], new_reg) - # TODO: handle errors - assert response.status_code == http_client.CREATED - - # "Instance of 'Field' has no key/contact member" bug: - # pylint: disable=no-member - return self._regr_from_response(response) - def _send_recv_regr(self, regr, body): response = self.net.post(regr.uri, body) @@ -153,21 +115,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes """ return self._send_recv_regr(regr, messages.UpdateRegistration()) - def agree_to_tos(self, regr): - """Agree to the terms-of-service. - - Agree to the terms-of-service in a Registration Resource. - - :param regr: Registration Resource. - :type regr: `.RegistrationResource` - - :returns: Updated Registration Resource. - :rtype: `.RegistrationResource` - - """ - return self.update_registration( - regr.update(body=regr.body.update(agreement=regr.terms_of_service))) - def _authzr_from_response(self, response, identifier=None, uri=None): authzr = messages.AuthorizationResource( body=messages.Authorization.from_json(response.json()), @@ -176,34 +123,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def _order_resource_from_response(self, response, uri=None): - body = messages.Order.from_json(response.json()) - authorizations = [] - for url in body.authorizations: - authorizations.append(self._authzr_from_response(self.net.get(url))) - fullchain_pem = None - if body.certificate is not None: - certificate_response = self.net.get(body.certificate, content_type=None) - if certificate_response.ok: - fullchain_pem = certificate_response.text - return messages.OrderResource( - body=body, - uri=response.headers.get('Location', uri), - fullchain_pem=fullchain_pem, - authorizations=authorizations) - - def new_order(self, csr_pem): - """Request challenges. - - :returns: List of Authorization Resources. - :rtype: `list` of `.AuthorizationResource` - """ - csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) - order = messages.NewOrder(csr=jose.ComparableX509(csr)) - response = self.net.post(self.directory.new_order, order) - order_response = self._order_resource_from_response(response) - return order_response - def request_challenges(self, identifier, new_authzr_uri=None): """Request challenges. @@ -316,38 +235,135 @@ class Client(object): # pylint: disable=too-many-instance-attributes response, authzr.body.identifier, authzr.uri) return updated_authzr, response - def poll_order_and_request_issuance(self, orderr): - """Poll Order Resource for status. + def revoke(self, cert, rsn): + """Revoke certificate. - :param orderr: Order Resource - :type orderr: `.OrderResource` + :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in + `.ComparableX509` - :returns: Updated Order Resource + :param int rsn: Reason code for certificate revocation. - :rtype: (`.OrderResource`) + :raises .ClientError: If revocation is unsuccessful. """ - while True: - response = self.net.get(orderr.uri) - latest = self._order_resource_from_response(response, uri=orderr.uri) - if any([a.body.status == messages.STATUS_PENDING for a in latest.authorizations]): - time.sleep(1) - else: - break - print latest.to_json() - for authz in latest.authorizations: - if authz.body.status != messages.STATUS_VALID: - for chall in authz.body.challenges: - if chall.error != None: - raise Exception("failed challenge for %s: %s" % - (authz.body.identifier.value, chall.error)) - raise Exception("failed authorization: %s" % authz.body) - while latest.fullchain_pem is None: - time.sleep(1) - response = self.net.get(orderr.uri) - latest = self._order_resource_from_response(response, uri=orderr.uri) - return latest + response = self.net.post(self.directory[messages.Revocation], + messages.Revocation( + certificate=cert, + reason=rsn), + content_type=None) + if response.status_code != http_client.OK: + raise errors.ClientError( + 'Successful revocation must return HTTP OK status') +class Client(ClientBase): + """ACME client for a v1 API. + + .. todo:: + Clean up raised error types hierarchy, document, and handle (wrap) + instances of `.DeserializationError` raised in `from_json()`. + + :ivar messages.Directory directory: + :ivar key: `.JWK` (private) + :ivar account: `.Registration` (private) + :ivar acme_version: `int` (private) + :ivar alg: `.JWASignature` + :ivar bool verify_ssl: Verify SSL certificates? + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + + """ + + def __init__(self, directory, key, account=None, acme_version=1, alg=jose.RS256, + verify_ssl=True, net=None): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + + """ + # pylint: disable=too-many-arguments + self.key = key + self.account = account + self.acme_version = acme_version + self.net = ClientNetwork(key, account=account, acme_version=acme_version, + alg=alg, verify_ssl=verify_ssl) if net is None else net + + if isinstance(directory, six.string_types): + self.directory = messages.Directory.from_json( + self.net.get(directory).json()) + else: + self.directory = directory + super(Client, self).__init__(directory=directory, net=net) + + def register(self, new_reg=None): + """Register. + + :param .NewRegistration new_reg: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` + + """ + new_reg = messages.NewRegistration() if new_reg is None else new_reg + response = self.net.post(self.directory[new_reg], new_reg) + # TODO: handle errors + assert response.status_code == http_client.CREATED + + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def agree_to_tos(self, regr): + """Agree to the terms-of-service. + + Agree to the terms-of-service in a Registration Resource. + + :param regr: Registration Resource. + :type regr: `.RegistrationResource` + + :returns: Updated Registration Resource. + :rtype: `.RegistrationResource` + + """ + return self.update_registration( + regr.update(body=regr.body.update(agreement=regr.terms_of_service))) + + def request_challenges(self, identifier, new_authzr_uri=None): + """Request challenges. + + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") + new_authz = messages.NewAuthorization(identifier=identifier) + response = self.net.post(self.directory.new_authz, new_authz) + # TODO: handle errors + assert response.status_code == http_client.CREATED + return self._authzr_from_response(response, identifier) + + def request_domain_challenges(self, domain, new_authzr_uri=None): + """Request challenges for domain names. + + This is simply a convenience function that wraps around + `request_challenges`, but works with domain names instead of + generic identifiers. See ``request_challenges`` for more + documentation. + + :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. + + :returns: Authorization Resource. + :rtype: `.AuthorizationResource` + + """ + return self.request_challenges(messages.Identifier( + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) def request_issuance(self, csr, authzrs): """Request issuance. @@ -469,7 +485,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param str uri: URI of certificate :returns: tuple of the form - (response, :class:`acme.jose.ComparableX509`) + (response, :class:`josepy.util.ComparableX509`) :rtype: tuple """ @@ -542,25 +558,99 @@ class Client(object): # pylint: disable=too-many-instance-attributes "Recursion limit reached. Didn't get {0}".format(uri)) return chain - def revoke(self, cert, rsn): - """Revoke certificate. - :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in - `.ComparableX509` - :param int rsn: Reason code for certificate revocation. +class ClientV2(ClientBase): + """ACME client for a v2 API. - :raises .ClientError: If revocation is unsuccessful. + :ivar messages.Directory directory: + :ivar .ClientNetwork net: Client network. Useful for testing. If not + supplied, it will be initialized using `key`, `alg` and + `verify_ssl`. + """ + + def __init__(self, directory, net): + """Initialize. + + :param directory: Directory Resource (`.messages.Directory`) or + URI from which the resource will be downloaded. + :ivar .ClientNetwork net: Client network. + """ + super(ClientV2, self).__init__(directory=directory, net=net) + + def new_account(self, new_account): + """Register. + + :param .NewRegistration new_account: + + :returns: Registration Resource. + :rtype: `.RegistrationResource` """ - response = self.net.post(self.directory[messages.Revocation], - messages.Revocation( - certificate=cert, - reason=rsn), - content_type=None) - if response.status_code != http_client.OK: - raise errors.ClientError( - 'Successful revocation must return HTTP OK status') + response = self.net.post(self.directory.new_account, new_account) + # "Instance of 'Field' has no key/contact member" bug: + # pylint: disable=no-member + return self._regr_from_response(response) + + def new_order(self, csr_pem): + """Request challenges. + + :returns: List of Authorization Resources. + :rtype: `list` of `.AuthorizationResource` + """ + csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) + order = messages.NewOrder(csr=jose.ComparableX509(csr)) + response = self.net.post(self.directory.new_order, order) + order_response = self._order_resource_from_response(response) + return order_response + + def poll_order_and_request_issuance(self, orderr): + """Poll Order Resource for status. + + :param orderr: Order Resource + :type orderr: `.OrderResource` + + :returns: Updated Order Resource + + :rtype: (`.OrderResource`) + + """ + while True: + response = self.net.get(orderr.uri) + latest = self._order_resource_from_response(response, uri=orderr.uri) + if any([a.body.status == messages.STATUS_PENDING for a in latest.authorizations]): + time.sleep(1) + else: + break + print latest.to_json() + for authz in latest.authorizations: + if authz.body.status != messages.STATUS_VALID: + for chall in authz.body.challenges: + if chall.error != None: + raise Exception("failed challenge for %s: %s" % + (authz.body.identifier.value, chall.error)) + raise Exception("failed authorization: %s" % authz.body) + while latest.fullchain_pem is None: + time.sleep(1) + response = self.net.get(orderr.uri) + latest = self._order_resource_from_response(response, uri=orderr.uri) + return latest + + def _order_resource_from_response(self, response, uri=None): + body = messages.Order.from_json(response.json()) + authorizations = [] + for url in body.authorizations: + authorizations.append(self._authzr_from_response(self.net.get(url))) + fullchain_pem = None + if body.certificate is not None: + certificate_response = self.net.get(body.certificate, content_type=None) + if certificate_response.ok: + fullchain_pem = certificate_response.text + return messages.OrderResource( + body=body, + uri=response.headers.get('Location', uri), + fullchain_pem=fullchain_pem, + authorizations=authorizations) class ClientNetwork(object): # pylint: disable=too-many-instance-attributes @@ -570,15 +660,19 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes JSON_ERROR_CONTENT_TYPE = 'application/problem+json' REPLAY_NONCE_HEADER = 'Replay-Nonce' - def __init__(self, key, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): + def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True, + user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT, + acme_version=1): + # pylint: disable=too-many-arguments self.key = key + self.account = account self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout + self.acme_version = acme_version def __del__(self): # Try to close the session, but don't show exceptions to the @@ -588,21 +682,31 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except Exception: # pylint: disable=broad-except pass - def _wrap_in_jws(self, obj, nonce): + def _wrap_in_jws(self, obj, nonce, url): """Wrap `JSONDeSerializable` object in JWS. .. todo:: Implement ``acmePath``. :param .JSONDeSerializable obj: + :param str url: The URL to which this object will be POSTed :param bytes nonce: :rtype: `.JWS` """ jobj = obj.json_dumps(indent=2).encode() logger.debug('JWS payload:\n%s', jobj) - return jws.JWS.sign( - payload=jobj, key=self.key, alg=self.alg, - nonce=nonce).json_dumps(indent=2) + kwargs = { + "alg": self.alg, + "nonce": nonce + } + if self.acme_version == 2: + # new ACME spec + kwargs["url"] = url + if self.account is not None: + kwargs["kid"] = self.account["uri"] + kwargs["key"] = self.key + # pylint: disable=star-args + return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2) @classmethod def _check_response(cls, response, content_type=None): @@ -776,7 +880,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes raise def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs): - data = self._wrap_in_jws(obj, self._get_nonce(url)) + data = self._wrap_in_jws(obj, self._get_nonce(url), url) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) self._add_nonce(response) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 4bd762865..aa9e6e041 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -5,12 +5,12 @@ import unittest from six.moves import http_client # pylint: disable=import-error +import josepy as jose import mock import requests from acme import challenges from acme import errors -from acme import jose from acme import jws as acme_jws from acme import messages from acme import messages_test @@ -104,6 +104,23 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.register(self.new_reg)) # TODO: test POST call arguments + def test_new_account_v2(self): + directory = messages.Directory({ + "new-account": 'https://www.letsencrypt-demo.org/acme/new-account', + }) + from acme.client import ClientV2 + client = ClientV2(directory=directory, net=self.net) + self.response.status_code = http_client.CREATED + self.response.json.return_value = self.regr.body.to_json() + self.response.headers['Location'] = self.regr.uri + + self.regr = messages.RegistrationResource( + body=messages.Registration( + contact=self.contact, key=KEY.public_key()), + uri='https://www.letsencrypt-demo.org/acme/reg/1') + + self.assertEqual(self.regr, client.new_account(self.regr)) + def test_update_registration(self): # "Instance of 'Field' has no to_json/update member" bug: # pylint: disable=no-member @@ -432,9 +449,22 @@ class ClientTest(unittest.TestCase): self.certr, self.rsn) +class MockJSONDeSerializable(jose.JSONDeSerializable): + # pylint: disable=missing-docstring + def __init__(self, value): + self.value = value + + def to_partial_json(self): + return {'foo': self.value} + + @classmethod + def from_json(cls, value): + pass # pragma: no cover + class ClientNetworkTest(unittest.TestCase): """Tests for acme.client.ClientNetwork.""" + # pylint: disable=too-many-public-methods def setUp(self): self.verify_ssl = mock.MagicMock() @@ -453,25 +483,26 @@ class ClientNetworkTest(unittest.TestCase): self.assertTrue(self.net.verify_ssl is self.verify_ssl) def test_wrap_in_jws(self): - class MockJSONDeSerializable(jose.JSONDeSerializable): - # pylint: disable=missing-docstring - def __init__(self, value): - self.value = value - - def to_partial_json(self): - return {'foo': self.value} - - @classmethod - def from_json(cls, value): - pass # pragma: no cover - # pylint: disable=protected-access jws_dump = self.net._wrap_in_jws( - MockJSONDeSerializable('foo'), nonce=b'Tg') + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url") jws = acme_jws.JWS.json_loads(jws_dump) self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) self.assertEqual(jws.signature.combined.nonce, b'Tg') + def test_wrap_in_jws_v2(self): + self.net.account = {'uri': 'acct-uri'} + self.net.acme_version = 2 + # pylint: disable=protected-access + jws_dump = self.net._wrap_in_jws( + MockJSONDeSerializable('foo'), nonce=b'Tg', url="url") + jws = acme_jws.JWS.json_loads(jws_dump) + self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'}) + self.assertEqual(jws.signature.combined.nonce, b'Tg') + self.assertEqual(jws.signature.combined.kid, u'acct-uri') + self.assertEqual(jws.signature.combined.url, u'url') + + def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -701,13 +732,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): self.assertEqual(self.checked_response, self.net.post( 'uri', self.obj, content_type=self.content_type)) self.net._wrap_in_jws.assert_called_once_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri") self.available_nonces = [] self.assertRaises(errors.MissingNonce, self.net.post, 'uri', self.obj, content_type=self.content_type) self.net._wrap_in_jws.assert_called_with( - self.obj, jose.b64decode(self.all_nonces.pop())) + self.obj, jose.b64decode(self.all_nonces.pop()), "uri") def test_post_wrong_initial_nonce(self): # HEAD self.available_nonces = [b'f', jose.b64encode(b'good')] diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index da433c5a2..1d7f83ccf 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,10 +8,10 @@ import unittest import six from six.moves import socketserver #type: ignore # pylint: disable=import-error +import josepy as jose import OpenSSL from acme import errors -from acme import jose from acme import test_util diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 9d991fd75..de5f9d1f4 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -1,5 +1,5 @@ """ACME errors.""" -from acme.jose import errors as jose_errors +from josepy import errors as jose_errors class Error(Exception): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 12d09acf4..d7ec78403 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -1,10 +1,9 @@ """ACME JSON fields.""" import logging +import josepy as jose import pyrfc3339 -from acme import jose - logger = logging.getLogger(__name__) diff --git a/acme/acme/fields_test.py b/acme/acme/fields_test.py index de852b6fa..69dde8b89 100644 --- a/acme/acme/fields_test.py +++ b/acme/acme/fields_test.py @@ -2,10 +2,9 @@ import datetime import unittest +import josepy as jose import pytz -from acme import jose - class FixedTest(unittest.TestCase): """Tests for acme.fields.Fixed.""" diff --git a/acme/acme/jose/__init__.py b/acme/acme/jose/__init__.py deleted file mode 100644 index 9116bc433..000000000 --- a/acme/acme/jose/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Javascript Object Signing and Encryption (jose). - -This package is a Python implementation of the standards developed by -IETF `Javascript Object Signing and Encryption (Active WG)`_, in -particular the following RFCs: - - - `JSON Web Algorithms (JWA)`_ - - `JSON Web Key (JWK)`_ - - `JSON Web Signature (JWS)`_ - - -.. _`Javascript Object Signing and Encryption (Active WG)`: - https://tools.ietf.org/wg/jose/ - -.. _`JSON Web Algorithms (JWA)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/ - -.. _`JSON Web Key (JWK)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/ - -.. _`JSON Web Signature (JWS)`: - https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/ - -""" -from acme.jose.b64 import ( - b64decode, - b64encode, -) - -from acme.jose.errors import ( - DeserializationError, - SerializationError, - Error, - UnrecognizedTypeError, -) - -from acme.jose.interfaces import JSONDeSerializable - -from acme.jose.json_util import ( - Field, - JSONObjectWithFields, - TypedJSONObjectWithFields, - decode_b64jose, - decode_cert, - decode_csr, - decode_hex16, - encode_b64jose, - encode_cert, - encode_csr, - encode_hex16, -) - -from acme.jose.jwa import ( - HS256, - HS384, - HS512, - JWASignature, - PS256, - PS384, - PS512, - RS256, - RS384, - RS512, -) - -from acme.jose.jwk import ( - JWK, - JWKRSA, -) - -from acme.jose.jws import ( - Header, - JWS, - Signature, -) - -from acme.jose.util import ( - ComparableX509, - ComparableKey, - ComparableRSAKey, - ImmutableMap, -) diff --git a/acme/acme/jose/b64.py b/acme/acme/jose/b64.py deleted file mode 100644 index cf79aa820..000000000 --- a/acme/acme/jose/b64.py +++ /dev/null @@ -1,61 +0,0 @@ -"""JOSE Base64. - -`JOSE Base64`_ is defined as: - - - URL-safe Base64 - - padding stripped - - -.. _`JOSE Base64`: - https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C - -.. Do NOT try to call this module "base64", as it will "shadow" the - standard library. - -""" -import base64 - -import six - - -def b64encode(data): - """JOSE Base64 encode. - - :param data: Data to be encoded. - :type data: `bytes` - - :returns: JOSE Base64 string. - :rtype: bytes - - :raises TypeError: if `data` is of incorrect type - - """ - if not isinstance(data, six.binary_type): - raise TypeError('argument should be {0}'.format(six.binary_type)) - return base64.urlsafe_b64encode(data).rstrip(b'=') - - -def b64decode(data): - """JOSE Base64 decode. - - :param data: Base64 string to be decoded. If it's unicode, then - only ASCII characters are allowed. - :type data: `bytes` or `unicode` - - :returns: Decoded data. - :rtype: bytes - - :raises TypeError: if input is of incorrect type - :raises ValueError: if input is unicode with non-ASCII characters - - """ - if isinstance(data, six.string_types): - try: - data = data.encode('ascii') - except UnicodeEncodeError: - raise ValueError( - 'unicode argument should contain only ASCII characters') - elif not isinstance(data, six.binary_type): - raise TypeError('argument should be a str or unicode') - - return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4))) diff --git a/acme/acme/jose/b64_test.py b/acme/acme/jose/b64_test.py deleted file mode 100644 index cbabe2251..000000000 --- a/acme/acme/jose/b64_test.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Tests for acme.jose.b64.""" -import unittest - -import six - - -# https://en.wikipedia.org/wiki/Base64#Examples -B64_PADDING_EXAMPLES = { - b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='), - b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='), - b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''), - b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='), - b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='), -} - - -B64_URL_UNSAFE_EXAMPLES = { - six.int2byte(251) + six.int2byte(239): b'--8', - six.int2byte(255) * 2: b'__8', -} - - -class B64EncodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64encode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64encode - return b64encode(data) - - def test_empty(self): - self.assertEqual(self._call(b''), b'') - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_different_paddings(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(text), b64) - - def test_unicode_fails_with_type_error(self): - self.assertRaises(TypeError, self._call, u'some unicode') - - -class B64DecodeTest(unittest.TestCase): - """Tests for acme.jose.b64.b64decode.""" - - @classmethod - def _call(cls, data): - from acme.jose.b64 import b64decode - return b64decode(data) - - def test_unsafe_url(self): - for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_without_padding(self): - for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64), text) - - def test_input_with_padding(self): - for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES): - self.assertEqual(self._call(b64 + pad), text) - - def test_unicode_with_ascii(self): - self.assertEqual(self._call(u'YQ'), b'a') - - def test_non_ascii_unicode_fails(self): - self.assertRaises(ValueError, self._call, u'\u0105') - - def test_type_error_no_unicode_or_bytes(self): - self.assertRaises(TypeError, self._call, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/errors.py b/acme/acme/jose/errors.py deleted file mode 100644 index 74c9443e1..000000000 --- a/acme/acme/jose/errors.py +++ /dev/null @@ -1,35 +0,0 @@ -"""JOSE errors.""" - - -class Error(Exception): - """Generic JOSE Error.""" - - -class DeserializationError(Error): - """JSON deserialization error.""" - - def __str__(self): - return "Deserialization error: {0}".format( - super(DeserializationError, self).__str__()) - - -class SerializationError(Error): - """JSON serialization error.""" - - -class UnrecognizedTypeError(DeserializationError): - """Unrecognized type error. - - :ivar str typ: The unrecognized type of the JSON object. - :ivar jobj: Full JSON object. - - """ - - def __init__(self, typ, jobj): - self.typ = typ - self.jobj = jobj - super(UnrecognizedTypeError, self).__init__(str(self)) - - def __str__(self): - return '{0} was not recognized, full message: {1}'.format( - self.typ, self.jobj) diff --git a/acme/acme/jose/errors_test.py b/acme/acme/jose/errors_test.py deleted file mode 100644 index 919980920..000000000 --- a/acme/acme/jose/errors_test.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for acme.jose.errors.""" -import unittest - - -class UnrecognizedTypeErrorTest(unittest.TestCase): - def setUp(self): - from acme.jose.errors import UnrecognizedTypeError - self.error = UnrecognizedTypeError('foo', {'type': 'foo'}) - - def test_str(self): - self.assertEqual( - "foo was not recognized, full message: {'type': 'foo'}", - str(self.error)) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py deleted file mode 100644 index f841848b3..000000000 --- a/acme/acme/jose/interfaces.py +++ /dev/null @@ -1,216 +0,0 @@ -"""JOSE interfaces.""" -import abc -import collections -import json - -import six - -from acme.jose import errors -from acme.jose import util - -# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class -# pylint: disable=too-few-public-methods - - -@six.add_metaclass(abc.ABCMeta) -class JSONDeSerializable(object): - # pylint: disable=too-few-public-methods - """Interface for (de)serializable JSON objects. - - Please recall, that standard Python library implements - :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform - translations based on respective :ref:`conversion tables - ` that look pretty much like the one below (for - complete tables see relevant Python documentation): - - .. _conversion-table: - - ====== ====== - JSON Python - ====== ====== - object dict - ... ... - ====== ====== - - While the above **conversion table** is about translation of JSON - documents to/from the basic Python types only, - :class:`JSONDeSerializable` introduces the following two concepts: - - serialization - Turning an arbitrary Python object into Python object that can - be encoded into a JSON document. **Full serialization** produces - a Python object composed of only basic types as required by the - :ref:`conversion table `. **Partial - serialization** (accomplished by :meth:`to_partial_json`) - produces a Python object that might also be built from other - :class:`JSONDeSerializable` objects. - - deserialization - Turning a decoded Python object (necessarily one of the basic - types as required by the :ref:`conversion table - `) into an arbitrary Python object. - - Serialization produces **serialized object** ("partially serialized - object" or "fully serialized object" for partial and full - serialization respectively) and deserialization produces - **deserialized object**, both usually denoted in the source code as - ``jobj``. - - Wording in the official Python documentation might be confusing - after reading the above, but in the light of those definitions, one - can view :meth:`json.JSONDecoder.decode` as decoder and - deserializer of basic types, :meth:`json.JSONEncoder.default` as - serializer of basic types, :meth:`json.JSONEncoder.encode` as - serializer and encoder of basic types. - - One could extend :mod:`json` to support arbitrary object - (de)serialization either by: - - - overriding :meth:`json.JSONDecoder.decode` and - :meth:`json.JSONEncoder.default` in subclasses - - - or passing ``object_hook`` argument (or ``object_hook_pairs``) - to :func:`json.load`/:func:`json.loads` or ``default`` argument - for :func:`json.dump`/:func:`json.dumps`. - - Interestingly, ``default`` is required to perform only partial - serialization, as :func:`json.dumps` applies ``default`` - recursively. This is the idea behind making :meth:`to_partial_json` - produce only partial serialization, while providing custom - :meth:`json_dumps` that dumps with ``default`` set to - :meth:`json_dump_default`. - - To make further documentation a bit more concrete, please, consider - the following imaginatory implementation example:: - - class Foo(JSONDeSerializable): - def to_partial_json(self): - return 'foo' - - @classmethod - def from_json(cls, jobj): - return Foo() - - class Bar(JSONDeSerializable): - def to_partial_json(self): - return [Foo(), Foo()] - - @classmethod - def from_json(cls, jobj): - return Bar() - - """ - - @abc.abstractmethod - def to_partial_json(self): # pragma: no cover - """Partially serialize. - - Following the example, **partial serialization** means the following:: - - assert isinstance(Bar().to_partial_json()[0], Foo) - assert isinstance(Bar().to_partial_json()[1], Foo) - - # in particular... - assert Bar().to_partial_json() != ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Partially serializable object. - - """ - raise NotImplementedError() - - def to_json(self): - """Fully serialize. - - Again, following the example from before, **full serialization** - means the following:: - - assert Bar().to_json() == ['foo', 'foo'] - - :raises acme.jose.errors.SerializationError: - in case of any serialization error. - :returns: Fully serialized object. - - """ - def _serialize(obj): - if isinstance(obj, JSONDeSerializable): - return _serialize(obj.to_partial_json()) - if isinstance(obj, six.string_types): # strings are Sequence - return obj - elif isinstance(obj, list): - return [_serialize(subobj) for subobj in obj] - elif isinstance(obj, collections.Sequence): - # default to tuple, otherwise Mapping could get - # unhashable list - return tuple(_serialize(subobj) for subobj in obj) - elif isinstance(obj, collections.Mapping): - return dict((_serialize(key), _serialize(value)) - for key, value in six.iteritems(obj)) - else: - return obj - - return _serialize(self) - - @util.abstractclassmethod - def from_json(cls, jobj): # pylint: disable=unused-argument - """Deserialize a decoded JSON document. - - :param jobj: Python object, composed of only other basic data - types, as decoded from JSON document. Not necessarily - :class:`dict` (as decoded from "JSON object" document). - - :raises acme.jose.errors.DeserializationError: - if decoding was unsuccessful, e.g. in case of unparseable - X509 certificate, or wrong padding in JOSE base64 encoded - string, etc. - - """ - # TypeError: Can't instantiate abstract class with - # abstract methods from_json, to_partial_json - return cls() # pylint: disable=abstract-class-instantiated - - @classmethod - def json_loads(cls, json_string): - """Deserialize from JSON document string.""" - try: - loads = json.loads(json_string) - except ValueError as error: - raise errors.DeserializationError(error) - return cls.from_json(loads) - - def json_dumps(self, **kwargs): - """Dump to JSON string using proper serializer. - - :returns: JSON document string. - :rtype: str - - """ - return json.dumps(self, default=self.json_dump_default, **kwargs) - - def json_dumps_pretty(self): - """Dump the object to pretty JSON document string. - - :rtype: str - - """ - return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) - - @classmethod - def json_dump_default(cls, python_object): - """Serialize Python object. - - This function is meant to be passed as ``default`` to - :func:`json.dump` or :func:`json.dumps`. They call - ``default(python_object)`` only for non-basic Python types, so - this function necessarily raises :class:`TypeError` if - ``python_object`` is not an instance of - :class:`IJSONSerializable`. - - Please read the class docstring for more information. - - """ - if isinstance(python_object, JSONDeSerializable): - return python_object.to_partial_json() - else: # this branch is necessary, cannot just "return" - raise TypeError(repr(python_object) + ' is not JSON serializable') diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py deleted file mode 100644 index cf98ff371..000000000 --- a/acme/acme/jose/interfaces_test.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Tests for acme.jose.interfaces.""" -import unittest - - -class JSONDeSerializableTest(unittest.TestCase): - # pylint: disable=too-many-instance-attributes - - def setUp(self): - from acme.jose.interfaces import JSONDeSerializable - - # pylint: disable=missing-docstring,invalid-name - - class Basic(JSONDeSerializable): - def __init__(self, v): - self.v = v - - def to_partial_json(self): - return self.v - - @classmethod - def from_json(cls, jobj): - return cls(jobj) - - class Sequence(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return [self.x, self.y] - - @classmethod - def from_json(cls, jobj): - return cls( - Basic.from_json(jobj[0]), Basic.from_json(jobj[1])) - - class Mapping(JSONDeSerializable): - def __init__(self, x, y): - self.x = x - self.y = y - - def to_partial_json(self): - return {self.x: self.y} - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - - self.basic1 = Basic('foo1') - self.basic2 = Basic('foo2') - self.seq = Sequence(self.basic1, self.basic2) - self.mapping = Mapping(self.basic1, self.basic2) - self.nested = Basic([[self.basic1]]) - self.tuple = Basic(('foo',)) - - # pylint: disable=invalid-name - self.Basic = Basic - self.Sequence = Sequence - self.Mapping = Mapping - - def test_to_json_sequence(self): - self.assertEqual(self.seq.to_json(), ['foo1', 'foo2']) - - def test_to_json_mapping(self): - self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'}) - - def test_to_json_other(self): - mock_value = object() - self.assertTrue(self.Basic(mock_value).to_json() is mock_value) - - def test_to_json_nested(self): - self.assertEqual(self.nested.to_json(), [['foo1']]) - - def test_to_json(self): - self.assertEqual(self.tuple.to_json(), (('foo', ))) - - def test_from_json_not_implemented(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx') - - def test_json_loads(self): - seq = self.Sequence.json_loads('["foo1", "foo2"]') - self.assertTrue(isinstance(seq, self.Sequence)) - self.assertTrue(isinstance(seq.x, self.Basic)) - self.assertTrue(isinstance(seq.y, self.Basic)) - self.assertEqual(seq.x.v, 'foo1') - self.assertEqual(seq.y.v, 'foo2') - - def test_json_dumps(self): - self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps()) - - def test_json_dumps_pretty(self): - self.assertEqual(self.seq.json_dumps_pretty(), - '[\n "foo1",\n "foo2"\n]') - - def test_json_dump_default(self): - from acme.jose.interfaces import JSONDeSerializable - - self.assertEqual( - 'foo1', JSONDeSerializable.json_dump_default(self.basic1)) - - jobj = JSONDeSerializable.json_dump_default(self.seq) - self.assertEqual(len(jobj), 2) - self.assertTrue(jobj[0] is self.basic1) - self.assertTrue(jobj[1] is self.basic2) - - def test_json_dump_default_type_error(self): - from acme.jose.interfaces import JSONDeSerializable - self.assertRaises( - TypeError, JSONDeSerializable.json_dump_default, object()) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py deleted file mode 100644 index 4baadda5e..000000000 --- a/acme/acme/jose/json_util.py +++ /dev/null @@ -1,485 +0,0 @@ -"""JSON (de)serialization framework. - -The framework presented here is somewhat based on `Go's "json" package`_ -(especially the ``omitempty`` functionality). - -.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/ - -""" -import abc -import binascii -import logging - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class Field(object): - """JSON object field. - - :class:`Field` is meant to be used together with - :class:`JSONObjectWithFields`. - - ``encoder`` (``decoder``) is a callable that accepts a single - parameter, i.e. a value to be encoded (decoded), and returns the - serialized (deserialized) value. In case of errors it should raise - :class:`~acme.jose.errors.SerializationError` - (:class:`~acme.jose.errors.DeserializationError`). - - Note, that ``decoder`` should perform partial serialization only. - - :ivar str json_name: Name of the field when encoded to JSON. - :ivar default: Default value (used when not present in JSON object). - :ivar bool omitempty: If ``True`` and the field value is empty, then - it will not be included in the serialized JSON object, and - ``default`` will be used for deserialization. Otherwise, if ``False``, - field is considered as required, value will always be included in the - serialized JSON objected, and it must also be present when - deserializing. - - """ - __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc') - - def __init__(self, json_name, default=None, omitempty=False, - decoder=None, encoder=None): - # pylint: disable=too-many-arguments - self.json_name = json_name - self.default = default - self.omitempty = omitempty - - self.fdec = self.default_decoder if decoder is None else decoder - self.fenc = self.default_encoder if encoder is None else encoder - - @classmethod - def _empty(cls, value): - """Is the provided value considered "empty" for this field? - - This is useful for subclasses that might want to override the - definition of being empty, e.g. for some more exotic data types. - - """ - return not isinstance(value, bool) and not value - - def omit(self, value): - """Omit the value in output?""" - return self._empty(value) and self.omitempty - - def _update_params(self, **kwargs): - current = dict(json_name=self.json_name, default=self.default, - omitempty=self.omitempty, - decoder=self.fdec, encoder=self.fenc) - current.update(kwargs) - return type(self)(**current) # pylint: disable=star-args - - def decoder(self, fdec): - """Descriptor to change the decoder on JSON object field.""" - return self._update_params(decoder=fdec) - - def encoder(self, fenc): - """Descriptor to change the encoder on JSON object field.""" - return self._update_params(encoder=fenc) - - def decode(self, value): - """Decode a value, optionally with context JSON object.""" - return self.fdec(value) - - def encode(self, value): - """Encode a value, optionally with context JSON object.""" - return self.fenc(value) - - @classmethod - def default_decoder(cls, value): - """Default decoder. - - Recursively deserialize into immutable types ( - :class:`acme.jose.util.frozendict` instead of - :func:`dict`, :func:`tuple` instead of :func:`list`). - - """ - # bases cases for different types returned by json.loads - if isinstance(value, list): - return tuple(cls.default_decoder(subvalue) for subvalue in value) - elif isinstance(value, dict): - return util.frozendict( - dict((cls.default_decoder(key), cls.default_decoder(value)) - for key, value in six.iteritems(value))) - else: # integer or string - return value - - @classmethod - def default_encoder(cls, value): - """Default (passthrough) encoder.""" - # field.to_partial_json() is no good as encoder has to do partial - # serialization only - return value - - -class JSONObjectWithFieldsMeta(abc.ABCMeta): - """Metaclass for :class:`JSONObjectWithFields` and its subclasses. - - It makes sure that, for any class ``cls`` with ``__metaclass__`` - set to ``JSONObjectWithFieldsMeta``: - - 1. All fields (attributes of type :class:`Field`) in the class - definition are moved to the ``cls._fields`` dictionary, where - keys are field attribute names and values are fields themselves. - - 2. ``cls.__slots__`` is extended by all field attribute names - (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__`` - are stored in ``cls._orig_slots``. - - In a consequence, for a field attribute name ``some_field``, - ``cls.some_field`` will be a slot descriptor and not an instance - of :class:`Field`. For example:: - - some_field = Field('someField', default=()) - - class Foo(object): - __metaclass__ = JSONObjectWithFieldsMeta - __slots__ = ('baz',) - some_field = some_field - - assert Foo.__slots__ == ('some_field', 'baz') - assert Foo._orig_slots == () - assert Foo.some_field is not Field - - assert Foo._fields.keys() == ['some_field'] - assert Foo._fields['some_field'] is some_field - - As an implementation note, this metaclass inherits from - :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate - the metaclass conflict (:class:`ImmutableMap` and - :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`, - use :class:`abc.ABCMeta` as its metaclass). - - """ - - def __new__(mcs, name, bases, dikt): - fields = {} - - for base in bases: - fields.update(getattr(base, '_fields', {})) - # Do not reorder, this class might override fields from base classes! - for key, value in tuple(six.iteritems(dikt)): - # not six.iterkeys() (in-place edit!) - if isinstance(value, Field): - fields[key] = dikt.pop(key) - - dikt['_orig_slots'] = dikt.get('__slots__', ()) - dikt['__slots__'] = tuple( - list(dikt['_orig_slots']) + list(six.iterkeys(fields))) - dikt['_fields'] = fields - - return abc.ABCMeta.__new__(mcs, name, bases, dikt) - - -@six.add_metaclass(JSONObjectWithFieldsMeta) -class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): - # pylint: disable=too-few-public-methods - """JSON object with fields. - - Example:: - - class Foo(JSONObjectWithFields): - bar = Field('Bar') - empty = Field('Empty', omitempty=True) - - @bar.encoder - def bar(value): - return value + 'bar' - - @bar.decoder - def bar(value): - if not value.endswith('bar'): - raise errors.DeserializationError('No bar suffix!') - return value[:-3] - - assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'} - assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz') - assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'}) - == Foo(bar='baz', empty='!')) - assert Foo(bar='baz').bar == 'baz' - - """ - - @classmethod - def _defaults(cls): - """Get default fields values.""" - return dict([(slot, field.default) for slot, field - in six.iteritems(cls._fields)]) - - def __init__(self, **kwargs): - # pylint: disable=star-args - super(JSONObjectWithFields, self).__init__( - **(dict(self._defaults(), **kwargs))) - - def encode(self, name): - """Encode a single field. - - :param str name: Name of the field to be encoded. - - :raises errors.SerializationError: if field cannot be serialized - :raises errors.Error: if field could not be found - - """ - try: - field = self._fields[name] - except KeyError: - raise errors.Error("Field not found: {0}".format(name)) - - return field.encode(getattr(self, name)) - - def fields_to_partial_json(self): - """Serialize fields to JSON.""" - jobj = {} - omitted = set() - for slot, field in six.iteritems(self._fields): - value = getattr(self, slot) - - if field.omit(value): - omitted.add((slot, value)) - else: - try: - jobj[field.json_name] = field.encode(value) - except errors.SerializationError as error: - raise errors.SerializationError( - 'Could not encode {0} ({1}): {2}'.format( - slot, value, error)) - return jobj - - def to_partial_json(self): - return self.fields_to_partial_json() - - @classmethod - def _check_required(cls, jobj): - missing = set() - for _, field in six.iteritems(cls._fields): - if not field.omitempty and field.json_name not in jobj: - missing.add(field.json_name) - - if missing: - raise errors.DeserializationError( - 'The following fields are required: {0}'.format( - ','.join(missing))) - - @classmethod - def fields_from_json(cls, jobj): - """Deserialize fields from JSON.""" - cls._check_required(jobj) - fields = {} - for slot, field in six.iteritems(cls._fields): - if field.json_name not in jobj and field.omitempty: - fields[slot] = field.default - else: - value = jobj[field.json_name] - try: - fields[slot] = field.decode(value) - except errors.DeserializationError as error: - raise errors.DeserializationError( - 'Could not decode {0!r} ({1!r}): {2}'.format( - slot, value, error)) - return fields - - @classmethod - def from_json(cls, jobj): - return cls(**cls.fields_from_json(jobj)) - - -def encode_b64jose(data): - """Encode JOSE Base-64 field. - - :param bytes data: - :rtype: `unicode` - - """ - # b64encode produces ASCII characters only - return b64.b64encode(data).decode('ascii') - - -def decode_b64jose(data, size=None, minimum=False): - """Decode JOSE Base-64 field. - - :param unicode data: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - error_cls = TypeError if six.PY2 else binascii.Error - try: - decoded = b64.b64decode(data.encode()) - except error_cls as error: - raise errors.DeserializationError(error) - - if size is not None and ((not minimum and len(decoded) != size) or - (minimum and len(decoded) < size)): - raise errors.DeserializationError( - "Expected at least or exactly {0} bytes".format(size)) - - return decoded - - -def encode_hex16(value): - """Hexlify. - - :param bytes value: - :rtype: unicode - - """ - return binascii.hexlify(value).decode() - - -def decode_hex16(value, size=None, minimum=False): - """Decode hexlified field. - - :param unicode value: - :param int size: Required length (after decoding). - :param bool minimum: If ``True``, then `size` will be treated as - minimum required length, as opposed to exact equality. - - :rtype: bytes - - """ - value = value.encode() - if size is not None and ((not minimum and len(value) != size * 2) or - (minimum and len(value) < size * 2)): - raise errors.DeserializationError() - error_cls = TypeError if six.PY2 else binascii.Error - try: - return binascii.unhexlify(value) - except error_cls as error: - raise errors.DeserializationError(error) - - -def encode_cert(cert): - """Encode certificate as JOSE Base-64 DER. - - :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) - - -def decode_cert(b64der): - """Decode JOSE Base-64 DER-encoded certificate. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -def encode_csr(csr): - """Encode CSR as JOSE Base-64 DER. - - :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - :rtype: unicode - - """ - return encode_b64jose(OpenSSL.crypto.dump_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) - - -def decode_csr(b64der): - """Decode JOSE Base-64 DER-encoded CSR. - - :param unicode b64der: - :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` - - """ - try: - return util.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class TypedJSONObjectWithFields(JSONObjectWithFields): - """JSON object with type.""" - - typ = NotImplemented - """Type of the object. Subclasses must override.""" - - type_field_name = "type" - """Field name used to distinguish different object types. - - Subclasses will probably have to override this. - - """ - - TYPES = NotImplemented - """Types registered for JSON deserialization""" - - @classmethod - def register(cls, type_cls, typ=None): - """Register class for JSON deserialization.""" - typ = type_cls.typ if typ is None else typ - cls.TYPES[typ] = type_cls - return type_cls - - @classmethod - def get_type_cls(cls, jobj): - """Get the registered class for ``jobj``.""" - if cls in six.itervalues(cls.TYPES): - if cls.type_field_name not in jobj: - raise errors.DeserializationError( - "Missing type field ({0})".format(cls.type_field_name)) - # cls is already registered type_cls, force to use it - # so that, e.g Revocation.from_json(jobj) fails if - # jobj["type"] != "revocation". - return cls - - if not isinstance(jobj, dict): - raise errors.DeserializationError( - "{0} is not a dictionary object".format(jobj)) - try: - typ = jobj[cls.type_field_name] - except KeyError: - raise errors.DeserializationError("missing type field") - - try: - return cls.TYPES[typ] - except KeyError: - raise errors.UnrecognizedTypeError(typ, jobj) - - def to_partial_json(self): - """Get JSON serializable object. - - :returns: Serializable JSON object representing ACME typed object. - :meth:`validate` will almost certainly not work, due to reasons - explained in :class:`acme.interfaces.IJSONSerializable`. - :rtype: dict - - """ - jobj = self.fields_to_partial_json() - jobj[self.type_field_name] = self.typ - return jobj - - @classmethod - def from_json(cls, jobj): - """Deserialize ACME object from valid JSON object. - - :raises acme.errors.UnrecognizedTypeError: if type - of the ACME object has not been registered. - - """ - # make sure subclasses don't cause infinite recursive from_json calls - type_cls = cls.get_type_cls(jobj) - return type_cls(**type_cls.fields_from_json(jobj)) diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py deleted file mode 100644 index 25e36211e..000000000 --- a/acme/acme/jose/json_util_test.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Tests for acme.jose.json_util.""" -import itertools -import unittest - -import mock -import six - -from acme import test_util - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import util - - -CERT = test_util.load_comparable_cert('cert.pem') -CSR = test_util.load_comparable_csr('csr.pem') - - -class FieldTest(unittest.TestCase): - """Tests for acme.jose.json_util.Field.""" - - def test_no_omit_boolean(self): - from acme.jose.json_util import Field - for default, omitempty, value in itertools.product( - [True, False], [True, False], [True, False]): - self.assertFalse( - Field("foo", default=default, omitempty=omitempty).omit(value)) - - def test_descriptors(self): - mock_value = mock.MagicMock() - - # pylint: disable=missing-docstring - - def decoder(unused_value): - return 'd' - - def encoder(unused_value): - return 'e' - - from acme.jose.json_util import Field - field = Field('foo') - - field = field.encoder(encoder) - self.assertEqual('e', field.encode(mock_value)) - - field = field.decoder(decoder) - self.assertEqual('e', field.encode(mock_value)) - self.assertEqual('d', field.decode(mock_value)) - - def test_default_encoder_is_partial(self): - class MockField(interfaces.JSONDeSerializable): - # pylint: disable=missing-docstring - def to_partial_json(self): - return 'foo' # pragma: no cover - - @classmethod - def from_json(cls, jobj): - pass # pragma: no cover - mock_field = MockField() - - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_field) is mock_field) - # in particular... - self.assertNotEqual('foo', Field.default_encoder(mock_field)) - - def test_default_encoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_encoder(mock_value) is mock_value) - - def test_default_decoder_list_to_tuple(self): - from acme.jose.json_util import Field - self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3])) - - def test_default_decoder_dict_to_frozendict(self): - from acme.jose.json_util import Field - obj = Field.default_decoder({'x': 2}) - self.assertTrue(isinstance(obj, util.frozendict)) - self.assertEqual(obj, util.frozendict(x=2)) - - def test_default_decoder_passthrough(self): - mock_value = mock.MagicMock() - from acme.jose.json_util import Field - self.assertTrue(Field.default_decoder(mock_value) is mock_value) - - -class JSONObjectWithFieldsMetaTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta.""" - - def setUp(self): - from acme.jose.json_util import Field - from acme.jose.json_util import JSONObjectWithFieldsMeta - self.field = Field('Baz') - self.field2 = Field('Baz2') - # pylint: disable=invalid-name,missing-docstring,too-few-public-methods - # pylint: disable=blacklisted-name - - @six.add_metaclass(JSONObjectWithFieldsMeta) - class A(object): - __slots__ = ('bar',) - baz = self.field - - class B(A): - pass - - class C(A): - baz = self.field2 - - self.a_cls = A - self.b_cls = B - self.c_cls = C - - def test_fields(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field}, self.a_cls._fields) - self.assertEqual({'baz': self.field}, self.b_cls._fields) - - def test_fields_inheritance(self): - # pylint: disable=protected-access,no-member - self.assertEqual({'baz': self.field2}, self.c_cls._fields) - - def test_slots(self): - self.assertEqual(('bar', 'baz'), self.a_cls.__slots__) - self.assertEqual(('baz',), self.b_cls.__slots__) - - def test_orig_slots(self): - # pylint: disable=protected-access,no-member - self.assertEqual(('bar',), self.a_cls._orig_slots) - self.assertEqual((), self.b_cls._orig_slots) - - -class JSONObjectWithFieldsTest(unittest.TestCase): - """Tests for acme.jose.json_util.JSONObjectWithFields.""" - # pylint: disable=protected-access - - def setUp(self): - from acme.jose.json_util import JSONObjectWithFields - from acme.jose.json_util import Field - - class MockJSONObjectWithFields(JSONObjectWithFields): - # pylint: disable=invalid-name,missing-docstring,no-self-argument - # pylint: disable=too-few-public-methods - x = Field('x', omitempty=True, - encoder=(lambda x: x * 2), - decoder=(lambda x: x / 2)) - y = Field('y') - z = Field('Z') # on purpose uppercase - - @y.encoder - def y(value): - if value == 500: - raise errors.SerializationError() - return value - - @y.decoder - def y(value): - if value == 500: - raise errors.DeserializationError() - return value - - # pylint: disable=invalid-name - self.MockJSONObjectWithFields = MockJSONObjectWithFields - self.mock = MockJSONObjectWithFields(x=None, y=2, z=3) - - def test_init_defaults(self): - self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3)) - - def test_encode(self): - self.assertEqual(10, self.MockJSONObjectWithFields( - x=5, y=0, z=0).encode("x")) - - def test_encode_wrong_field(self): - self.assertRaises(errors.Error, self.mock.encode, 'foo') - - def test_encode_serialization_error_passthrough(self): - self.assertRaises( - errors.SerializationError, - self.MockJSONObjectWithFields(y=500, z=None).encode, "y") - - def test_fields_to_partial_json_omits_empty(self): - self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3}) - - def test_fields_from_json_fills_default_for_empty(self): - self.assertEqual( - {'x': None, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3})) - - def test_fields_from_json_fails_on_missing(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'Z': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0}) - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0}) - - def test_fields_to_partial_json_encoder(self): - self.assertEqual( - self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(), - {'x': 2, 'y': 2, 'Z': 3}) - - def test_fields_from_json_decoder(self): - self.assertEqual( - {'x': 2, 'y': 2, 'z': 3}, - self.MockJSONObjectWithFields.fields_from_json( - {'x': 4, 'y': 2, 'Z': 3})) - - def test_fields_to_partial_json_error_passthrough(self): - self.assertRaises( - errors.SerializationError, self.MockJSONObjectWithFields( - x=1, y=500, z=3).to_partial_json) - - def test_fields_from_json_error_passthrough(self): - self.assertRaises( - errors.DeserializationError, - self.MockJSONObjectWithFields.from_json, - {'x': 4, 'y': 500, 'Z': 3}) - - -class DeEncodersTest(unittest.TestCase): - def setUp(self): - self.b64_cert = ( - u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM' - u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz' - u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF' - u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx' - u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI' - u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW' - u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD' - u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1' - u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE' - u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd' - u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o' - ) - self.b64_csr = ( - u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F' - u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw' - u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb' - u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As' - u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3' - u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG' - u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW' - u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg' - ) - - def test_encode_b64jose(self): - from acme.jose.json_util import encode_b64jose - encoded = encode_b64jose(b'x') - self.assertTrue(isinstance(encoded, six.string_types)) - self.assertEqual(u'eA', encoded) - - def test_decode_b64jose(self): - from acme.jose.json_util import decode_b64jose - decoded = decode_b64jose(u'eA') - self.assertTrue(isinstance(decoded, six.binary_type)) - self.assertEqual(b'x', decoded) - - def test_decode_b64jose_padding_error(self): - from acme.jose.json_util import decode_b64jose - self.assertRaises(errors.DeserializationError, decode_b64jose, u'x') - - def test_decode_b64jose_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3)) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=2) - self.assertRaises( - errors.DeserializationError, decode_b64jose, u'Zm9v', size=4) - - def test_decode_b64jose_minimum_size(self): - from acme.jose.json_util import decode_b64jose - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True)) - self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_b64jose, - u'Zm9v', size=4, minimum=True) - - def test_encode_hex16(self): - from acme.jose.json_util import encode_hex16 - encoded = encode_hex16(b'foo') - self.assertEqual(u'666f6f', encoded) - self.assertTrue(isinstance(encoded, six.string_types)) - - def test_decode_hex16(self): - from acme.jose.json_util import decode_hex16 - decoded = decode_hex16(u'666f6f') - self.assertEqual(b'foo', decoded) - self.assertTrue(isinstance(decoded, six.binary_type)) - - def test_decode_hex16_minimum_size(self): - from acme.jose.json_util import decode_hex16 - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True)) - self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True)) - self.assertRaises(errors.DeserializationError, decode_hex16, - u'666f6f', size=4, minimum=True) - - def test_decode_hex16_odd_length(self): - from acme.jose.json_util import decode_hex16 - self.assertRaises(errors.DeserializationError, decode_hex16, u'x') - - def test_encode_cert(self): - from acme.jose.json_util import encode_cert - self.assertEqual(self.b64_cert, encode_cert(CERT)) - - def test_decode_cert(self): - from acme.jose.json_util import decode_cert - cert = decode_cert(self.b64_cert) - self.assertTrue(isinstance(cert, util.ComparableX509)) - self.assertEqual(cert, CERT) - self.assertRaises(errors.DeserializationError, decode_cert, u'') - - def test_encode_csr(self): - from acme.jose.json_util import encode_csr - self.assertEqual(self.b64_csr, encode_csr(CSR)) - - def test_decode_csr(self): - from acme.jose.json_util import decode_csr - csr = decode_csr(self.b64_csr) - self.assertTrue(isinstance(csr, util.ComparableX509)) - self.assertEqual(csr, CSR) - self.assertRaises(errors.DeserializationError, decode_csr, u'') - - -class TypedJSONObjectWithFieldsTest(unittest.TestCase): - - def setUp(self): - from acme.jose.json_util import TypedJSONObjectWithFields - - # pylint: disable=missing-docstring,abstract-method - # pylint: disable=too-few-public-methods - - class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields): - TYPES = {} - type_field_name = 'type' - - @MockParentTypedJSONObjectWithFields.register - class MockTypedJSONObjectWithFields( - MockParentTypedJSONObjectWithFields): - typ = 'test' - __slots__ = ('foo',) - - @classmethod - def fields_from_json(cls, jobj): - return {'foo': jobj['foo']} - - def fields_to_partial_json(self): - return {'foo': self.foo} - - self.parent_cls = MockParentTypedJSONObjectWithFields - self.msg = MockTypedJSONObjectWithFields(foo='bar') - - def test_to_partial_json(self): - self.assertEqual(self.msg.to_partial_json(), { - 'type': 'test', - 'foo': 'bar', - }) - - def test_from_json_non_dict_fails(self): - for value in [[], (), 5, "asd"]: # all possible input types - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, value) - - def test_from_json_dict_no_type_fails(self): - self.assertRaises( - errors.DeserializationError, self.parent_cls.from_json, {}) - - def test_from_json_unknown_type_fails(self): - self.assertRaises(errors.UnrecognizedTypeError, - self.parent_cls.from_json, {'type': 'bar'}) - - def test_from_json_returns_obj(self): - self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json( - {'type': 'test', 'foo': 'bar'})) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py deleted file mode 100644 index 9b682ecab..000000000 --- a/acme/acme/jose/jwa.py +++ /dev/null @@ -1,180 +0,0 @@ -"""JSON Web Algorithm. - -https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40 - -""" -import abc -import collections -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import hmac # type: ignore -from cryptography.hazmat.primitives.asymmetric import padding # type: ignore - -from acme.jose import errors -from acme.jose import interfaces -from acme.jose import jwk - - -logger = logging.getLogger(__name__) - - -class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method - # pylint: disable=too-few-public-methods - # for some reason disable=abstract-method has to be on the line - # above... - """JSON Web Algorithm.""" - - -class JWASignature(JWA, collections.Hashable): # type: ignore - """JSON Web Signature Algorithm.""" - SIGNATURES = {} # type: dict - - def __init__(self, name): - self.name = name - - def __eq__(self, other): - if not isinstance(other, JWASignature): - return NotImplemented - return self.name == other.name - - def __hash__(self): - return hash((self.__class__, self.name)) - - def __ne__(self, other): - return not self == other - - @classmethod - def register(cls, signature_cls): - """Register class for JSON deserialization.""" - cls.SIGNATURES[signature_cls.name] = signature_cls - return signature_cls - - def to_partial_json(self): - return self.name - - @classmethod - def from_json(cls, jobj): - return cls.SIGNATURES[jobj] - - @abc.abstractmethod - def sign(self, key, msg): # pragma: no cover - """Sign the ``msg`` using ``key``.""" - raise NotImplementedError() - - @abc.abstractmethod - def verify(self, key, msg, sig): # pragma: no cover - """Verify the ``msg` and ``sig`` using ``key``.""" - raise NotImplementedError() - - def __repr__(self): - return self.name - - -class _JWAHS(JWASignature): - - kty = jwk.JWKOct - - def __init__(self, name, hash_): - super(_JWAHS, self).__init__(name) - self.hash = hash_() - - def sign(self, key, msg): - signer = hmac.HMAC(key, self.hash, backend=default_backend()) - signer.update(msg) - return signer.finalize() - - def verify(self, key, msg, sig): - verifier = hmac.HMAC(key, self.hash, backend=default_backend()) - verifier.update(msg) - try: - verifier.verify(sig) - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARSA(object): - - kty = jwk.JWKRSA - padding = NotImplemented - hash = NotImplemented - - def sign(self, key, msg): - """Sign the ``msg`` using ``key``.""" - try: - signer = key.signer(self.padding, self.hash) - except AttributeError as error: - logger.debug(error, exc_info=True) - raise errors.Error("Public key cannot be used for signing") - except ValueError as error: # digest too large - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - signer.update(msg) - try: - return signer.finalize() - except ValueError as error: - logger.debug(error, exc_info=True) - raise errors.Error(str(error)) - - def verify(self, key, msg, sig): - """Verify the ``msg` and ``sig`` using ``key``.""" - verifier = key.verifier(sig, self.padding, self.hash) - verifier.update(msg) - try: - verifier.verify() - except cryptography.exceptions.InvalidSignature as error: - logger.debug(error, exc_info=True) - return False - else: - return True - - -class _JWARS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWARS, self).__init__(name) - self.padding = padding.PKCS1v15() - self.hash = hash_() - - -class _JWAPS(_JWARSA, JWASignature): - - def __init__(self, name, hash_): - super(_JWAPS, self).__init__(name) - self.padding = padding.PSS( - mgf=padding.MGF1(hash_()), - salt_length=padding.PSS.MAX_LENGTH) - self.hash = hash_() - - -class _JWAES(JWASignature): # pylint: disable=abstract-class-not-used - - # TODO: implement ES signatures - - def sign(self, key, msg): # pragma: no cover - raise NotImplementedError() - - def verify(self, key, msg, sig): # pragma: no cover - raise NotImplementedError() - - -HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256)) -HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384)) -HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512)) - -RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256)) -RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384)) -RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512)) - -PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256)) -PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384)) -PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512)) - -ES256 = JWASignature.register(_JWAES('ES256')) -ES384 = JWASignature.register(_JWAES('ES384')) -ES512 = JWASignature.register(_JWAES('ES512')) diff --git a/acme/acme/jose/jwa_test.py b/acme/acme/jose/jwa_test.py deleted file mode 100644 index 3328d083a..000000000 --- a/acme/acme/jose/jwa_test.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tests for acme.jose.jwa.""" -import unittest - -from acme import test_util - -from acme.jose import errors - - -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') -RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem') - - -class JWASignatureTest(unittest.TestCase): - """Tests for acme.jose.jwa.JWASignature.""" - - def setUp(self): - from acme.jose.jwa import JWASignature - - class MockSig(JWASignature): - # pylint: disable=missing-docstring,too-few-public-methods - # pylint: disable=abstract-class-not-used - def sign(self, key, msg): - raise NotImplementedError() # pragma: no cover - - def verify(self, key, msg, sig): - raise NotImplementedError() # pragma: no cover - - # pylint: disable=invalid-name - self.Sig1 = MockSig('Sig1') - self.Sig2 = MockSig('Sig2') - - def test_eq(self): - self.assertEqual(self.Sig1, self.Sig1) - - def test_ne(self): - self.assertNotEqual(self.Sig1, self.Sig2) - - def test_ne_other_type(self): - self.assertNotEqual(self.Sig1, 5) - - def test_repr(self): - self.assertEqual('Sig1', repr(self.Sig1)) - self.assertEqual('Sig2', repr(self.Sig2)) - - def test_to_partial_json(self): - self.assertEqual(self.Sig1.to_partial_json(), 'Sig1') - self.assertEqual(self.Sig2.to_partial_json(), 'Sig2') - - def test_from_json(self): - from acme.jose.jwa import JWASignature - from acme.jose.jwa import RS256 - self.assertTrue(JWASignature.from_json('RS256') is RS256) - - -class JWAHSTest(unittest.TestCase): # pylint: disable=too-few-public-methods - - def test_it(self): - from acme.jose.jwa import HS256 - sig = ( - b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4" - b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO" - ) - self.assertEqual(HS256.sign(b'some key', b'foo'), sig) - self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True) - self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False) - - -class JWARSTest(unittest.TestCase): - - def test_sign_no_private_part(self): - from acme.jose.jwa import RS256 - self.assertRaises( - errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo') - - def test_sign_key_too_small(self): - from acme.jose.jwa import RS256 - from acme.jose.jwa import PS256 - self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo') - self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo') - - def test_rs(self): - from acme.jose.jwa import RS256 - sig = ( - b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O' - b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c' - b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99' - b'\xd2\xb9.>}\xfd' - ) - self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig) - self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig)) - self.assertFalse(RS256.verify( - RSA512_KEY.public_key(), b'foo', sig + b'!')) - - def test_ps(self): - from acme.jose.jwa import PS256 - sig = PS256.sign(RSA1024_KEY, b'foo') - self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig)) - self.assertFalse(PS256.verify( - RSA1024_KEY.public_key(), b'foo', sig + b'!')) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py deleted file mode 100644 index 54423f670..000000000 --- a/acme/acme/jose/jwk.py +++ /dev/null @@ -1,281 +0,0 @@ -"""JSON Web Key.""" -import abc -import binascii -import json -import logging - -import cryptography.exceptions -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes # type: ignore -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec # type: ignore -from cryptography.hazmat.primitives.asymmetric import rsa - -import six - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -logger = logging.getLogger(__name__) - - -class JWK(json_util.TypedJSONObjectWithFields): - # pylint: disable=too-few-public-methods - """JSON Web Key.""" - type_field_name = 'kty' - TYPES = {} # type: dict - cryptography_key_types = () # type: tuple - """Subclasses should override.""" - - required = NotImplemented - """Required members of public key's representation as defined by JWK/JWA.""" - - _thumbprint_json_dumps_params = { - # "no whitespace or line breaks before or after any syntactic - # elements" - 'indent': None, - 'separators': (',', ':'), - # "members ordered lexicographically by the Unicode [UNICODE] - # code points of the member names" - 'sort_keys': True, - } - - def thumbprint(self, hash_function=hashes.SHA256): - """Compute JWK Thumbprint. - - https://tools.ietf.org/html/rfc7638 - - :returns bytes: - - """ - digest = hashes.Hash(hash_function(), backend=default_backend()) - digest.update(json.dumps( - dict((k, v) for k, v in six.iteritems(self.to_json()) - if k in self.required), - **self._thumbprint_json_dumps_params).encode()) - return digest.finalize() - - @abc.abstractmethod - def public_key(self): # pragma: no cover - """Generate JWK with public key. - - For symmetric cryptosystems, this would return ``self``. - - """ - raise NotImplementedError() - - @classmethod - def _load_cryptography_key(cls, data, password=None, backend=None): - backend = default_backend() if backend is None else backend - exceptions = {} - - # private key? - for loader in (serialization.load_pem_private_key, - serialization.load_der_private_key): - try: - return loader(data, password, backend) - except (ValueError, TypeError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # public key? - for loader in (serialization.load_pem_public_key, - serialization.load_der_public_key): - try: - return loader(data, backend) - except (ValueError, - cryptography.exceptions.UnsupportedAlgorithm) as error: - exceptions[loader] = error - - # no luck - raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) - - @classmethod - def load(cls, data, password=None, backend=None): - """Load serialized key as JWK. - - :param str data: Public or private key serialized as PEM or DER. - :param str password: Optional password. - :param backend: A `.PEMSerializationBackend` and - `.DERSerializationBackend` provider. - - :raises errors.Error: if unable to deserialize, or unsupported - JWK algorithm - - :returns: JWK of an appropriate type. - :rtype: `JWK` - - """ - try: - key = cls._load_cryptography_key(data, password, backend) - except errors.Error as error: - logger.debug('Loading symmetric key, asymmetric failed: %s', error) - return JWKOct(key=data) - - if cls.typ is not NotImplemented and not isinstance( - key, cls.cryptography_key_types): - raise errors.Error('Unable to deserialize {0} into {1}'.format( - key.__class__, cls.__class__)) - for jwk_cls in six.itervalues(cls.TYPES): - if isinstance(key, jwk_cls.cryptography_key_types): - return jwk_cls(key=key) - raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) - - -@JWK.register -class JWKES(JWK): # pragma: no cover - # pylint: disable=abstract-class-not-used - """ES JWK. - - .. warning:: This is not yet implemented! - - """ - typ = 'ES' - cryptography_key_types = ( - ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) - required = ('crv', JWK.type_field_name, 'x', 'y') - - def fields_to_partial_json(self): - raise NotImplementedError() - - @classmethod - def fields_from_json(cls, jobj): - raise NotImplementedError() - - def public_key(self): - raise NotImplementedError() - - -@JWK.register -class JWKOct(JWK): - """Symmetric JWK.""" - typ = 'oct' - __slots__ = ('key',) - required = ('k', JWK.type_field_name) - - def fields_to_partial_json(self): - # TODO: An "alg" member SHOULD also be present to identify the - # algorithm intended to be used with the key, unless the - # application uses another means or convention to determine - # the algorithm used. - return {'k': json_util.encode_b64jose(self.key)} - - @classmethod - def fields_from_json(cls, jobj): - return cls(key=json_util.decode_b64jose(jobj['k'])) - - def public_key(self): - return self - - -@JWK.register -class JWKRSA(JWK): - """RSA JWK. - - :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey` - or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped - in `.ComparableRSAKey` - - """ - typ = 'RSA' - cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) - __slots__ = ('key',) - required = ('e', JWK.type_field_name, 'n') - - def __init__(self, *args, **kwargs): - if 'key' in kwargs and not isinstance( - kwargs['key'], util.ComparableRSAKey): - kwargs['key'] = util.ComparableRSAKey(kwargs['key']) - super(JWKRSA, self).__init__(*args, **kwargs) - - @classmethod - def _encode_param(cls, data): - """Encode Base64urlUInt. - - :type data: long - :rtype: unicode - - """ - def _leading_zeros(arg): - if len(arg) % 2: - return '0' + arg - return arg - - return json_util.encode_b64jose(binascii.unhexlify( - _leading_zeros(hex(data)[2:].rstrip('L')))) - - @classmethod - def _decode_param(cls, data): - """Decode Base64urlUInt.""" - try: - return int(binascii.hexlify(json_util.decode_b64jose(data)), 16) - except ValueError: # invalid literal for long() with base 16 - raise errors.DeserializationError() - - def public_key(self): - return type(self)(key=self.key.public_key()) - - @classmethod - def fields_from_json(cls, jobj): - # pylint: disable=invalid-name - n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e')) - public_numbers = rsa.RSAPublicNumbers(e=e, n=n) - if 'd' not in jobj: # public key - key = public_numbers.public_key(default_backend()) - else: # private key - d = cls._decode_param(jobj['d']) - if ('p' in jobj or 'q' in jobj or 'dp' in jobj or - 'dq' in jobj or 'qi' in jobj or 'oth' in jobj): - # "If the producer includes any of the other private - # key parameters, then all of the others MUST be - # present, with the exception of "oth", which MUST - # only be present when more than two prime factors - # were used." - p, q, dp, dq, qi, = all_params = tuple( - jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) - if tuple(param for param in all_params if param is None): - raise errors.Error( - 'Some private parameters are missing: {0}'.format( - all_params)) - p, q, dp, dq, qi = tuple( - cls._decode_param(x) for x in all_params) - - # TODO: check for oth - else: - # cryptography>=0.8 - p, q = rsa.rsa_recover_prime_factors(n, e, d) - dp = rsa.rsa_crt_dmp1(d, p) - dq = rsa.rsa_crt_dmq1(d, q) - qi = rsa.rsa_crt_iqmp(p, q) - - key = rsa.RSAPrivateNumbers( - p, q, d, dp, dq, qi, public_numbers).private_key( - default_backend()) - - return cls(key=key) - - def fields_to_partial_json(self): - # pylint: disable=protected-access - if isinstance(self.key._wrapped, rsa.RSAPublicKey): - numbers = self.key.public_numbers() - params = { - 'n': numbers.n, - 'e': numbers.e, - } - else: # rsa.RSAPrivateKey - private = self.key.private_numbers() - public = self.key.public_key().public_numbers() - params = { - 'n': public.n, - 'e': public.e, - 'd': private.d, - 'p': private.p, - 'q': private.q, - 'dp': private.dmp1, - 'dq': private.dmq1, - 'qi': private.iqmp, - } - return dict((key, self._encode_param(value)) - for key, value in six.iteritems(params)) diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py deleted file mode 100644 index eea5793bf..000000000 --- a/acme/acme/jose/jwk_test.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Tests for acme.jose.jwk.""" -import binascii -import unittest - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import util - - -DSA_PEM = test_util.load_vector('dsa512_key.pem') -RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') -RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') - - -class JWKTest(unittest.TestCase): - """Tests for acme.jose.jwk.JWK.""" - - def test_load(self): - from acme.jose.jwk import JWK - self.assertRaises(errors.Error, JWK.load, DSA_PEM) - - def test_load_subclass_wrong_type(self): - from acme.jose.jwk import JWKRSA - self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) - - -class JWKTestBaseMixin(object): - """Mixin test for JWK subclass tests.""" - - thumbprint = NotImplemented - - def test_thumbprint_private(self): - self.assertEqual(self.thumbprint, self.jwk.thumbprint()) - - def test_thumbprint_public(self): - self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) - - -class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKOct.""" - - thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>" - b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea" - b"\x8e(\x8a\xb2i\x1c") - - def setUp(self): - from acme.jose.jwk import JWKOct - self.jwk = JWKOct(key=b'foo') - self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')} - - def test_to_partial_json(self): - self.assertEqual(self.jwk.to_partial_json(), self.jobj) - - def test_from_json(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.from_json(self.jobj)) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWKOct - hash(JWKOct.from_json(self.jobj)) - - def test_load(self): - from acme.jose.jwk import JWKOct - self.assertEqual(self.jwk, JWKOct.load(b'foo')) - - def test_public_key(self): - self.assertTrue(self.jwk.public_key() is self.jwk) - - -class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): - """Tests for acme.jose.jwk.JWKRSA.""" - # pylint: disable=too-many-instance-attributes - - thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c' - b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y') - - def setUp(self): - from acme.jose.jwk import JWKRSA - self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) - self.jwk256json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk', - } - # pylint: disable=protected-access - self.jwk256_not_comparable = JWKRSA( - key=RSA256_KEY.public_key()._wrapped) - self.jwk512 = JWKRSA(key=RSA512_KEY.public_key()) - self.jwk512json = { - 'kty': 'RSA', - 'e': 'AQAB', - 'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5' - '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q', - } - self.private = JWKRSA(key=RSA256_KEY) - self.private_json_small = self.jwk256json.copy() - self.private_json_small['d'] = ( - 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE') - self.private_json = self.jwk256json.copy() - self.private_json.update({ - 'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE', - 'p': 'zUVNZn4lLLBD1R6NE8TKNQ', - 'q': 'wcfKfc7kl5jfqXArCRSURQ', - 'dp': 'CWJFq43QvT5Bm5iN8n1okQ', - 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', - 'qi': 'oi45cEkbVoJjAbnQpFY87Q', - }) - self.jwk = self.private - - def test_init_auto_comparable(self): - self.assertTrue(isinstance( - self.jwk256_not_comparable.key, util.ComparableRSAKey)) - self.assertEqual(self.jwk256, self.jwk256_not_comparable) - - def test_encode_param_zero(self): - from acme.jose.jwk import JWKRSA - # pylint: disable=protected-access - # TODO: move encode/decode _param to separate class - self.assertEqual('AA', JWKRSA._encode_param(0)) - - def test_equals(self): - self.assertEqual(self.jwk256, self.jwk256) - self.assertEqual(self.jwk512, self.jwk512) - - def test_not_equals(self): - self.assertNotEqual(self.jwk256, self.jwk512) - self.assertNotEqual(self.jwk512, self.jwk256) - - def test_load(self): - from acme.jose.jwk import JWKRSA - self.assertEqual(self.private, JWKRSA.load( - test_util.load_vector('rsa256_key.pem'))) - - def test_public_key(self): - self.assertEqual(self.jwk256, self.private.public_key()) - - def test_to_partial_json(self): - self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json) - self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json) - self.assertEqual(self.private.to_partial_json(), self.private_json) - - def test_from_json(self): - from acme.jose.jwk import JWK - self.assertEqual( - self.jwk256, JWK.from_json(self.jwk256json)) - self.assertEqual( - self.jwk512, JWK.from_json(self.jwk512json)) - self.assertEqual(self.private, JWK.from_json(self.private_json)) - - def test_from_json_private_small(self): - from acme.jose.jwk import JWK - self.assertEqual(self.private, JWK.from_json(self.private_json_small)) - - def test_from_json_missing_one_additional(self): - from acme.jose.jwk import JWK - del self.private_json['q'] - self.assertRaises(errors.Error, JWK.from_json, self.private_json) - - def test_from_json_hashable(self): - from acme.jose.jwk import JWK - hash(JWK.from_json(self.jwk256json)) - - def test_from_json_non_schema_errors(self): - # valid against schema, but still failing - from acme.jose.jwk import JWK - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': ''}) - self.assertRaises(errors.DeserializationError, JWK.from_json, - {'kty': 'RSA', 'e': 'AQAB', 'n': '1'}) - - def test_thumbprint_go_jose(self): - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344 - # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384 - from acme.jose.jwk import JWKRSA - key = JWKRSA.json_loads("""{ - "kty": "RSA", - "kid": "bilbo.baggins@hobbiton.example", - "use": "sig", - "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw", - "e": "AQAB" -}""") - self.assertEqual( - binascii.hexlify(key.thumbprint()), - b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932") - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py deleted file mode 100644 index 5f446e4b1..000000000 --- a/acme/acme/jose/jws.py +++ /dev/null @@ -1,433 +0,0 @@ -"""JOSE Web Signature.""" -import argparse -import base64 -import sys - -import OpenSSL -import six - -from acme.jose import b64 -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk -from acme.jose import util - - -class MediaType(object): - """MediaType field encoder/decoder.""" - - PREFIX = 'application/' - """MIME Media Type and Content Type prefix.""" - - @classmethod - def decode(cls, value): - """Decoder.""" - # 4.1.10 - if '/' not in value: - if ';' in value: - raise errors.DeserializationError('Unexpected semi-colon') - return cls.PREFIX + value - return value - - @classmethod - def encode(cls, value): - """Encoder.""" - # 4.1.10 - if ';' not in value: - assert value.startswith(cls.PREFIX) - return value[len(cls.PREFIX):] - return value - - -class Header(json_util.JSONObjectWithFields): - """JOSE Header. - - .. warning:: This class supports **only** Registered Header - Parameter Names (as defined in section 4.1 of the - protocol). If you need Public Header Parameter Names (4.2) - or Private Header Parameter Names (4.3), you must subclass - and override :meth:`from_json` and :meth:`to_partial_json` - appropriately. - - .. warning:: This class does not support any extensions through - the "crit" (Critical) Header Parameter (4.1.11) and as a - conforming implementation, :meth:`from_json` treats its - occurrence as an error. Please subclass if you seek for - a different behaviour. - - :ivar x5tS256: "x5t#S256" - :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`. - :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`. - - """ - alg = json_util.Field( - 'alg', decoder=jwa.JWASignature.from_json, omitempty=True) - jku = json_util.Field('jku', omitempty=True) - jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True) - kid = json_util.Field('kid', omitempty=True) - x5u = json_util.Field('x5u', omitempty=True) - x5c = json_util.Field('x5c', omitempty=True, default=()) - x5t = json_util.Field( - 'x5t', decoder=json_util.decode_b64jose, omitempty=True) - x5tS256 = json_util.Field( - 'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True) - typ = json_util.Field('typ', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - cty = json_util.Field('cty', encoder=MediaType.encode, - decoder=MediaType.decode, omitempty=True) - crit = json_util.Field('crit', omitempty=True, default=()) - - def not_omitted(self): - """Fields that would not be omitted in the JSON object.""" - return dict((name, getattr(self, name)) - for name, field in six.iteritems(self._fields) - if not field.omit(getattr(self, name))) - - def __add__(self, other): - if not isinstance(other, type(self)): - raise TypeError('Header cannot be added to: {0}'.format( - type(other))) - - not_omitted_self = self.not_omitted() - not_omitted_other = other.not_omitted() - - if set(not_omitted_self).intersection(not_omitted_other): - raise TypeError('Addition of overlapping headers not defined') - - not_omitted_self.update(not_omitted_other) - return type(self)(**not_omitted_self) # pylint: disable=star-args - - def find_key(self): - """Find key based on header. - - .. todo:: Supports only "jwk" header parameter lookup. - - :returns: (Public) key found in the header. - :rtype: .JWK - - :raises acme.jose.errors.Error: if key could not be found - - """ - if self.jwk is None: - raise errors.Error('No key found') - return self.jwk - - @crit.decoder - def crit(unused_value): - # pylint: disable=missing-docstring,no-self-argument,no-self-use - raise errors.DeserializationError( - '"crit" is not supported, please subclass') - - # x5c does NOT use JOSE Base64 (4.1.6) - - @x5c.encoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - return [base64.b64encode(OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] - - @x5c.decoder # type: ignore - def x5c(value): # pylint: disable=missing-docstring,no-self-argument - try: - return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - base64.b64decode(cert))) for cert in value) - except OpenSSL.crypto.Error as error: - raise errors.DeserializationError(error) - - -class Signature(json_util.JSONObjectWithFields): - """JWS Signature. - - :ivar combined: Combined Header (protected and unprotected, - :class:`Header`). - :ivar unicode protected: JWS protected header (Jose Base-64 decoded). - :ivar header: JWS Unprotected Header (:class:`Header`). - :ivar str signature: The signature. - - """ - header_cls = Header - - __slots__ = ('combined',) - protected = json_util.Field('protected', omitempty=True, default='') - header = json_util.Field( - 'header', omitempty=True, default=header_cls(), - decoder=header_cls.from_json) - signature = json_util.Field( - 'signature', decoder=json_util.decode_b64jose, - encoder=json_util.encode_b64jose) - - @protected.encoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - # wrong type guess (Signature, not bytes) | pylint: disable=no-member - return json_util.encode_b64jose(value.encode('utf-8')) - - @protected.decoder # type: ignore - def protected(value): # pylint: disable=missing-docstring,no-self-argument - return json_util.decode_b64jose(value).decode('utf-8') - - def __init__(self, **kwargs): - if 'combined' not in kwargs: - kwargs = self._with_combined(kwargs) - super(Signature, self).__init__(**kwargs) - assert self.combined.alg is not None - - @classmethod - def _with_combined(cls, kwargs): - assert 'combined' not in kwargs - header = kwargs.get('header', cls._fields['header'].default) - protected = kwargs.get('protected', cls._fields['protected'].default) - - if protected: - combined = header + cls.header_cls.json_loads(protected) - else: - combined = header - - kwargs['combined'] = combined - return kwargs - - @classmethod - def _msg(cls, protected, payload): - return (b64.b64encode(protected.encode('utf-8')) + b'.' + - b64.b64encode(payload)) - - def verify(self, payload, key=None): - """Verify. - - :param JWK key: Key used for verification. - - """ - key = self.combined.find_key() if key is None else key - return self.combined.alg.verify( - key=key.key, sig=self.signature, - msg=self._msg(self.protected, payload)) - - @classmethod - def sign(cls, payload, key, alg, include_jwk=True, - protect=frozenset(), **kwargs): - """Sign. - - :param JWK key: Key for signature. - - """ - assert isinstance(key, alg.kty) - - header_params = kwargs - header_params['alg'] = alg - if include_jwk: - header_params['jwk'] = key.public_key() - - assert set(header_params).issubset(cls.header_cls._fields) - assert protect.issubset(cls.header_cls._fields) - - protected_params = {} - for header in protect: - if header in header_params: - protected_params[header] = header_params.pop(header) - if protected_params: - # pylint: disable=star-args - protected = cls.header_cls(**protected_params).json_dumps() - else: - protected = '' - - header = cls.header_cls(**header_params) # pylint: disable=star-args - signature = alg.sign(key.key, cls._msg(protected, payload)) - - return cls(protected=protected, header=header, signature=signature) - - def fields_to_partial_json(self): - fields = super(Signature, self).fields_to_partial_json() - if not fields['header'].not_omitted(): - del fields['header'] - return fields - - @classmethod - def fields_from_json(cls, jobj): - fields = super(Signature, cls).fields_from_json(jobj) - fields_with_combined = cls._with_combined(fields) - if 'alg' not in fields_with_combined['combined'].not_omitted(): - raise errors.DeserializationError('alg not present') - return fields_with_combined - - -class JWS(json_util.JSONObjectWithFields): - """JSON Web Signature. - - :ivar str payload: JWS Payload. - :ivar str signature: JWS Signatures. - - """ - __slots__ = ('payload', 'signatures') - - signature_cls = Signature - - def verify(self, key=None): - """Verify.""" - return all(sig.verify(self.payload, key) for sig in self.signatures) - - @classmethod - def sign(cls, payload, **kwargs): - """Sign.""" - return cls(payload=payload, signatures=( - cls.signature_cls.sign(payload=payload, **kwargs),)) - - @property - def signature(self): - """Get a singleton signature. - - :rtype: `signature_cls` - - """ - assert len(self.signatures) == 1 - return self.signatures[0] - - def to_compact(self): - """Compact serialization. - - :rtype: bytes - - """ - assert len(self.signatures) == 1 - - assert 'alg' not in self.signature.header.not_omitted() - # ... it must be in protected - - return ( - b64.b64encode(self.signature.protected.encode('utf-8')) + - b'.' + - b64.b64encode(self.payload) + - b'.' + - b64.b64encode(self.signature.signature)) - - @classmethod - def from_compact(cls, compact): - """Compact deserialization. - - :param bytes compact: - - """ - try: - protected, payload, signature = compact.split(b'.') - except ValueError: - raise errors.DeserializationError( - 'Compact JWS serialization should comprise of exactly' - ' 3 dot-separated components') - - sig = cls.signature_cls( - protected=b64.b64decode(protected).decode('utf-8'), - signature=b64.b64decode(signature)) - return cls(payload=b64.b64decode(payload), signatures=(sig,)) - - def to_partial_json(self, flat=True): # pylint: disable=arguments-differ - assert self.signatures - payload = json_util.encode_b64jose(self.payload) - - if flat and len(self.signatures) == 1: - ret = self.signatures[0].to_partial_json() - ret['payload'] = payload - return ret - else: - return { - 'payload': payload, - 'signatures': self.signatures, - } - - @classmethod - def from_json(cls, jobj): - if 'signature' in jobj and 'signatures' in jobj: - raise errors.DeserializationError('Flat mixed with non-flat') - elif 'signature' in jobj: # flat - return cls(payload=json_util.decode_b64jose(jobj.pop('payload')), - signatures=(cls.signature_cls.from_json(jobj),)) - else: - return cls(payload=json_util.decode_b64jose(jobj['payload']), - signatures=tuple(cls.signature_cls.from_json(sig) - for sig in jobj['signatures'])) - - -class CLI(object): - """JWS CLI.""" - - @classmethod - def sign(cls, args): - """Sign.""" - key = args.alg.kty.load(args.key.read()) - args.key.close() - if args.protect is None: - args.protect = [] - if args.compact: - args.protect.append('alg') - - sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg, - protect=set(args.protect)) - - if args.compact: - six.print_(sig.to_compact().decode('utf-8')) - else: # JSON - six.print_(sig.json_dumps_pretty()) - - @classmethod - def verify(cls, args): - """Verify.""" - if args.compact: - sig = JWS.from_compact(sys.stdin.read().encode()) - else: # JSON - try: - sig = JWS.json_loads(sys.stdin.read()) - except errors.Error as error: - six.print_(error) - return -1 - - if args.key is not None: - assert args.kty is not None - key = args.kty.load(args.key.read()).public_key() - args.key.close() - else: - key = None - - sys.stdout.write(sig.payload) - return not sig.verify(key=key) - - @classmethod - def _alg_type(cls, arg): - return jwa.JWASignature.from_json(arg) - - @classmethod - def _header_type(cls, arg): - assert arg in Signature.header_cls._fields - return arg - - @classmethod - def _kty_type(cls, arg): - assert arg in jwk.JWK.TYPES - return jwk.JWK.TYPES[arg] - - @classmethod - def run(cls, args=sys.argv[1:]): - """Parse arguments and sign/verify.""" - parser = argparse.ArgumentParser() - parser.add_argument('--compact', action='store_true') - - subparsers = parser.add_subparsers() - parser_sign = subparsers.add_parser('sign') - parser_sign.set_defaults(func=cls.sign) - parser_sign.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=True) - parser_sign.add_argument( - '-a', '--alg', type=cls._alg_type, default=jwa.RS256) - parser_sign.add_argument( - '-p', '--protect', action='append', type=cls._header_type) - - parser_verify = subparsers.add_parser('verify') - parser_verify.set_defaults(func=cls.verify) - parser_verify.add_argument( - '-k', '--key', type=argparse.FileType('rb'), required=False) - parser_verify.add_argument( - '--kty', type=cls._kty_type, required=False) - - parsed = parser.parse_args(args) - return parsed.func(parsed) - - -if __name__ == '__main__': - exit(CLI.run()) # pragma: no cover diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py deleted file mode 100644 index ec91f6a1b..000000000 --- a/acme/acme/jose/jws_test.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Tests for acme.jose.jws.""" -import base64 -import unittest - -import mock -import OpenSSL - -from acme import test_util - -from acme.jose import errors -from acme.jose import json_util -from acme.jose import jwa -from acme.jose import jwk - - -CERT = test_util.load_comparable_cert('cert.pem') -KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) - - -class MediaTypeTest(unittest.TestCase): - """Tests for acme.jose.jws.MediaType.""" - - def test_decode(self): - from acme.jose.jws import MediaType - self.assertEqual('application/app', MediaType.decode('application/app')) - self.assertEqual('application/app', MediaType.decode('app')) - self.assertRaises( - errors.DeserializationError, MediaType.decode, 'app;foo') - - def test_encode(self): - from acme.jose.jws import MediaType - self.assertEqual('app', MediaType.encode('application/app')) - self.assertEqual('application/app;foo', - MediaType.encode('application/app;foo')) - - -class HeaderTest(unittest.TestCase): - """Tests for acme.jose.jws.Header.""" - - def setUp(self): - from acme.jose.jws import Header - self.header1 = Header(jwk='foo') - self.header2 = Header(jwk='bar') - self.crit = Header(crit=('a', 'b')) - self.empty = Header() - - def test_add_non_empty(self): - from acme.jose.jws import Header - self.assertEqual(Header(jwk='foo', crit=('a', 'b')), - self.header1 + self.crit) - - def test_add_empty(self): - self.assertEqual(self.header1, self.header1 + self.empty) - self.assertEqual(self.header1, self.empty + self.header1) - - def test_add_overlapping_error(self): - self.assertRaises(TypeError, self.header1.__add__, self.header2) - - def test_add_wrong_type_error(self): - self.assertRaises(TypeError, self.header1.__add__, 'xxx') - - def test_crit_decode_always_errors(self): - from acme.jose.jws import Header - self.assertRaises(errors.DeserializationError, Header.from_json, - {'crit': ['a', 'b']}) - - def test_x5c_decoding(self): - from acme.jose.jws import Header - header = Header(x5c=(CERT, CERT)) - jobj = header.to_partial_json() - cert_asn1 = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) - cert_b64 = base64.b64encode(cert_asn1) - self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) - self.assertEqual(header, Header.from_json(jobj)) - jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) - self.assertRaises(errors.DeserializationError, Header.from_json, jobj) - - def test_find_key(self): - self.assertEqual('foo', self.header1.find_key()) - self.assertEqual('bar', self.header2.find_key()) - self.assertRaises(errors.Error, self.crit.find_key) - - -class SignatureTest(unittest.TestCase): - """Tests for acme.jose.jws.Signature.""" - - def test_from_json(self): - from acme.jose.jws import Header - from acme.jose.jws import Signature - self.assertEqual( - Signature(signature=b'foo', header=Header(alg=jwa.RS256)), - Signature.from_json( - {'signature': 'Zm9v', 'header': {'alg': 'RS256'}})) - - def test_from_json_no_alg_error(self): - from acme.jose.jws import Signature - self.assertRaises(errors.DeserializationError, - Signature.from_json, {'signature': 'foo'}) - - -class JWSTest(unittest.TestCase): - """Tests for acme.jose.jws.JWS.""" - - def setUp(self): - self.privkey = KEY - self.pubkey = self.privkey.public_key() - - from acme.jose.jws import JWS - self.unprotected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256) - self.protected = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['jwk', 'alg'])) - self.mixed = JWS.sign( - payload=b'foo', key=self.privkey, alg=jwa.RS256, - protect=frozenset(['alg'])) - - def test_pubkey_jwk(self): - self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.protected.signature.combined.jwk, self.pubkey) - self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey) - - def test_sign_unprotected(self): - self.assertTrue(self.unprotected.verify()) - - def test_sign_protected(self): - self.assertTrue(self.protected.verify()) - - def test_sign_mixed(self): - self.assertTrue(self.mixed.verify()) - - def test_compact_lost_unprotected(self): - compact = self.mixed.to_compact() - self.assertEqual( - b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb' - b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA', - compact) - - from acme.jose.jws import JWS - mixed = JWS.from_compact(compact) - - self.assertNotEqual(self.mixed, mixed) - self.assertEqual( - set(['alg']), set(mixed.signature.combined.not_omitted())) - - def test_from_compact_missing_components(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.') - - def test_json_omitempty(self): - protected_jobj = self.protected.to_partial_json(flat=True) - unprotected_jobj = self.unprotected.to_partial_json(flat=True) - - self.assertTrue('protected' not in unprotected_jobj) - self.assertTrue('header' not in protected_jobj) - - unprotected_jobj['header'] = unprotected_jobj['header'].to_json() - - from acme.jose.jws import JWS - self.assertEqual(JWS.from_json(protected_jobj), self.protected) - self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected) - - def test_json_flat(self): - jobj_to = { - 'signature': json_util.encode_b64jose( - self.mixed.signature.signature), - 'payload': json_util.encode_b64jose(b'foo'), - 'header': self.mixed.signature.header, - 'protected': json_util.encode_b64jose( - self.mixed.signature.protected.encode('utf-8')), - } - jobj_from = jobj_to.copy() - jobj_from['header'] = jobj_from['header'].to_json() - - self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_json_not_flat(self): - jobj_to = { - 'signatures': (self.mixed.signature,), - 'payload': json_util.encode_b64jose(b'foo'), - } - jobj_from = jobj_to.copy() - jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()] - - self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to) - from acme.jose.jws import JWS - self.assertEqual(self.mixed, JWS.from_json(jobj_from)) - - def test_from_json_mixed_flat(self): - from acme.jose.jws import JWS - self.assertRaises(errors.DeserializationError, JWS.from_json, - {'signatures': (), 'signature': 'foo'}) - - def test_from_json_hashable(self): - from acme.jose.jws import JWS - hash(JWS.from_json(self.mixed.to_json())) - - -class CLITest(unittest.TestCase): - - def setUp(self): - self.key_path = test_util.vector_path('rsa512_key.pem') - - def test_unverified(self): - from acme.jose.jws import CLI - with mock.patch('sys.stdin') as sin: - sin.read.return_value = '{"payload": "foo", "signature": "xxx"}' - with mock.patch('sys.stdout'): - self.assertEqual(-1, CLI.run(['verify'])) - - def test_json(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['sign', '-k', self.key_path, '-a', 'RS256', - '-p', 'jwk']) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run(['verify'])) - - def test_compact(self): - from acme.jose.jws import CLI - - with mock.patch('sys.stdin') as sin: - sin.read.return_value = 'foo' - with mock.patch('sys.stdout') as sout: - CLI.run(['--compact', 'sign', '-k', self.key_path]) - sin.read.return_value = sout.write.mock_calls[0][1][0] - self.assertEqual(0, CLI.run([ - '--compact', 'verify', '--kty', 'RSA', - '-k', self.key_path])) - - -if __name__ == '__main__': - unittest.main() # pragma: no cover diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py deleted file mode 100644 index 26b7e0c5a..000000000 --- a/acme/acme/jose/util.py +++ /dev/null @@ -1,226 +0,0 @@ -"""JOSE utilities.""" -import collections - -from cryptography.hazmat.primitives.asymmetric import rsa -import OpenSSL -import six - - -class abstractclassmethod(classmethod): - # pylint: disable=invalid-name,too-few-public-methods - """Descriptor for an abstract classmethod. - - It augments the :mod:`abc` framework with an abstract - classmethod. This is implemented as :class:`abc.abstractclassmethod` - in the standard Python library starting with version 3.2. - - This particular implementation, allegedly based on Python 3.3 source - code, is stolen from - http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod. - - """ - __isabstractmethod__ = True - - def __init__(self, target): - target.__isabstractmethod__ = True - super(abstractclassmethod, self).__init__(target) - - -class ComparableX509(object): # pylint: disable=too-few-public-methods - """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. - - :ivar wrapped: Wrapped certificate or certificate request. - :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. - - """ - def __init__(self, wrapped): - assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( - wrapped, OpenSSL.crypto.X509Req) - self.wrapped = wrapped - - def __getattr__(self, name): - return getattr(self.wrapped, name) - - def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): - """Dumps the object into a buffer with the specified encoding. - - :param int filetype: The desired encoding. Should be one of - `OpenSSL.crypto.FILETYPE_ASN1`, - `OpenSSL.crypto.FILETYPE_PEM`, or - `OpenSSL.crypto.FILETYPE_TEXT`. - - :returns: Encoded X509 object. - :rtype: str - - """ - if isinstance(self.wrapped, OpenSSL.crypto.X509): - func = OpenSSL.crypto.dump_certificate - else: # assert in __init__ makes sure this is X509Req - func = OpenSSL.crypto.dump_certificate_request - return func(filetype, self.wrapped) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return NotImplemented - # pylint: disable=protected-access - return self._dump() == other._dump() - - def __hash__(self): - return hash((self.__class__, self._dump())) - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) - - -class ComparableKey(object): # pylint: disable=too-few-public-methods - """Comparable wrapper for `cryptography` keys. - - See https://github.com/pyca/cryptography/issues/2122. - - """ - __hash__ = NotImplemented - - def __init__(self, wrapped): - self._wrapped = wrapped - - def __getattr__(self, name): - return getattr(self._wrapped, name) - - def __eq__(self, other): - # pylint: disable=protected-access - if (not isinstance(other, self.__class__) or - self._wrapped.__class__ is not other._wrapped.__class__): - return NotImplemented - elif hasattr(self._wrapped, 'private_numbers'): - return self.private_numbers() == other.private_numbers() - elif hasattr(self._wrapped, 'public_numbers'): - return self.public_numbers() == other.public_numbers() - else: - return NotImplemented - - def __ne__(self, other): - return not self == other - - def __repr__(self): - return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) - - def public_key(self): - """Get wrapped public key.""" - return self.__class__(self._wrapped.public_key()) - - -class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods - """Wrapper for `cryptography` RSA keys. - - Wraps around: - - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey` - - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey` - - """ - - def __hash__(self): - # public_numbers() hasn't got stable hash! - # https://github.com/pyca/cryptography/issues/2143 - if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization): - priv = self.private_numbers() - pub = priv.public_numbers - return hash((self.__class__, priv.p, priv.q, priv.dmp1, - priv.dmq1, priv.iqmp, pub.n, pub.e)) - elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization): - pub = self.public_numbers() - return hash((self.__class__, pub.n, pub.e)) - - -class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=too-few-public-methods - """Immutable key to value mapping with attribute access.""" - - __slots__ = () - """Must be overridden in subclasses.""" - - def __init__(self, **kwargs): - if set(kwargs) != set(self.__slots__): - raise TypeError( - '__init__() takes exactly the following arguments: {0} ' - '({1} given)'.format(', '.join(self.__slots__), - ', '.join(kwargs) if kwargs else 'none')) - for slot in self.__slots__: - object.__setattr__(self, slot, kwargs.pop(slot)) - - def update(self, **kwargs): - """Return updated map.""" - items = dict(self) - items.update(kwargs) - return type(self)(**items) # pylint: disable=star-args - - def __getitem__(self, key): - try: - return getattr(self, key) - except AttributeError: - raise KeyError(key) - - def __iter__(self): - return iter(self.__slots__) - - def __len__(self): - return len(self.__slots__) - - def __hash__(self): - return hash(tuple(getattr(self, slot) for slot in self.__slots__)) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return '{0}({1})'.format(self.__class__.__name__, ', '.join( - '{0}={1!r}'.format(key, value) - for key, value in six.iteritems(self))) - - -class frozendict(collections.Mapping, collections.Hashable): # type: ignore - # pylint: disable=invalid-name,too-few-public-methods - """Frozen dictionary.""" - __slots__ = ('_items', '_keys') - - def __init__(self, *args, **kwargs): - if kwargs and not args: - items = dict(kwargs) - elif len(args) == 1 and isinstance(args[0], collections.Mapping): - items = args[0] - else: - raise TypeError() - # TODO: support generators/iterators - - object.__setattr__(self, '_items', items) - object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items)))) - - def __getitem__(self, key): - return self._items[key] - - def __iter__(self): - return iter(self._keys) - - def __len__(self): - return len(self._items) - - def _sorted_items(self): - return tuple((key, self[key]) for key in self._keys) - - def __hash__(self): - return hash(self._sorted_items()) - - def __getattr__(self, name): - try: - return self._items[name] - except KeyError: - raise AttributeError(name) - - def __setattr__(self, name, value): - raise AttributeError("can't set attribute") - - def __repr__(self): - return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format( - key, value) for key, value in self._sorted_items())) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py deleted file mode 100644 index 0038a6cc1..000000000 --- a/acme/acme/jose/util_test.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Tests for acme.jose.util.""" -import functools -import unittest - -import six - -from acme import test_util - - -class ComparableX509Test(unittest.TestCase): - """Tests for acme.jose.util.ComparableX509.""" - - def setUp(self): - # test_util.load_comparable_{csr,cert} return ComparableX509 - self.req1 = test_util.load_comparable_csr('csr.pem') - self.req2 = test_util.load_comparable_csr('csr.pem') - self.req_other = test_util.load_comparable_csr('csr-san.pem') - - self.cert1 = test_util.load_comparable_cert('cert.pem') - self.cert2 = test_util.load_comparable_cert('cert.pem') - self.cert_other = test_util.load_comparable_cert('cert-san.pem') - - def test_getattr_proxy(self): - self.assertTrue(self.cert1.has_expired()) - - def test_eq(self): - self.assertEqual(self.req1, self.req2) - self.assertEqual(self.cert1, self.cert2) - - def test_ne(self): - self.assertNotEqual(self.req1, self.req_other) - self.assertNotEqual(self.cert1, self.cert_other) - - def test_ne_wrong_types(self): - self.assertNotEqual(self.req1, 5) - self.assertNotEqual(self.cert1, 5) - - def test_hash(self): - self.assertEqual(hash(self.req1), hash(self.req2)) - self.assertNotEqual(hash(self.req1), hash(self.req_other)) - - self.assertEqual(hash(self.cert1), hash(self.cert2)) - self.assertNotEqual(hash(self.cert1), hash(self.cert_other)) - - def test_repr(self): - for x509 in self.req1, self.cert1: - self.assertEqual(repr(x509), - ''.format(x509.wrapped)) - - -class ComparableRSAKeyTest(unittest.TestCase): - """Tests for acme.jose.util.ComparableRSAKey.""" - - def setUp(self): - # test_utl.load_rsa_private_key return ComparableRSAKey - self.key = test_util.load_rsa_private_key('rsa256_key.pem') - self.key_same = test_util.load_rsa_private_key('rsa256_key.pem') - self.key2 = test_util.load_rsa_private_key('rsa512_key.pem') - - def test_getattr_proxy(self): - self.assertEqual(256, self.key.key_size) - - def test_eq(self): - self.assertEqual(self.key, self.key_same) - - def test_ne(self): - self.assertNotEqual(self.key, self.key2) - - def test_ne_different_types(self): - self.assertNotEqual(self.key, 5) - - def test_ne_not_wrapped(self): - # pylint: disable=protected-access - self.assertNotEqual(self.key, self.key_same._wrapped) - - def test_ne_no_serialization(self): - from acme.jose.util import ComparableRSAKey - self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5)) - - def test_hash(self): - self.assertTrue(isinstance(hash(self.key), int)) - self.assertEqual(hash(self.key), hash(self.key_same)) - self.assertNotEqual(hash(self.key), hash(self.key2)) - - def test_repr(self): - self.assertTrue(repr(self.key).startswith( - '=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', + # formerly known as acme.jose: + 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13', @@ -74,10 +76,5 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, - entry_points={ - 'console_scripts': [ - 'jws = acme.jose.jws:CLI.run', - ], - }, test_suite='acme', ) diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index b4a24f137..f03c9da87 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -93,4 +93,8 @@ def parse_define_file(filepath, varname): if v == "-D" and len(a_opts) >= i+2: var_parts = a_opts[i+1].partition("=") return_vars[var_parts[0]] = var_parts[2] + elif len(v) > 2 and v.startswith("-D"): + # Found var with no whitespace separator + var_parts = v[2:].partition("=") + return_vars[var_parts[0]] = var_parts[2] return return_vars diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index d4d4e96b9..92f1d4a20 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,6 +49,7 @@ class GentooParser(parser.ApacheParser): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() + self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -56,3 +57,10 @@ class GentooParser(parser.ApacheParser): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] + + def update_modules(self): + """Get loaded modules from httpd process, and add them to DOM""" + mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") + for mod in matches: + self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index 7ca47a4d5..d7a2a2fd9 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -118,6 +118,8 @@ class MultipleVhostsTestCentOS(util.ApacheTest): self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) self.assertTrue("mock_value" in self.config.parser.variables.keys()) self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) + self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) + self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index 0f2b96818..cfbaffac7 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,6 +2,8 @@ import os import unittest +import mock + from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -46,9 +48,10 @@ class MultipleVhostsTestGentoo(util.ApacheTest): config_root=config_root, vhost_root=vhost_root) - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -78,9 +81,47 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - self.config.parser.update_runtime_variables() + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) + @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") + def test_no_binary_configdump(self, mock_subprocess): + """Make sure we don't call binary dumps other than modules from Apache + as this is not supported in Gentoo currently""" + + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertFalse(mock_subprocess.called) + + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertTrue(mock_subprocess.called) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_opportunistic_httpd_runtime_parsing(self, mock_get): + mod_val = ( + 'Loaded Modules:\n' + ' mock_module (static)\n' + ' another_module (static)\n' + ) + def mock_get_cfg(command): + """Mock httpd process stdout""" + if command == ['apache2ctl', 'modules']: + return mod_val + mock_get.side_effect = mock_get_cfg + self.config.parser.modules = set() + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("gentoo", "123") + self.config.parser.update_runtime_variables() + + self.assertEquals(mock_get.call_count, 1) + self.assertEquals(len(self.config.parser.modules), 4) + self.assertTrue("mod_another.c" in self.config.parser.modules) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd index 0bf6b176c..4bcb300c2 100644 --- a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -14,7 +14,7 @@ # To pass additional options (for instance, -D definitions) to the # httpd binary at startup, set OPTIONS here. # -OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE -DMOCK_NOSEP -DNOSEP_TWO=NOSEP_VAL" # # This setting ensures the httpd process is started in the "C" locale diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 2405110c5..ca667465c 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -5,11 +5,10 @@ import sys import unittest import augeas +import josepy as jose import mock import zope.component -from acme import jose - from certbot.display import util as display_util from certbot.plugins import common diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 8dc283f2d..3270f2c79 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index 25f2ce889..444bee1b9 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.19.0" +LE_AUTO_VERSION="0.20.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1062,9 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1077,18 +1078,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index af951aa6a..4155944bd 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -6,7 +6,8 @@ import re import shutil import tarfile -from acme import jose +import josepy as jose + from acme import test_util from certbot import constants diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 166c383b3..1faf30643 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 6392e483c..428271045 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 304acf110..4a103193f 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py index 0fdacf4ad..3b8edce64 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -131,7 +131,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) - correct_record_mock.destroy.assert_called() + self.assertTrue(correct_record_mock.destroy.called) self.assertFalse(first_record_mock.destroy.call_args_list) self.assertFalse(last_record_mock.destroy.call_args_list) diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 489321435..23098d4b6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 67d68ee16..4ed5a06ca 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 88e02304e..8a0b88aab 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b40899e80..b00bd1ac3 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 1b72168e8..b8f50254e 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e9dc2b31d..2a388e487 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 79b523aed..78007afb5 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py index ff07b6ccd..d5f1b2816 100644 --- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py +++ b/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py @@ -31,7 +31,7 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest self.auth._change_txt_record.assert_called_once_with("UPSERT", '_acme-challenge.' + DOMAIN, mock.ANY) - self.auth._wait_for_change.assert_called_once() + self.assertEqual(self.auth._wait_for_change.call_count, 1) def test_perform_no_credentials_error(self): self.auth._change_txt_record = mock.MagicMock(side_effect=NoCredentialsError) @@ -183,7 +183,8 @@ class ClientTest(unittest.TestCase): self.client._change_txt_record("FOO", DOMAIN, "foo") - self.client.r53.change_resource_record_sets.assert_called_once() + call_count = self.client.r53.change_resource_record_sets.call_count + self.assertEqual(call_count, 1) def test_wait_for_change(self): self.client.r53.get_change = mock.MagicMock( @@ -192,7 +193,7 @@ class ClientTest(unittest.TestCase): self.client._wait_for_change(1) - self.client.r53.get_change.assert_called() + self.assertTrue(self.client.r53.get_change.called) if __name__ == "__main__": diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2a14e8ab1..7d1eb0bc9 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 98990664f..8af474c5e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -23,43 +23,24 @@ from certbot import util from certbot.plugins import common from certbot_nginx import constants -from certbot_nginx import tls_sni_01 +from certbot_nginx import nginxparser from certbot_nginx import parser +from certbot_nginx import tls_sni_01 logger = logging.getLogger(__name__) -REDIRECT_BLOCK = [[ - ['\n ', 'if', ' ', '($scheme', ' ', '!=', ' ', '"https")'], - [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], - '\n '] -], ['\n']] - -TEST_REDIRECT_BLOCK = [ - [ - ['if', '($scheme', '!=', '"https")'], - [ - ['return', '301', 'https://$host$request_uri'] - ] - ], - ['#', ' managed by Certbot'] +REDIRECT_BLOCK = [ + ['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], + ['\n'] ] REDIRECT_COMMENT_BLOCK = [ ['\n ', '#', ' Redirect non-https traffic to https'], - ['\n ', '#', ' if ($scheme != "https") {'], - ['\n ', '#', " return 301 https://$host$request_uri;"], - ['\n ', '#', " } # managed by Certbot"], + ['\n ', '#', ' return 301 https://$host$request_uri;'], ['\n'] ] -TEST_REDIRECT_COMMENT_BLOCK = [ - ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', " return 301 https://$host$request_uri;"], - ['#', " } # managed by Certbot"], -] - @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller) @zope.interface.provider(interfaces.IPluginFactory) class NginxConfigurator(common.Installer): @@ -194,16 +175,14 @@ class NginxConfigurator(common.Installer): "The nginx plugin currently requires --fullchain-path to " "install a cert.") - vhost = self.choose_vhost(domain, raise_if_no_match=False) - if vhost is None: - vhost = self._vhost_from_duplicated_default(domain) + vhost = self.choose_vhost(domain, create_if_no_match=True) cert_directives = [['\n ', 'ssl_certificate', ' ', fullchain_path], ['\n ', 'ssl_certificate_key', ' ', key_path]] self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, vhost.names) + vhost.filep, ", ".join(vhost.names)) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, @@ -214,7 +193,7 @@ class NginxConfigurator(common.Installer): ####################### # Vhost parsing methods ####################### - def choose_vhost(self, target_name, raise_if_no_match=True): + def choose_vhost(self, target_name, create_if_no_match=False): """Chooses a virtual host based on the given domain name. .. note:: This makes the vhost SSL-enabled if it isn't already. Follows @@ -228,8 +207,8 @@ class NginxConfigurator(common.Installer): hostname. Currently we just ignore this. :param str target_name: domain name - :param bool raise_if_no_match: True iff not finding a match is an error; - otherwise, return None + :param bool create_if_no_match: If we should create a new vhost from default + when there is no match found :returns: ssl vhost associated with name :rtype: :class:`~certbot_nginx.obj.VirtualHost` @@ -240,7 +219,9 @@ class NginxConfigurator(common.Installer): matches = self._get_ranked_matches(target_name) vhost = self._select_best_name_match(matches) if not vhost: - if raise_if_no_match: + if create_if_no_match: + vhost = self._vhost_from_duplicated_default(target_name) + else: # No matches. Raise a misconfiguration error. raise errors.MisconfigurationError( ("Cannot find a VirtualHost matching domain %s. " @@ -248,16 +229,12 @@ class NginxConfigurator(common.Installer): "please add a corresponding server_name directive to your " "nginx configuration: " "https://nginx.org/en/docs/http/server_names.html") % (target_name)) - else: - return None - else: - # Note: if we are enhancing with ocsp, vhost should already be ssl. - if not vhost.ssl: - self._make_server_ssl(vhost) + # Note: if we are enhancing with ocsp, vhost should already be ssl. + if not vhost.ssl: + self._make_server_ssl(vhost) return vhost - def ipv6_info(self, port): """Returns tuple of booleans (ipv6_active, ipv6only_present) ipv6_active is true if any server block listens ipv6 address in any port @@ -285,18 +262,19 @@ class NginxConfigurator(common.Installer): def _vhost_from_duplicated_default(self, domain): if self.new_vhost is None: default_vhost = self._get_default_vhost() - self.new_vhost = self.parser.create_new_vhost_from_default(default_vhost) - if not self.new_vhost.ssl: - self._make_server_ssl(self.new_vhost) + self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True) self.new_vhost.names = set() - self.new_vhost.names.add(domain) + self._add_server_name_to_vhost(self.new_vhost, domain) + return self.new_vhost + + def _add_server_name_to_vhost(self, vhost, domain): + vhost.names.add(domain) name_block = [['\n ', 'server_name']] - for name in self.new_vhost.names: + for name in vhost.names: name_block[0].append(' ') name_block[0].append(name) - self.parser.add_server_directives(self.new_vhost, name_block, replace=True) - return self.new_vhost + self.parser.add_server_directives(vhost, name_block, replace=True) def _get_default_vhost(self): vhost_list = self.parser.get_vhosts() @@ -505,11 +483,7 @@ class NginxConfigurator(common.Installer): def _make_server_ssl(self, vhost): """Make a server SSL. - Make a server SSL based on server_name and filename by adding a - ``listen IConfig.tls_sni_01_port ssl`` directive to the server block. - - .. todo:: Maybe this should create a new block instead of modifying - the existing one? + Make a server SSL by adding new listen and SSL directives. :param vhost: The vhost to add SSL to. :type vhost: :class:`~certbot_nginx.obj.VirtualHost` @@ -529,7 +503,9 @@ class NginxConfigurator(common.Installer): ipv6_block = ['\n ', 'listen', ' ', - '[::]:{0} ssl'.format(self.config.tls_sni_01_port)] + '[::]:{0}'.format(self.config.tls_sni_01_port), + ' ', + 'ssl'] if not ipv6info[1]: # ipv6only=on is absent in global config ipv6_block.append(' ') @@ -539,8 +515,9 @@ class NginxConfigurator(common.Installer): ipv4_block = ['\n ', 'listen', ' ', - '{0} ssl'.format(self.config.tls_sni_01_port)] - + '{0}'.format(self.config.tls_sni_01_port), + ' ', + 'ssl'] snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() @@ -584,10 +561,12 @@ class NginxConfigurator(common.Installer): raise def _has_certbot_redirect(self, vhost): - return vhost.contains_list(TEST_REDIRECT_BLOCK) + test_redirect_block = _test_block_from_block(REDIRECT_BLOCK) + return vhost.contains_list(test_redirect_block) def _has_certbot_redirect_comment(self, vhost): - return vhost.contains_list(TEST_REDIRECT_COMMENT_BLOCK) + test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK) + return vhost.contains_list(test_redirect_comment_block) def _add_redirect_block(self, vhost, active=True): """Add redirect directive to vhost @@ -603,7 +582,8 @@ class NginxConfigurator(common.Installer): def _enable_redirect(self, domain, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. - Add rewrite directive to non https traffic + If the vhost is listening plaintextishly, separate out the + relevant directives into a new server block and add a rewrite directive. .. note:: This function saves the configuration @@ -616,26 +596,46 @@ class NginxConfigurator(common.Installer): vhost = None # If there are blocks listening plaintextishly on self.DEFAULT_LISTEN_PORT, # choose the most name-matching one. + vhost = self.choose_redirect_vhost(domain, port) if vhost is None: logger.info("No matching insecure server blocks listening on port %s found.", self.DEFAULT_LISTEN_PORT) + return + + if vhost.ssl: + new_vhost = self.parser.duplicate_vhost(vhost, + only_directives=['listen', 'server_name']) + + def _ssl_match_func(directive): + return 'ssl' in directive + + def _no_ssl_match_func(directive): + return 'ssl' not in directive + + # remove all ssl addresses from the new block + self.parser.remove_server_directives(new_vhost, 'listen', match_func=_ssl_match_func) + + # remove all non-ssl addresses from the existing block + self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func) + + vhost = new_vhost + + if self._has_certbot_redirect(vhost): + logger.info("Traffic on port %s already redirecting to ssl in %s", + self.DEFAULT_LISTEN_PORT, vhost.filep) + elif vhost.has_redirect(): + if not self._has_certbot_redirect_comment(vhost): + self._add_redirect_block(vhost, active=False) + logger.info("The appropriate server block is already redirecting " + "traffic. To enable redirect anyway, uncomment the " + "redirect lines in %s.", vhost.filep) else: - if self._has_certbot_redirect(vhost): - logger.info("Traffic on port %s already redirecting to ssl in %s", - self.DEFAULT_LISTEN_PORT, vhost.filep) - elif vhost.has_redirect(): - if not self._has_certbot_redirect_comment(vhost): - self._add_redirect_block(vhost, active=False) - logger.info("The appropriate server block is already redirecting " - "traffic. To enable redirect anyway, uncomment the " - "redirect lines in %s.", vhost.filep) - else: - # Redirect plaintextish host to https - self._add_redirect_block(vhost, active=True) - logger.info("Redirecting all traffic on port %s to ssl in %s", - self.DEFAULT_LISTEN_PORT, vhost.filep) + # Redirect plaintextish host to https + self._add_redirect_block(vhost, active=True) + logger.info("Redirecting all traffic on port %s to ssl in %s", + self.DEFAULT_LISTEN_PORT, vhost.filep) def _enable_ocsp_stapling(self, domain, chain_path): """Include OCSP response in TLS handshake @@ -809,6 +809,7 @@ class NginxConfigurator(common.Installer): """ super(NginxConfigurator, self).recovery_routine() + self.new_vhost = None self.parser.load() def revert_challenge_config(self): @@ -818,6 +819,7 @@ class NginxConfigurator(common.Installer): """ self.revert_temporary_config() + self.new_vhost = None self.parser.load() def rollback_checkpoints(self, rollback=1): @@ -830,6 +832,7 @@ class NginxConfigurator(common.Installer): """ super(NginxConfigurator, self).rollback_checkpoints(rollback) + self.new_vhost = None self.parser.load() ########################################################################### @@ -882,6 +885,11 @@ class NginxConfigurator(common.Installer): self.restart() +def _test_block_from_block(block): + test_block = nginxparser.UnspacedList(block) + parser.comment_directive(test_block, 0) + return test_block[:-1] + def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 5816c5571..f5ac5c2e3 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -205,7 +205,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in six.moves.range(0, len(self.raw) - len(test)): + for i in six.moves.range(0, len(self.raw) - len(test) + 1): if self.raw[i:i + len(test)] == test: return True return False @@ -220,6 +220,8 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def ipv4_enabled(self): """Return true if one or more of the listen directives in vhost are IPv4 only""" + if self.addrs is None or len(self.addrs) == 0: + return True for a in self.addrs: if not a.ipv6: return True diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 3eb6264aa..9f13bc59f 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -1,5 +1,6 @@ """NginxParser is a member object of the NginxConfigurator class.""" import copy +import functools import glob import logging import os @@ -294,6 +295,30 @@ class NginxParser(object): :param bool replace: Whether to only replace existing directives """ + self._modify_server_directives(vhost, + functools.partial(_add_directives, directives, replace)) + + def remove_server_directives(self, vhost, directive_name, match_func=None): + """Remove all directives of type directive_name. + + :param :class:`~certbot_nginx.obj.VirtualHost` vhost: The vhost + to remove directives from + :param string directive_name: The directive type to remove + :param callable match_func: Function of the directive that returns true for directives + to be deleted. + """ + self._modify_server_directives(vhost, + functools.partial(_remove_directives, directive_name, match_func)) + + def _update_vhost_based_on_new_directives(self, vhost, directives_list): + new_server = self._get_included_directives(directives_list) + parsed_server = self.parse_server(new_server) + vhost.addrs = parsed_server['addrs'] + vhost.ssl = parsed_server['ssl'] + vhost.names = parsed_server['names'] + vhost.raw = new_server + + def _modify_server_directives(self, vhost, block_func): filename = vhost.filep try: result = self.parsed[filename] @@ -302,42 +327,52 @@ class NginxParser(object): if not isinstance(result, list) or len(result) != 2: raise errors.MisconfigurationError("Not a server block.") result = result[1] - _add_directives(result, directives, replace) + block_func(result) - # update vhost based on new directives - new_server = self._get_included_directives(result) - parsed_server = self.parse_server(new_server) - vhost.addrs = parsed_server['addrs'] - vhost.ssl = parsed_server['ssl'] - vhost.names = parsed_server['names'] - vhost.raw = new_server + self._update_vhost_based_on_new_directives(vhost, result) except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def create_new_vhost_from_default(self, vhost_template): - """Duplicate the default vhost in the configuration files. + def duplicate_vhost(self, vhost_template, delete_default=False, only_directives=None): + """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx.obj.VirtualHost` vhost_template: The vhost whose information we copy + :param bool delete_default: If we should remove default_server + from listen directives in the block. + :param list only_directives: If it exists, only duplicate the named directives. Only + looks at first level of depth; does not expand includes. :returns: A vhost object for the newly created vhost :rtype: :class:`~certbot_nginx.obj.VirtualHost` """ # TODO: https://github.com/certbot/certbot/issues/5185 # put it in the same file as the template, at the same level + new_vhost = copy.deepcopy(vhost_template) + enclosing_block = self.parsed[vhost_template.filep] for index in vhost_template.path[:-1]: enclosing_block = enclosing_block[index] - new_location = vhost_template.path[-1] + 1 raw_in_parsed = copy.deepcopy(enclosing_block[vhost_template.path[-1]]) - enclosing_block.insert(new_location, raw_in_parsed) - new_vhost = copy.deepcopy(vhost_template) - new_vhost.path[-1] = new_location - for addr in new_vhost.addrs: - addr.default = False - for directive in enclosing_block[new_vhost.path[-1]][1]: - if len(directive) > 0 and directive[0] == 'listen' and 'default_server' in directive: - del directive[directive.index('default_server')] + + if only_directives is not None: + new_directives = nginxparser.UnspacedList([]) + for directive in raw_in_parsed[1]: + if len(directive) > 0 and directive[0] in only_directives: + new_directives.append(directive) + raw_in_parsed[1] = new_directives + + self._update_vhost_based_on_new_directives(new_vhost, new_directives) + + enclosing_block.append(raw_in_parsed) + new_vhost.path[-1] = len(enclosing_block) - 1 + if delete_default: + for addr in new_vhost.addrs: + addr.default = False + for directive in enclosing_block[new_vhost.path[-1]][1]: + if (len(directive) > 0 and directive[0] == 'listen' + and 'default_server' in directive): + del directive[directive.index('default_server')] return new_vhost def _parse_ssl_options(ssl_options): @@ -486,7 +521,7 @@ def _is_ssl_on_directive(entry): len(entry) == 2 and entry[0] == 'ssl' and entry[1] == 'on') -def _add_directives(block, directives, replace): +def _add_directives(directives, replace, block): """Adds or replaces directives in a config block. When replace=False, it's an error to try and add a directive that already @@ -498,8 +533,9 @@ def _add_directives(block, directives, replace): ..todo :: Find directives that are in included files. - :param list block: The block to replace in :param list directives: The new directives. + :param bool replace: Described above. + :param list block: The block to replace in """ for directive in directives: @@ -513,8 +549,12 @@ REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] -def _comment_directive(block, location): - """Add a comment to the end of the line at location.""" +def comment_directive(block, location): + """Add a ``#managed by Certbot`` comment to the end of the line at location. + + :param list block: The block containing the directive to be commented + :param int location: The location within ``block`` of the directive to be commented + """ next_entry = block[location + 1] if location + 1 < len(block) else None if isinstance(next_entry, list) and next_entry: if len(next_entry) >= 2 and next_entry[-2] == "#" and COMMENT in next_entry[-1]: @@ -551,6 +591,12 @@ def _comment_out_directive(block, location, include_location): block[location] = new_dir[0] # set the now-single-line-comment directive back in place +def _find_location(block, directive_name, match_func=None): + """Finds the index of the first instance of directive_name in block. + If no line exists, use None.""" + return next((index for index, line in enumerate(block) \ + if line and line[0] == directive_name and (match_func is None or match_func(line))), None) + def _add_directive(block, directive, replace): """Adds or replaces a single directive in a config block. @@ -566,19 +612,12 @@ def _add_directive(block, directive, replace): block.append(directive) return - def find_location(direc): - """ Find the index of a config line where the name of the directive matches - the name of the directive we want to add. If no line exists, use None. - """ - return next((index for index, line in enumerate(block) \ - if line and line[0] == direc[0]), None) - - location = find_location(directive) + location = _find_location(block, directive[0]) if replace: if location is not None: block[location] = directive - _comment_directive(block, location) + comment_directive(block, location) return # Append directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value @@ -602,7 +641,7 @@ def _add_directive(block, directive, replace): included_directives = _parse_ssl_options(directive[1]) for included_directive in included_directives: - included_dir_loc = find_location(included_directive) + included_dir_loc = _find_location(block, included_directive[0]) included_dir_name = included_directive[0] if not is_whitespace_or_comment(included_directive) \ and not can_append(included_dir_loc, included_dir_name): @@ -614,10 +653,19 @@ def _add_directive(block, directive, replace): if can_append(location, directive_name): block.append(directive) - _comment_directive(block, len(block) - 1) + comment_directive(block, len(block) - 1) elif block[location] != directive: raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) +def _remove_directives(directive_name, match_func, block): + """Removes directives of name directive_name from a config block if match_func matches. + """ + while True: + location = _find_location(block, directive_name, match_func=match_func) + if location is None: + return + del block[location] + def _apply_global_addr_ssl(addr_to_ssl, parsed_server): """Apply global sslishness information to the parsed server block """ diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 996bd238b..e708b159a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -443,10 +443,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_redirect_enhance(self): # Test that we successfully add a redirect when there is # a listen directive - expected = [ - ['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ] + expected = ['return', '301', 'https://$host$request_uri'] example_conf = self.config.parser.abs_path('sites-enabled/example.com') self.config.enhance("www.example.com", "redirect") @@ -462,6 +459,35 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[migration_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + def test_split_for_redirect(self): + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + self.config.deploy_cert( + "example.org", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + self.config.enhance("www.example.com", "redirect") + generated_conf = self.config.parser.parsed[example_conf] + self.assertEqual( + [[['server'], [ + ['server_name', '.example.com'], + ['server_name', 'example.*'], [], + ['listen', '5001', 'ssl'], ['#', ' managed by Certbot'], + ['ssl_certificate', 'example/fullchain.pem'], ['#', ' managed by Certbot'], + ['ssl_certificate_key', 'example/key.pem'], ['#', ' managed by Certbot'], + ['include', self.config.mod_ssl_conf], ['#', ' managed by Certbot'], + ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'], + [], []]], + [['server'], [ + ['listen', '69.50.225.155:9000'], + ['listen', '127.0.0.1'], + ['server_name', '.example.com'], + ['server_name', 'example.*'], + ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], + [], []]]], + generated_conf) + @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list): @@ -494,9 +520,38 @@ class NginxConfiguratorTest(util.NginxTest): generated_conf = self.config.parser.parsed[example_conf] expected = [ ['#', ' Redirect non-https traffic to https'], - ['#', ' if ($scheme != "https") {'], - ['#', ' return 301 https://$host$request_uri;'], - ['#', ' } # managed by Certbot'] + ['#', ' return 301 https://$host$request_uri;'], + ] + for line in expected: + self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) + + @mock.patch('certbot_nginx.obj.VirtualHost.contains_list') + @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect') + def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list): + # Test that we add a redirect as a comment if there is already a + # redirect-class statement in the block that isn't managed by certbot + example_conf = self.config.parser.abs_path('sites-enabled/example.com') + + self.config.deploy_cert( + "example.org", + "example/cert.pem", + "example/key.pem", + "example/chain.pem", + "example/fullchain.pem") + + # Has a non-Certbot redirect, and has no existing comment + mock_contains_list.return_value = False + mock_has_redirect.return_value = True + with mock.patch("certbot_nginx.configurator.logger") as mock_logger: + self.config.enhance("www.example.com", "redirect") + self.assertEqual(mock_logger.info.call_args[0][0], + "The appropriate server block is already redirecting " + "traffic. To enable redirect anyway, uncomment the " + "redirect lines in %s.") + generated_conf = self.config.parser.parsed[example_conf] + expected = [ + ['#', ' Redirect non-https traffic to https'], + ['#', ' return 301 https://$host$request_uri;'], ] for line in expected: self.assertTrue(util.contains_at_depth(generated_conf, line, 2)) @@ -704,14 +759,18 @@ class NginxConfiguratorTest(util.NginxTest): self.config.parser.load() - expected = [ - ['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ] + expected = ['return', '301', 'https://$host$request_uri'] generated_conf = self.config.parser.parsed[default_conf] self.assertTrue(util.contains_at_depth(generated_conf, expected, 2)) + @mock.patch('certbot.reverter.logger') + @mock.patch('certbot_nginx.parser.NginxParser.load') + def test_parser_reload_after_config_changes(self, mock_parser_load, unused_mock_logger): + self.config.recovery_routine() + self.config.revert_challenge_config() + self.config.rollback_checkpoints() + self.assertTrue(mock_parser_load.call_count == 3) class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index ba136bb78..92cb0e086 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -171,8 +171,8 @@ class VirtualHostTest(unittest.TestCase): def test_contains_list(self): from certbot_nginx.obj import VirtualHost from certbot_nginx.obj import Addr - from certbot_nginx.configurator import TEST_REDIRECT_BLOCK - test_needle = TEST_REDIRECT_BLOCK + from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block + test_needle = _test_block_from_block(REDIRECT_BLOCK) test_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], @@ -181,9 +181,7 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme', '!=', '"https")'], - [['return', '301', 'https://$host$request_uri']] - ], + ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index ca5de7ff6..e21acb8ea 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -334,9 +334,9 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ["\n", "a", " ", "b", "\n"], ["c", " ", "d"], ["\n", "e", " ", "f"]]) - from certbot_nginx.parser import _comment_directive, COMMENT_BLOCK - _comment_directive(block, 1) - _comment_directive(block, 0) + from certbot_nginx.parser import comment_directive, COMMENT_BLOCK + comment_directive(block, 1) + comment_directive(block, 0) self.assertEqual(block.spaced, [ ["\n", "a", " ", "b", "\n"], COMMENT_BLOCK, @@ -406,12 +406,12 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods ]) self.assertTrue(server['ssl']) - def test_create_new_vhost_from_default(self): + def test_duplicate_vhost(self): nparser = parser.NginxParser(self.config_path) vhosts = nparser.get_vhosts() default = [x for x in vhosts if 'default' in x.filep][0] - new_vhost = nparser.create_new_vhost_from_default(default) + new_vhost = nparser.duplicate_vhost(default, delete_default=True) nparser.filedump(ext='') # check properties of new vhost diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 6e1b0d8ff..7b32d8e82 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -5,11 +5,10 @@ import pkg_resources import tempfile import unittest +import josepy as jose import mock import zope.component -from acme import jose - from certbot import configuration from certbot.tests import util as test_util diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 7f597ac4a..eca198bfe 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -55,7 +55,7 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.tls_sni_01_port) for achall in self.achalls: - vhost = self.configurator.choose_vhost(achall.domain, raise_if_no_match=False) + vhost = self.configurator.choose_vhost(achall.domain, create_if_no_match=True) if vhost is not None and vhost.addrs: addresses.append(list(vhost.addrs)) diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index f3919413d..2ad7aaf08 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.20.0.dev0' +version = '0.21.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 231a0f5f5..cbea701ee 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.20.0.dev0' +__version__ = '0.21.0.dev0' diff --git a/certbot/account.py b/certbot/account.py index 389f96791..41e980097 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,13 +7,13 @@ import shutil import socket from cryptography.hazmat.primitives import serialization +import josepy as jose import pyrfc3339 import pytz import six import zope.component from acme import fields as acme_fields -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/achallenges.py b/certbot/achallenges.py index f39bb4cec..6535a6b63 100644 --- a/certbot/achallenges.py +++ b/certbot/achallenges.py @@ -19,8 +19,9 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging +import josepy as jose + from acme import challenges -from acme import jose logger = logging.getLogger(__name__) diff --git a/certbot/client.py b/certbot/client.py index ed70fda71..b735421f5 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,13 +5,13 @@ import platform from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa +import josepy as jose import OpenSSL import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors -from acme import jose from acme import messages import certbot diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 112ef7c85..3ae16529d 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,9 +14,9 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 +import josepy as jose from acme import crypto_util as acme_crypto_util -from acme import jose from certbot import errors from certbot import interfaces @@ -368,7 +368,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in - `acme.jose.ComparableX509`). + :class:`josepy.util.ComparableX509`). """ # XXX: returns empty string when no chain is available, which diff --git a/certbot/main.py b/certbot/main.py index 9e2850891..1c6432fd9 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -6,9 +6,9 @@ import os import sys import configobj +import josepy as jose import zope.component -from acme import jose from acme import errors as acme_errors import certbot @@ -43,7 +43,15 @@ logger = logging.getLogger(__name__) def _suggest_donation_if_appropriate(config): - """Potentially suggest a donation to support Certbot.""" + """Potentially suggest a donation to support Certbot. + + :param config: Configuration object + :type config: interfaces.IConfig + + :returns: `None` + :rtype: None + + """ assert config.verb != "renew" if config.staging: # --dry-run implies --staging @@ -55,6 +63,15 @@ def _suggest_donation_if_appropriate(config): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) def _report_successful_dry_run(config): + """Reports on successful dry run + + :param config: Configuration object + :type config: interfaces.IConfig + + :returns: `None` + :rtype: None + + """ reporter_util = zope.component.getUtility(interfaces.IReporter) assert config.verb != "renew" reporter_util.add_message("The dry run was successful.", @@ -68,8 +85,23 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N then performs that action. Includes calls to hooks, various reports, checks, and requests for user input. + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: List of domain names to get a certificate. Defaults to `None` + :type domains: `list` of `str` + + :param certname: Name of new certificate. Defaults to `None` + :type certname: str + + :param lineage: Certificate lineage object. Defaults to `None` + :type lineage: storage.RenewableCert + :returns: the issued certificate or `None` if doing a dry run - :rtype: `storage.RenewableCert` or `None` + :rtype: storage.RenewableCert or None + + :raises errors.Error: if certificate could not be obtained + """ hooks.pre_hook(config) try: @@ -96,11 +128,18 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested - :param storage.RenewableCert cert: + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: List of domain names + :type domains: `list` of `str` + + :param cert: Certificate object + :type cert: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" - :rtype: tuple + :rtype: `tuple` of `str` """ existing = ", ".join(cert.names()) @@ -137,11 +176,15 @@ def _handle_subset_cert_request(config, domains, cert): def _handle_identical_cert_request(config, lineage): """Figure out what to do if a lineage has the same names as a previously obtained one - :param storage.RenewableCert lineage: + :param config: Configuration object + :type config: interfaces.IConfig + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert :returns: Tuple of (str action, cert_or_None) as per _find_lineage_for_domains_and_certname action can be: "newcert" | "renew" | "reinstall" - :rtype: tuple + :rtype: `tuple` of `str` """ if not lineage.ensure_deployed(): @@ -186,11 +229,18 @@ def _find_lineage_for_domains(config, domains): the client run if the user chooses to cancel the operation when prompted). + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: List of domain names + :type domains: `list` of `str` + :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either - a RenewableCert instance or None if renewal shouldn't occur. + a RenewableCert instance or `None` if renewal shouldn't occur. + :rtype: `tuple` of `str` and :class:`storage.RenewableCert` or `None` - :raises .Error: If the user would like to rerun the client again. + :raises errors.Error: If the user would like to rerun the client again. """ # Considering the possibility that the requested certificate is @@ -214,9 +264,20 @@ def _find_lineage_for_domains(config, domains): def _find_cert(config, domains, certname): """Finds an existing certificate object given domains and/or a certificate name. + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: List of domain names + :type domains: `list` of `str` + + :param certname: Name of certificate + :type certname: str + :returns: Two-element tuple of a boolean that indicates if this function should be followed by a call to fetch a certificate from the server, and either a RenewableCert instance or None. + :rtype: `tuple` of `bool` and :class:`storage.RenewableCert` or `None` + """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) if action == "reinstall": @@ -226,11 +287,22 @@ def _find_cert(config, domains, certname): def _find_lineage_for_domains_and_certname(config, domains, certname): """Find appropriate lineage based on given domains and/or certname. + :param config: Configuration object + :type config: interfaces.IConfig + + :param domains: List of domain names + :type domains: `list` of `str` + + :param certname: Name of certificate + :type certname: str + :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either - a RenewableCert instance or None if renewal shouldn't occur. + a RenewableCert instance or None if renewal should not occur. - :raises .Error: If the user would like to rerun the client again. + :rtype: `tuple` of `str` and :class:`storage.RenewableCert` or `None` + + :raises errors.Error: If the user would like to rerun the client again. """ if not certname: @@ -253,18 +325,57 @@ def _find_lineage_for_domains_and_certname(config, domains, certname): "Use -d to specify domains, or run certbot --certificates to see " "possible certificate names.".format(certname)) +def _get_added_removed(after, before): + """Get lists of items removed from `before` + and a lists of items added to `after` + """ + added = list(set(after) - set(before)) + removed = list(set(before) - set(after)) + added.sort() + removed.sort() + return added, removed + +def _format_list(character, strings): + """Format list with given character + """ + formatted = "{br}{ch} " + "{br}{ch} ".join(strings) + return formatted.format( + ch=character, + br=os.linesep + ) + def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): """Ask user to confirm update cert certname to contain new_domains. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param new_domains: List of new domain names + :type new_domains: `list` of `str` + + :param certname: Name of certificate + :type certname: str + + :param old_domains: List of old domain names + :type old_domains: `list` of `str` + + :returns: None + :rtype: None + + :raises errors.ConfigurationError: if cert name and domains mismatch + """ if config.renew_with_new_domains: return - msg = ("You are updating certificate {0} to include domains: {1}{br}{br}" - "It previously included domains: {2}{br}{br}" + added, removed = _get_added_removed(new_domains, old_domains) + + msg = ("You are updating certificate {0} to include new domain(s): {1}{br}{br}" + "You are also removing previously included domain(s): {2}{br}{br}" "Did you intend to make this change?".format( certname, - ", ".join(new_domains), - ", ".join(old_domains), + _format_list("+", added), + _format_list("-", removed), br=os.linesep)) obj = zope.component.getUtility(interfaces.IDisplay) if not obj.yesno(msg, "Update cert", "Cancel", default=True): @@ -272,6 +383,19 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): def _find_domains_or_certname(config, installer): """Retrieve domains and certname from config or user input. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param installer: Installer object + :type installer: interfaces.IInstaller + + + :returns: Two-part tuple of domains and certname + :rtype: `tuple` of list of `str` and `str` + + :raises errors.Error: Usage message, if parameters are not used correctly + """ domains = None certname = config.certname @@ -299,9 +423,17 @@ def _find_domains_or_certname(config, installer): def _report_new_cert(config, cert_path, fullchain_path, key_path=None): """Reports the creation of a new certificate to the user. - :param str cert_path: path to cert - :param str fullchain_path: path to full chain - :param str key_path: path to private key, if available + :param cert_path: path to certificate + :type cert_path: str + + :param fullchain_path: path to full chain + :type fullchain_path: str + + :param key_path: path to private key, if available + :type key_path: str + + :returns: `None` + :rtype: None """ if config.dry_run: @@ -337,14 +469,14 @@ def _determine_account(config): if ``config.account`` is ``None``, it will be updated based on the user input. Same for ``config.email``. - :param argparse.Namespace config: CLI arguments - :param certbot.interface.IConfig config: Configuration object - :param .AccountStorage account_storage: Account storage. + :param config: Configuration object + :type config: interfaces.IConfig :returns: Account and optionally ACME client API (biproduct of new registration). - :rtype: `tuple` of `certbot.account.Account` and - `acme.client.Client` + :rtype: tuple of :class:`certbot.account.Account` and :class:`acme.client.Client` + + :raises errors.Error: If unable to register an account with ACME server """ account_storage = account.AccountFileStorage(config) @@ -392,9 +524,13 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b deleting happens automatically, unless if both `--cert-name` and `--cert-path` were specified with conflicting values. - :param `configuration.NamespaceConfig` config: parsed command line arguments + :param config: parsed command line arguments + :type config: interfaces.IConfig - :raises `error.Errors`: If anything goes wrong, including bad user input, if an overlapping + :returns: `None` + :rtype: None + + :raises errors.Error: If anything goes wrong, including bad user input, if an overlapping archive dir is found for the specified lineage, etc ... """ display = zope.component.getUtility(interfaces.IDisplay) @@ -474,6 +610,20 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b def _init_le_client(config, authenticator, installer): + """Initialize Let's Encrypt Client + + :param config: Configuration object + :type config: interfaces.IConfig + + :param authenticator: Acme authentication handler + :type authenticator: interfaces.IAuthenticator + :param installer: Installer object + :type installer: interfaces.IInstaller + + :returns: client: Client object + :rtype: client.Client + + """ if authenticator is not None: # if authenticator was given, then we will need account... acc, acme = _determine_account(config) @@ -487,7 +637,18 @@ def _init_le_client(config, authenticator, installer): def unregister(config, unused_plugins): - """Deactivate account on server""" + """Deactivate account on server + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ account_storage = account.AccountFileStorage(config) accounts = account_storage.find_all() reporter_util = zope.component.getUtility(interfaces.IReporter) @@ -516,8 +677,18 @@ def unregister(config, unused_plugins): def register(config, unused_plugins): - """Create or modify accounts on the server.""" + """Create or modify accounts on the server. + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` or a string indicating and error + :rtype: None or str + + """ # Portion of _determine_account logic to see whether accounts already # exist or not. account_storage = account.AccountFileStorage(config) @@ -558,6 +729,24 @@ def register(config, unused_plugins): add_msg("Your e-mail address was updated to {0}.".format(config.email)) def _install_cert(config, le_client, domains, lineage=None): + """Install a cert + + :param config: Configuration object + :type config: interfaces.IConfig + + :param le_client: Client object + :type le_client: client.Client + + :param plugins: List of domains + :type plugins: `list` of `str` + + :param lineage: Certificate lineage object. Defaults to `None` + :type lineage: storage.RenewableCert + + :returns: `None` + :rtype: None + + """ path_provider = lineage if lineage else config assert path_provider.cert_path is not None @@ -566,7 +755,18 @@ def _install_cert(config, le_client, domains, lineage=None): le_client.enhance_config(domains, path_provider.chain_path) def install(config, plugins): - """Install a previously obtained cert in a server.""" + """Install a previously obtained cert in a server. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ # XXX: Update for renewer/RenewableCert # FIXME: be consistent about whether errors are raised or returned from # this function ... @@ -582,7 +782,18 @@ def install(config, plugins): def plugins_cmd(config, plugins): - """List server software plugins.""" + """List server software plugins. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ logger.debug("Expected interfaces: %s", config.ifaces) ifaces = [] if config.ifaces is None else config.ifaces @@ -610,7 +821,18 @@ def plugins_cmd(config, plugins): def rollback(config, plugins): - """Rollback server configuration changes made during install.""" + """Rollback server configuration changes made during install. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ client.rollback(config.installer, config.checkpoints, config, plugins) @@ -619,6 +841,15 @@ def config_changes(config, unused_plugins): View checkpoints and associated configuration changes. + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + """ client.view_config_changes(config, num=config.num) @@ -627,6 +858,16 @@ def update_symlinks(config, unused_plugins): Use the information in the config file to make symlinks point to the correct archive directory. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + """ cert_manager.update_live_symlinks(config) @@ -635,6 +876,16 @@ def rename(config, unused_plugins): Use the information in the config file to rename an existing lineage. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + """ cert_manager.rename_lineage(config) @@ -643,16 +894,47 @@ def delete(config, unused_plugins): Use the information in the config file to delete an existing lineage. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + """ cert_manager.delete(config) def certificates(config, unused_plugins): """Display information about certs configured with Certbot + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + """ cert_manager.certificates(config) def revoke(config, unused_plugins): # TODO: coop with renewal config - """Revoke a previously obtained certificate.""" + """Revoke a previously obtained certificate. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` or string indicating error in case of error + :rtype: None or str + + """ # For user-agent construction config.installer = config.authenticator = "None" if config.key_path is not None: # revocation by cert key @@ -678,7 +960,18 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals - """Obtain a certificate and install.""" + """Obtain a certificate and install. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ # TODO: Make run as close to auth + install as possible # Possible difficulties: config.csr was hacked into auth try: @@ -718,6 +1011,16 @@ def _csr_get_and_save_cert(config, le_client): This works differently in the CSR case (for now) because we don't have the privkey, and therefore can't construct the files for a lineage. So we just save the cert & chain to disk :/ + + :param config: Configuration object + :type config: interfaces.IConfig + + :param client: Client object + :type client: client.Client + + :returns: `cert_path` and `fullchain_path` as absolute paths to the actual files + :rtype: `tuple` of `str` + """ csr, _ = config.actual_csr certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) @@ -730,7 +1033,23 @@ def _csr_get_and_save_cert(config, le_client): return cert_path, fullchain_path def renew_cert(config, plugins, lineage): - """Renew & save an existing cert. Do not install it.""" + """Renew & save an existing cert. Do not install it. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :param lineage: Certificate lineage object + :type lineage: storage.RenewableCert + + :returns: `None` + :rtype: None + + :raises errors.PluginSelectionError: MissingCommandlineFlag if supplied parameters do not pass + + """ try: # installers are used in auth mode to determine domain names installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") @@ -757,8 +1076,20 @@ def renew_cert(config, plugins, lineage): def certonly(config, plugins): """Authenticate & obtain cert, but do not install it. - This implements the 'certonly' subcommand.""" + This implements the 'certonly' subcommand. + :param config: Configuration object + :type config: interfaces.IConfig + + :param plugins: List of plugins + :type plugins: `list` of `str` + + :returns: `None` + :rtype: None + + :raises errors.Error: If specified plugin could not be used + + """ # SETUP: Select plugins and construct a client instance try: # installers are used in auth mode to determine domain names @@ -792,7 +1123,18 @@ def certonly(config, plugins): _suggest_donation_if_appropriate(config) def renew(config, unused_plugins): - """Renew previously-obtained certificates.""" + """Renew previously-obtained certificates. + + :param config: Configuration object + :type config: interfaces.IConfig + + :param unused_plugins: List of plugins (deprecated) + :type unused_plugins: `list` of `str` + + :returns: `None` + :rtype: None + + """ try: renewal.handle_renewal_request(config) finally: @@ -800,7 +1142,15 @@ def renew(config, unused_plugins): def make_or_verify_needed_dirs(config): - """Create or verify existence of config, work, and hook directories.""" + """Create or verify existence of config, work, and hook directories. + + :param config: Configuration object + :type config: interfaces.IConfig + + :returns: `None` + :rtype: None + + """ util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, @@ -816,7 +1166,15 @@ def make_or_verify_needed_dirs(config): def set_displayer(config): - """Set the displayer""" + """Set the displayer + + :param config: Configuration object + :type config: interfaces.IConfig + + :returns: `None` + :rtype: None + + """ if config.quiet: config.noninteractive_mode = True displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) @@ -829,7 +1187,14 @@ def set_displayer(config): def main(cli_args=sys.argv[1:]): - """Command line argument parsing and main script execution.""" + """Command line argument parsing and main script execution. + + :returns: result of requested command + + :raises errors.Error: OS errors triggered by wrong permissions + :raises errors.Error: error if plugin command is not supported + + """ log.pre_arg_parse_setup() plugins = plugins_disco.PluginsRegistry.find_all() @@ -850,6 +1215,9 @@ def main(cli_args=sys.argv[1:]): # Let plugins_cmd be run as un-privileged user. if config.func != plugins_cmd: raise + if sys.version_info[:2] == (3, 3): + logger.warning("Python 3.3 support will be dropped in the next release " + "of Certbot - please upgrade your Python version.") set_displayer(config) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 420d15679..002d2f225 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -9,7 +9,7 @@ import OpenSSL import pkg_resources import zope.interface -from acme.jose import util as jose_util +from josepy import util as jose_util from certbot import constants from certbot import crypto_util diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 8ce68bbb5..1a1ca7dcb 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -5,11 +5,11 @@ import shutil import tempfile import unittest +import josepy as jose import mock import OpenSSL from acme import challenges -from acme import jose from certbot import achallenges from certbot import crypto_util diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py index d8cd29404..54b656b20 100644 --- a/certbot/plugins/dns_test_common.py +++ b/certbot/plugins/dns_test_common.py @@ -3,10 +3,10 @@ import os import configobj +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot.tests import acme_util diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py index f9c5735e8..a221cf1bf 100644 --- a/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" +import josepy as jose import mock -from acme import jose from requests.exceptions import HTTPError, RequestException from certbot import errors diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 1ae731e42..5227bc59e 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -3,11 +3,11 @@ import argparse import socket import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 92160bdfa..36e2ffba6 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -10,11 +10,11 @@ import stat import tempfile import unittest +import josepy as jose import mock import six from acme import challenges -from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7245ad6a1..8ebda56af 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -6,10 +6,10 @@ import shutil import stat import unittest +import josepy as jose import mock import pytz -from acme import jose from acme import messages from certbot import errors diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index f0549666a..53a2f214a 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -1,10 +1,10 @@ """ACME utilities for testing.""" import datetime +import josepy as jose import six from acme import challenges -from acme import jose from acme import messages from certbot import auth_handler diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 09c4a50ca..204f46323 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,11 +4,11 @@ import shutil import tempfile import unittest +import josepy as jose import OpenSSL import mock from acme import errors as acme_errors -from acme import jose from certbot import account from certbot import errors diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index cb0fb32e3..57d82f839 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,10 @@ import os import sys import unittest +import josepy as jose import mock import zope.component -from acme import jose from acme import messages from certbot import account diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 45e5db1df..04b71dcc7 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,11 +11,10 @@ import unittest import datetime import pytz +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import account from certbot import cli from certbot import constants @@ -356,7 +355,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_for_cert_name.return_value = "/some/reasonable/path" mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -375,7 +374,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_to_lineage.return_value = "example.com" mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -396,7 +395,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_full_archive_dir.return_value = "" mock_match_and_check_overlaps.return_value = "" self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -415,7 +414,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): mock_cert_path_to_lineage.return_value = config.certname mock_overlapping_archive_dirs.return_value = False self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.cert_manager.match_and_check_overlaps') @@ -442,7 +441,7 @@ class DeleteIfAppropriateTest(unittest.TestCase): util_mock = mock_get_utility() util_mock.menu.return_value = (display_util.OK, 0) self._call(config) - mock_delete.assert_called_once() + self.assertEqual(mock_delete.call_count, 1) # pylint: disable=too-many-arguments @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/certbot/tests/util.py b/certbot/tests/util.py index c43b44522..ddd4a1aec 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -14,11 +14,10 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import mock import OpenSSL +import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error -from acme import jose - from certbot import constants from certbot import interfaces from certbot import storage diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 6b43fd0a2..abaa95b9b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -107,7 +107,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.19.0 (certbot; + "". (default: CertbotACMEClient/0.20.0 (certbot; Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags encoded in the user agent are: --duplicate, --force- @@ -121,7 +121,7 @@ optional arguments: (Example: Foo-Wrapper/1.0) (default: None) automation: - Arguments for automating execution & other tweaks + Flags for automating execution & other tweaks --keep-until-expiring, --keep, --reinstall If the requested certificate matches an existing @@ -228,7 +228,7 @@ testing: False) paths: - Arguments changing execution paths & servers + Flags for changing execution paths & servers --cert-path CERT_PATH Path to where certificate is saved (with auth --csr), diff --git a/docs/using.rst b/docs/using.rst index 4fd0b5ec8..ab4670052 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -106,7 +106,7 @@ specified ``--webroot-path``. So, for instance, :: - certbot certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net + certbot certonly --webroot -w /var/www/example -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and diff --git a/letsencrypt-auto b/letsencrypt-auto index 25f2ce889..444bee1b9 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.19.0" +LE_AUTO_VERSION="0.20.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1062,9 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1077,18 +1078,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 834358464..eeab78cd6 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJZ1TJAAAoJEE0XyZXNl3XyWjAIAKxR5v0qbSyOEwM1LrSoLqud -V3KkyEUlMq7IPHxoPKXbqUrIi4eZuhpJz+84LtVJe4ZQ6HYP9lPogX+PtmWW7dyO -YerxA2rUVGB9rFZofZYwTuJyvO5Nc0aDyp1FHHPg/5khWWhhhxKpWqqG3zT01+Vf -W8Lvvn7vr7sjTvxBdqHQ3z3hlUY62P2IKui9C5un5ozlSQpDrWh3Thi9r6CxbASL -/r1PQ6EfnNdPAizVrJWe5iUd0Nzj7VMkFwZ02A3OlOUvrHGVb1H6oj0S1lZ8LEpj -awOTys8PVBQ3vW2qbAL3Zk7Lr+CGfVfmoWC9TQEKiSN1woYFrFD39S527vB1onc= -=Meks +iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt +XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf +p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX +OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4 +X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI +/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE= +=Is56 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 215b684cf..93e3e7b83 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.20.0.dev0" +LE_AUTO_VERSION="0.21.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -983,9 +983,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,9 +1069,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==2.0.0 \ - --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ - --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba # Contains the requirements for the letsencrypt package. # @@ -1077,18 +1081,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.19.0 \ - --hash=sha256:3207ee5319bfc37e855c25a43148275fcfb37869eefde9087405012049734a20 \ - --hash=sha256:a7230791dff5d085738119fc22d88ad9d8a35d0b6a3d67806fe33990c7c79d53 -acme==0.19.0 \ - --hash=sha256:c612eafe234d722d97bb5d3dbc49e5522f44be29611f7577954eb893e5c2d6de \ - --hash=sha256:1fa23d64d494aaf001e6fe857c461fcfff10f75a1c2c35ec831447f641e1e822 -certbot-apache==0.19.0 \ - --hash=sha256:fadb28b33bfabc85cdb962b5b149bef58b98f0606b78581db7895fe38323f37c \ - --hash=sha256:70306ca2d5be7f542af68d46883c0ae39527cf202f17ef92cd256fb0bc3f1619 -certbot-nginx==0.19.0 \ - --hash=sha256:4909cb3db49919fb35590793cac28e1c0b6dbd29cbedf887b9106e5fcef5362c \ - --hash=sha256:cb5a224a3f277092555c25096d1678fc735306fd3a43447649ebe524c7ca79e1 +certbot==0.20.0 \ + --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \ + --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88 +acme==0.20.0 \ + --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \ + --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5 +certbot-apache==0.20.0 \ + --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \ + --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f +certbot-nginx==0.20.0 \ + --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \ + --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 708bbbee6..e276aae53 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,2 +1,2 @@ -לHɭm+~ESM4 KYjL7ZճddI.:dZMO|Km|g($bljЅ/A^`ra^0x咁w9ckN.[ ? -h/́8!\!U59bRlb-1(p> -%u2gn \ No newline at end of file +HtPdM-b_8Gݵx\cf<9n$-^56zOy3o-N~ֹn% ww''}q;̰: + M4\@)-:ǺzDY=쟭*'=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT - 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', - 'six', 'zope.component', 'zope.interface', ] diff --git a/tools/deactivate.py b/tools/deactivate.py index 5facc8436..d43b84552 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -18,10 +18,10 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization +import josepy as jose from acme import client as acme_client from acme import errors as acme_errors -from acme import jose from acme import messages DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') diff --git a/tools/pip_constraints.txt b/tools/dev_constraints.txt similarity index 71% rename from tools/pip_constraints.txt rename to tools/dev_constraints.txt index cacec37d6..afc362ff8 100644 --- a/tools/pip_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,16 +1,15 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. We should avoid listing packages in -# both places because if both files are used as constraints for the same pip -# invocation, some constraints may be ignored due to pip's lack of dependency -# resolution. +# letsencrypt-auto's requirements file. alabaster==0.7.10 apipkg==1.4 +asn1crypto==0.22.0 astroid==1.3.5 +attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.8.1 +cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -19,7 +18,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.6.4 +google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -27,20 +26,22 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 +josepy==1.0.1 +logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -oauth2client==4.1.2 +ndg-httpsclient==0.3.2 +oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.3.7 -pyasn1-modules==0.1.5 +pyasn1==0.1.9 +pyasn1-modules==0.0.10 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -48,7 +49,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.12 +python-digitalocean==1.11 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -65,6 +66,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==3.0.0 +uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index d57f0974e..0d39e0594 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,8 +2,9 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.sh for more information on the versions pinned +# to. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" @@ -22,5 +23,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - pytest --numprocesses auto --quiet --pyargs $pkg + "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py new file mode 100755 index 000000000..c8fb95351 --- /dev/null +++ b/tools/merge_requirements.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""Merges multiple Python requirements files into one file. + +Requirements files specified later take precedence over earlier ones. Only +simple SomeProject==1.2.3 format is currently supported. + +""" + +from __future__ import print_function + +import sys + + +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: mapping from a project to its pinned version + :rtype: dict + + """ + d = {} + with open(file_path) as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + project, version = line.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(line)) + d[project] = version + return d + + +def print_requirements(requirements): + """Prints requirements to stdout. + + :param dict requirements: mapping from a project to its pinned version + + """ + print('\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items()))) + + +def merge_requirements_files(*files): + """Merges multiple requirements files together and prints the result. + + Requirement files specified later in the list take precedence over earlier + files. + + :param tuple files: paths to requirements files + + """ + d = {} + for f in files: + d.update(read_file(f)) + print_requirements(d) + + +if __name__ == '__main__': + merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt new file mode 100644 index 000000000..de2b83ad8 --- /dev/null +++ b/tools/oldest_constraints.txt @@ -0,0 +1,51 @@ +# This file contains the oldest versions of our dependencies we say we require +# in our packages or versions we need to support to maintain compatibility with +# the versions included in the various Linux distros where we are packaged. + +# CentOS/RHEL 7 EPEL constraints +cffi==1.6.0 +chardet==2.2.1 +configobj==4.7.2 +ipaddress==1.0.16 +mock==1.0.1 +ndg-httpsclient==0.3.2 +ply==3.4 +pyasn1==0.1.9 +pycparser==2.14 +pyOpenSSL==0.13.1 +pyparsing==1.5.6 +pyRFC3339==1.0 +python-augeas==0.5.0 +six==1.9.0 +# setuptools 0.9.8 is the actual version packaged, but some other dependencies +# in this file require setuptools>=1.0 and there are no relevant changes for us +# between these versions. +setuptools==1.0.0 +urllib3==1.10.2 +zope.component==4.1.0 +zope.event==4.0.3 +zope.interface==4.0.5 + +# Debian Jessie Backports constraints +PyICU==1.8 +colorama==0.3.2 +enum34==1.0.3 +html5lib==0.999 +idna==2.0 +pbr==1.8.0 +pytz==2012rc0 + +# Our setup.py constraints +cloudflare==1.5.1 +cryptography==1.2.0 +google-api-python-client==1.5 +oauth2client==2.0 +parsedatetime==1.3 +pyparsing==1.5.5 +python-digitalocean==1.11 +requests[security]==2.4.1 + +# Ubuntu Xenial constraints +ConfigArgParse==0.10.0 +funcsigs==0.4 +zope.hookable==4.0.4 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index fafd58e54..d2aae4a43 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,17 +1,26 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions +#!/bin/bash -e +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt and +# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's +# requirements file and tools/dev_constraints.txt is used. The other file +# always takes precedence over tools/dev_constraints.txt. # get the root of the Certbot repo -my_path=$("$(dirname $0)/readlink.py" $0) -repo_root=$(dirname $(dirname $my_path)) -requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" -certbot_auto_constraints=$(mktemp) -trap "rm -f $certbot_auto_constraints" EXIT -# extract pinned requirements without hashes -sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints -dev_constraints="$(dirname $my_path)/pip_constraints.txt" +tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) +dev_constraints="$tools_dir/dev_constraints.txt" +merge_reqs="$tools_dir/merge_requirements.py" +test_constraints=$(mktemp) +trap "rm -f $test_constraints" EXIT + +if [ "$CERTBOT_OLDEST" = 1 ]; then + cp "$tools_dir/oldest_constraints.txt" "$test_constraints" +else + repo_root=$(dirname "$tools_dir") + certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" + sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" +fi set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" +pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" diff --git a/tools/pytest.sh b/tools/pytest.sh new file mode 100755 index 000000000..8e3619d5d --- /dev/null +++ b/tools/pytest.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Runs pytest with the provided arguments, adding --numprocesses to the command +# line. This argument is set to "auto" if the environmnent variable TRAVIS is +# not set, otherwise, it is set to 2. This works around +# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis +# environnment provides two cores. See +# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. + +if ${TRAVIS:-false}; then + NUMPROCESSES="2" +else + NUMPROCESSES="auto" +fi + +pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index 3f0a5f72e..bc0e5a8bf 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -16,7 +16,7 @@ fi cover () { if [ "$1" = "certbot" ]; then - min=97 + min=98 elif [ "$1" = "acme" ]; then min=100 elif [ "$1" = "certbot_apache" ]; then @@ -24,23 +24,23 @@ cover () { elif [ "$1" = "certbot_dns_cloudflare" ]; then min=98 elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 elif [ "$1" = "certbot_dns_dnsimple" ]; then min=98 elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_dns_luadns" ]; then min=98 elif [ "$1" = "certbot_dns_nsone" ]; then - min=98 + min=99 elif [ "$1" = "certbot_dns_rfc2136" ]; then min=99 elif [ "$1" = "certbot_dns_route53" ]; then - min=91 + min=92 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then @@ -50,10 +50,13 @@ cover () { exit 1 fi - pytest --cov "$1" --cov-report term-missing \ - --cov-fail-under "$min" --numprocesses auto --pyargs "$1" + pkg_dir=$(echo "$1" | tr _ -) + pytest="$(dirname $0)/tools/pytest.sh" + "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" + coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } +rm -f .coverage # --cov-append is on, make sure stats are correct for pkg in $pkgs do cover $pkg diff --git a/tox.ini b/tox.ini index bb421daa5..20f5cda32 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,8 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# before the script moves on to the next package. All dependencies are pinned +# to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -62,6 +61,7 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 +passenv = TRAVIS [testenv] commands = @@ -70,48 +70,25 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 +passenv = + {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 +passenv = + {[testenv]passenv} [testenv:py27-oldest] commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_NO_PIN=1 -deps = - PyOpenSSL==0.13 - cffi==1.5.2 - configargparse==0.10.0 - configargparse==0.10.0 - configobj==4.7.2 - cryptography==1.2.3 - enum34==0.9.23 - google-api-python-client==1.5 - idna==2.0 - ipaddress==1.0.16 - mock==1.0.1 - ndg-httpsclient==0.3.2 - oauth2client==2.0 - parsedatetime==1.4 - pyasn1-modules==0.0.5 - pyasn1==0.1.9 - pyparsing==1.5.6 - pyrfc3339==1.0 - pytest==3.2.5 - python-augeas==0.4.1 - pytz==2012c - requests[security]==2.6.0 - setuptools==0.9.8 - six==1.9.0 - urllib3==1.10 - zope.component==4.0.2 - zope.event==4.0.1 - zope.interface==4.0.5 + CERTBOT_OLDEST=1 +passenv = + {[testenv]passenv} [testenv:py27_install] basepython = python2.7 @@ -123,6 +100,8 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh +passenv = + {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187)