diff --git a/.gitignore b/.gitignore index a01d2e1c7..b63e40d1c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,3 @@ tests/letstest/*.pem tests/letstest/venv/ .venv - -# pytest cache -.cache diff --git a/.travis.yml b/.travis.yml index 866f7b12a..b4b48ae71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_install: - cp .travis.yml /tmp/travis.yml - git pull origin master --strategy=recursive --strategy-option=theirs --no-edit - if ! git diff .travis.yml /tmp/travis.yml ; then echo "Please merge master into test-everything"; exit 1; fi - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' + - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)' before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 96997297b..14641af10 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -6,13 +6,13 @@ import logging import socket from cryptography.hazmat.primitives import hashes # type: ignore -import josepy as jose import OpenSSL import requests from acme import errors from acme import crypto_util from acme import fields +from acme import jose logger = logging.getLogger(__name__) diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 834d569aa..49e790102 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -1,7 +1,6 @@ """Tests for acme.challenges.""" import unittest -import josepy as jose import mock import OpenSSL import requests @@ -9,6 +8,7 @@ import requests from six.moves.urllib import parse as urllib_parse # pylint: disable=import-error from acme import errors +from acme import jose from acme import test_util CERT = test_util.load_comparable_cert('cert.pem') diff --git a/acme/acme/client.py b/acme/acme/client.py index dc5efbe86..2e07d34d7 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -10,13 +10,13 @@ import time import six from six.moves import http_client # pylint: disable=import-error -import josepy as jose import OpenSSL import re import requests import sys from acme import errors +from acme import jose from acme import jws from acme import messages @@ -408,7 +408,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param str uri: URI of certificate :returns: tuple of the form - (response, :class:`josepy.util.ComparableX509`) + (response, :class:`acme.jose.ComparableX509`) :rtype: tuple """ diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 84620fc99..4bd762865 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -5,12 +5,12 @@ import unittest from six.moves import http_client # pylint: disable=import-error -import josepy as jose import mock import requests from acme import challenges from acme import errors +from acme import jose from acme import jws as acme_jws from acme import messages from acme import messages_test diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 1d7f83ccf..da433c5a2 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,10 +8,10 @@ import unittest import six from six.moves import socketserver #type: ignore # pylint: disable=import-error -import josepy as jose import OpenSSL from acme import errors +from acme import jose from acme import test_util diff --git a/acme/acme/errors.py b/acme/acme/errors.py index de5f9d1f4..9d991fd75 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -1,5 +1,5 @@ """ACME errors.""" -from josepy import errors as jose_errors +from acme.jose import errors as jose_errors class Error(Exception): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index d7ec78403..12d09acf4 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -1,9 +1,10 @@ """ACME JSON fields.""" import logging -import josepy as jose import pyrfc3339 +from acme import jose + logger = logging.getLogger(__name__) diff --git a/acme/acme/fields_test.py b/acme/acme/fields_test.py index 69dde8b89..de852b6fa 100644 --- a/acme/acme/fields_test.py +++ b/acme/acme/fields_test.py @@ -2,9 +2,10 @@ import datetime import unittest -import josepy as jose import pytz +from acme import jose + class FixedTest(unittest.TestCase): """Tests for acme.fields.Fixed.""" diff --git a/acme/acme/jose/__init__.py b/acme/acme/jose/__init__.py new file mode 100644 index 000000000..9116bc433 --- /dev/null +++ b/acme/acme/jose/__init__.py @@ -0,0 +1,82 @@ +"""Javascript Object Signing and Encryption (jose). + +This package is a Python implementation of the standards developed by +IETF `Javascript Object Signing and Encryption (Active WG)`_, in +particular the following RFCs: + + - `JSON Web Algorithms (JWA)`_ + - `JSON Web Key (JWK)`_ + - `JSON Web Signature (JWS)`_ + + +.. _`Javascript Object Signing and Encryption (Active WG)`: + https://tools.ietf.org/wg/jose/ + +.. _`JSON Web Algorithms (JWA)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-algorithms/ + +.. _`JSON Web Key (JWK)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-key/ + +.. _`JSON Web Signature (JWS)`: + https://datatracker.ietf.org/doc/draft-ietf-jose-json-web-signature/ + +""" +from acme.jose.b64 import ( + b64decode, + b64encode, +) + +from acme.jose.errors import ( + DeserializationError, + SerializationError, + Error, + UnrecognizedTypeError, +) + +from acme.jose.interfaces import JSONDeSerializable + +from acme.jose.json_util import ( + Field, + JSONObjectWithFields, + TypedJSONObjectWithFields, + decode_b64jose, + decode_cert, + decode_csr, + decode_hex16, + encode_b64jose, + encode_cert, + encode_csr, + encode_hex16, +) + +from acme.jose.jwa import ( + HS256, + HS384, + HS512, + JWASignature, + PS256, + PS384, + PS512, + RS256, + RS384, + RS512, +) + +from acme.jose.jwk import ( + JWK, + JWKRSA, +) + +from acme.jose.jws import ( + Header, + JWS, + Signature, +) + +from acme.jose.util import ( + ComparableX509, + ComparableKey, + ComparableRSAKey, + ImmutableMap, +) diff --git a/acme/acme/jose/b64.py b/acme/acme/jose/b64.py new file mode 100644 index 000000000..cf79aa820 --- /dev/null +++ b/acme/acme/jose/b64.py @@ -0,0 +1,61 @@ +"""JOSE Base64. + +`JOSE Base64`_ is defined as: + + - URL-safe Base64 + - padding stripped + + +.. _`JOSE Base64`: + https://tools.ietf.org/html/draft-ietf-jose-json-web-signature-37#appendix-C + +.. Do NOT try to call this module "base64", as it will "shadow" the + standard library. + +""" +import base64 + +import six + + +def b64encode(data): + """JOSE Base64 encode. + + :param data: Data to be encoded. + :type data: `bytes` + + :returns: JOSE Base64 string. + :rtype: bytes + + :raises TypeError: if `data` is of incorrect type + + """ + if not isinstance(data, six.binary_type): + raise TypeError('argument should be {0}'.format(six.binary_type)) + return base64.urlsafe_b64encode(data).rstrip(b'=') + + +def b64decode(data): + """JOSE Base64 decode. + + :param data: Base64 string to be decoded. If it's unicode, then + only ASCII characters are allowed. + :type data: `bytes` or `unicode` + + :returns: Decoded data. + :rtype: bytes + + :raises TypeError: if input is of incorrect type + :raises ValueError: if input is unicode with non-ASCII characters + + """ + if isinstance(data, six.string_types): + try: + data = data.encode('ascii') + except UnicodeEncodeError: + raise ValueError( + 'unicode argument should contain only ASCII characters') + elif not isinstance(data, six.binary_type): + raise TypeError('argument should be a str or unicode') + + return base64.urlsafe_b64decode(data + b'=' * (4 - (len(data) % 4))) diff --git a/acme/acme/jose/b64_test.py b/acme/acme/jose/b64_test.py new file mode 100644 index 000000000..cbabe2251 --- /dev/null +++ b/acme/acme/jose/b64_test.py @@ -0,0 +1,77 @@ +"""Tests for acme.jose.b64.""" +import unittest + +import six + + +# https://en.wikipedia.org/wiki/Base64#Examples +B64_PADDING_EXAMPLES = { + b'any carnal pleasure.': (b'YW55IGNhcm5hbCBwbGVhc3VyZS4', b'='), + b'any carnal pleasure': (b'YW55IGNhcm5hbCBwbGVhc3VyZQ', b'=='), + b'any carnal pleasur': (b'YW55IGNhcm5hbCBwbGVhc3Vy', b''), + b'any carnal pleasu': (b'YW55IGNhcm5hbCBwbGVhc3U', b'='), + b'any carnal pleas': (b'YW55IGNhcm5hbCBwbGVhcw', b'=='), +} + + +B64_URL_UNSAFE_EXAMPLES = { + six.int2byte(251) + six.int2byte(239): b'--8', + six.int2byte(255) * 2: b'__8', +} + + +class B64EncodeTest(unittest.TestCase): + """Tests for acme.jose.b64.b64encode.""" + + @classmethod + def _call(cls, data): + from acme.jose.b64 import b64encode + return b64encode(data) + + def test_empty(self): + self.assertEqual(self._call(b''), b'') + + def test_unsafe_url(self): + for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): + self.assertEqual(self._call(text), b64) + + def test_different_paddings(self): + for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(text), b64) + + def test_unicode_fails_with_type_error(self): + self.assertRaises(TypeError, self._call, u'some unicode') + + +class B64DecodeTest(unittest.TestCase): + """Tests for acme.jose.b64.b64decode.""" + + @classmethod + def _call(cls, data): + from acme.jose.b64 import b64decode + return b64decode(data) + + def test_unsafe_url(self): + for text, b64 in six.iteritems(B64_URL_UNSAFE_EXAMPLES): + self.assertEqual(self._call(b64), text) + + def test_input_without_padding(self): + for text, (b64, _) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(b64), text) + + def test_input_with_padding(self): + for text, (b64, pad) in six.iteritems(B64_PADDING_EXAMPLES): + self.assertEqual(self._call(b64 + pad), text) + + def test_unicode_with_ascii(self): + self.assertEqual(self._call(u'YQ'), b'a') + + def test_non_ascii_unicode_fails(self): + self.assertRaises(ValueError, self._call, u'\u0105') + + def test_type_error_no_unicode_or_bytes(self): + self.assertRaises(TypeError, self._call, object()) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/errors.py b/acme/acme/jose/errors.py new file mode 100644 index 000000000..74c9443e1 --- /dev/null +++ b/acme/acme/jose/errors.py @@ -0,0 +1,35 @@ +"""JOSE errors.""" + + +class Error(Exception): + """Generic JOSE Error.""" + + +class DeserializationError(Error): + """JSON deserialization error.""" + + def __str__(self): + return "Deserialization error: {0}".format( + super(DeserializationError, self).__str__()) + + +class SerializationError(Error): + """JSON serialization error.""" + + +class UnrecognizedTypeError(DeserializationError): + """Unrecognized type error. + + :ivar str typ: The unrecognized type of the JSON object. + :ivar jobj: Full JSON object. + + """ + + def __init__(self, typ, jobj): + self.typ = typ + self.jobj = jobj + super(UnrecognizedTypeError, self).__init__(str(self)) + + def __str__(self): + return '{0} was not recognized, full message: {1}'.format( + self.typ, self.jobj) diff --git a/acme/acme/jose/errors_test.py b/acme/acme/jose/errors_test.py new file mode 100644 index 000000000..919980920 --- /dev/null +++ b/acme/acme/jose/errors_test.py @@ -0,0 +1,17 @@ +"""Tests for acme.jose.errors.""" +import unittest + + +class UnrecognizedTypeErrorTest(unittest.TestCase): + def setUp(self): + from acme.jose.errors import UnrecognizedTypeError + self.error = UnrecognizedTypeError('foo', {'type': 'foo'}) + + def test_str(self): + self.assertEqual( + "foo was not recognized, full message: {'type': 'foo'}", + str(self.error)) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/interfaces.py b/acme/acme/jose/interfaces.py new file mode 100644 index 000000000..f841848b3 --- /dev/null +++ b/acme/acme/jose/interfaces.py @@ -0,0 +1,216 @@ +"""JOSE interfaces.""" +import abc +import collections +import json + +import six + +from acme.jose import errors +from acme.jose import util + +# pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class +# pylint: disable=too-few-public-methods + + +@six.add_metaclass(abc.ABCMeta) +class JSONDeSerializable(object): + # pylint: disable=too-few-public-methods + """Interface for (de)serializable JSON objects. + + Please recall, that standard Python library implements + :class:`json.JSONEncoder` and :class:`json.JSONDecoder` that perform + translations based on respective :ref:`conversion tables + ` that look pretty much like the one below (for + complete tables see relevant Python documentation): + + .. _conversion-table: + + ====== ====== + JSON Python + ====== ====== + object dict + ... ... + ====== ====== + + While the above **conversion table** is about translation of JSON + documents to/from the basic Python types only, + :class:`JSONDeSerializable` introduces the following two concepts: + + serialization + Turning an arbitrary Python object into Python object that can + be encoded into a JSON document. **Full serialization** produces + a Python object composed of only basic types as required by the + :ref:`conversion table `. **Partial + serialization** (accomplished by :meth:`to_partial_json`) + produces a Python object that might also be built from other + :class:`JSONDeSerializable` objects. + + deserialization + Turning a decoded Python object (necessarily one of the basic + types as required by the :ref:`conversion table + `) into an arbitrary Python object. + + Serialization produces **serialized object** ("partially serialized + object" or "fully serialized object" for partial and full + serialization respectively) and deserialization produces + **deserialized object**, both usually denoted in the source code as + ``jobj``. + + Wording in the official Python documentation might be confusing + after reading the above, but in the light of those definitions, one + can view :meth:`json.JSONDecoder.decode` as decoder and + deserializer of basic types, :meth:`json.JSONEncoder.default` as + serializer of basic types, :meth:`json.JSONEncoder.encode` as + serializer and encoder of basic types. + + One could extend :mod:`json` to support arbitrary object + (de)serialization either by: + + - overriding :meth:`json.JSONDecoder.decode` and + :meth:`json.JSONEncoder.default` in subclasses + + - or passing ``object_hook`` argument (or ``object_hook_pairs``) + to :func:`json.load`/:func:`json.loads` or ``default`` argument + for :func:`json.dump`/:func:`json.dumps`. + + Interestingly, ``default`` is required to perform only partial + serialization, as :func:`json.dumps` applies ``default`` + recursively. This is the idea behind making :meth:`to_partial_json` + produce only partial serialization, while providing custom + :meth:`json_dumps` that dumps with ``default`` set to + :meth:`json_dump_default`. + + To make further documentation a bit more concrete, please, consider + the following imaginatory implementation example:: + + class Foo(JSONDeSerializable): + def to_partial_json(self): + return 'foo' + + @classmethod + def from_json(cls, jobj): + return Foo() + + class Bar(JSONDeSerializable): + def to_partial_json(self): + return [Foo(), Foo()] + + @classmethod + def from_json(cls, jobj): + return Bar() + + """ + + @abc.abstractmethod + def to_partial_json(self): # pragma: no cover + """Partially serialize. + + Following the example, **partial serialization** means the following:: + + assert isinstance(Bar().to_partial_json()[0], Foo) + assert isinstance(Bar().to_partial_json()[1], Foo) + + # in particular... + assert Bar().to_partial_json() != ['foo', 'foo'] + + :raises acme.jose.errors.SerializationError: + in case of any serialization error. + :returns: Partially serializable object. + + """ + raise NotImplementedError() + + def to_json(self): + """Fully serialize. + + Again, following the example from before, **full serialization** + means the following:: + + assert Bar().to_json() == ['foo', 'foo'] + + :raises acme.jose.errors.SerializationError: + in case of any serialization error. + :returns: Fully serialized object. + + """ + def _serialize(obj): + if isinstance(obj, JSONDeSerializable): + return _serialize(obj.to_partial_json()) + if isinstance(obj, six.string_types): # strings are Sequence + return obj + elif isinstance(obj, list): + return [_serialize(subobj) for subobj in obj] + elif isinstance(obj, collections.Sequence): + # default to tuple, otherwise Mapping could get + # unhashable list + return tuple(_serialize(subobj) for subobj in obj) + elif isinstance(obj, collections.Mapping): + return dict((_serialize(key), _serialize(value)) + for key, value in six.iteritems(obj)) + else: + return obj + + return _serialize(self) + + @util.abstractclassmethod + def from_json(cls, jobj): # pylint: disable=unused-argument + """Deserialize a decoded JSON document. + + :param jobj: Python object, composed of only other basic data + types, as decoded from JSON document. Not necessarily + :class:`dict` (as decoded from "JSON object" document). + + :raises acme.jose.errors.DeserializationError: + if decoding was unsuccessful, e.g. in case of unparseable + X509 certificate, or wrong padding in JOSE base64 encoded + string, etc. + + """ + # TypeError: Can't instantiate abstract class with + # abstract methods from_json, to_partial_json + return cls() # pylint: disable=abstract-class-instantiated + + @classmethod + def json_loads(cls, json_string): + """Deserialize from JSON document string.""" + try: + loads = json.loads(json_string) + except ValueError as error: + raise errors.DeserializationError(error) + return cls.from_json(loads) + + def json_dumps(self, **kwargs): + """Dump to JSON string using proper serializer. + + :returns: JSON document string. + :rtype: str + + """ + return json.dumps(self, default=self.json_dump_default, **kwargs) + + def json_dumps_pretty(self): + """Dump the object to pretty JSON document string. + + :rtype: str + + """ + return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': ')) + + @classmethod + def json_dump_default(cls, python_object): + """Serialize Python object. + + This function is meant to be passed as ``default`` to + :func:`json.dump` or :func:`json.dumps`. They call + ``default(python_object)`` only for non-basic Python types, so + this function necessarily raises :class:`TypeError` if + ``python_object`` is not an instance of + :class:`IJSONSerializable`. + + Please read the class docstring for more information. + + """ + if isinstance(python_object, JSONDeSerializable): + return python_object.to_partial_json() + else: # this branch is necessary, cannot just "return" + raise TypeError(repr(python_object) + ' is not JSON serializable') diff --git a/acme/acme/jose/interfaces_test.py b/acme/acme/jose/interfaces_test.py new file mode 100644 index 000000000..cf98ff371 --- /dev/null +++ b/acme/acme/jose/interfaces_test.py @@ -0,0 +1,114 @@ +"""Tests for acme.jose.interfaces.""" +import unittest + + +class JSONDeSerializableTest(unittest.TestCase): + # pylint: disable=too-many-instance-attributes + + def setUp(self): + from acme.jose.interfaces import JSONDeSerializable + + # pylint: disable=missing-docstring,invalid-name + + class Basic(JSONDeSerializable): + def __init__(self, v): + self.v = v + + def to_partial_json(self): + return self.v + + @classmethod + def from_json(cls, jobj): + return cls(jobj) + + class Sequence(JSONDeSerializable): + def __init__(self, x, y): + self.x = x + self.y = y + + def to_partial_json(self): + return [self.x, self.y] + + @classmethod + def from_json(cls, jobj): + return cls( + Basic.from_json(jobj[0]), Basic.from_json(jobj[1])) + + class Mapping(JSONDeSerializable): + def __init__(self, x, y): + self.x = x + self.y = y + + def to_partial_json(self): + return {self.x: self.y} + + @classmethod + def from_json(cls, jobj): + pass # pragma: no cover + + self.basic1 = Basic('foo1') + self.basic2 = Basic('foo2') + self.seq = Sequence(self.basic1, self.basic2) + self.mapping = Mapping(self.basic1, self.basic2) + self.nested = Basic([[self.basic1]]) + self.tuple = Basic(('foo',)) + + # pylint: disable=invalid-name + self.Basic = Basic + self.Sequence = Sequence + self.Mapping = Mapping + + def test_to_json_sequence(self): + self.assertEqual(self.seq.to_json(), ['foo1', 'foo2']) + + def test_to_json_mapping(self): + self.assertEqual(self.mapping.to_json(), {'foo1': 'foo2'}) + + def test_to_json_other(self): + mock_value = object() + self.assertTrue(self.Basic(mock_value).to_json() is mock_value) + + def test_to_json_nested(self): + self.assertEqual(self.nested.to_json(), [['foo1']]) + + def test_to_json(self): + self.assertEqual(self.tuple.to_json(), (('foo', ))) + + def test_from_json_not_implemented(self): + from acme.jose.interfaces import JSONDeSerializable + self.assertRaises(TypeError, JSONDeSerializable.from_json, 'xxx') + + def test_json_loads(self): + seq = self.Sequence.json_loads('["foo1", "foo2"]') + self.assertTrue(isinstance(seq, self.Sequence)) + self.assertTrue(isinstance(seq.x, self.Basic)) + self.assertTrue(isinstance(seq.y, self.Basic)) + self.assertEqual(seq.x.v, 'foo1') + self.assertEqual(seq.y.v, 'foo2') + + def test_json_dumps(self): + self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps()) + + def test_json_dumps_pretty(self): + self.assertEqual(self.seq.json_dumps_pretty(), + '[\n "foo1",\n "foo2"\n]') + + def test_json_dump_default(self): + from acme.jose.interfaces import JSONDeSerializable + + self.assertEqual( + 'foo1', JSONDeSerializable.json_dump_default(self.basic1)) + + jobj = JSONDeSerializable.json_dump_default(self.seq) + self.assertEqual(len(jobj), 2) + self.assertTrue(jobj[0] is self.basic1) + self.assertTrue(jobj[1] is self.basic2) + + def test_json_dump_default_type_error(self): + from acme.jose.interfaces import JSONDeSerializable + self.assertRaises( + TypeError, JSONDeSerializable.json_dump_default, object()) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/json_util.py b/acme/acme/jose/json_util.py new file mode 100644 index 000000000..4baadda5e --- /dev/null +++ b/acme/acme/jose/json_util.py @@ -0,0 +1,485 @@ +"""JSON (de)serialization framework. + +The framework presented here is somewhat based on `Go's "json" package`_ +(especially the ``omitempty`` functionality). + +.. _`Go's "json" package`: http://golang.org/pkg/encoding/json/ + +""" +import abc +import binascii +import logging + +import OpenSSL +import six + +from acme.jose import b64 +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import util + + +logger = logging.getLogger(__name__) + + +class Field(object): + """JSON object field. + + :class:`Field` is meant to be used together with + :class:`JSONObjectWithFields`. + + ``encoder`` (``decoder``) is a callable that accepts a single + parameter, i.e. a value to be encoded (decoded), and returns the + serialized (deserialized) value. In case of errors it should raise + :class:`~acme.jose.errors.SerializationError` + (:class:`~acme.jose.errors.DeserializationError`). + + Note, that ``decoder`` should perform partial serialization only. + + :ivar str json_name: Name of the field when encoded to JSON. + :ivar default: Default value (used when not present in JSON object). + :ivar bool omitempty: If ``True`` and the field value is empty, then + it will not be included in the serialized JSON object, and + ``default`` will be used for deserialization. Otherwise, if ``False``, + field is considered as required, value will always be included in the + serialized JSON objected, and it must also be present when + deserializing. + + """ + __slots__ = ('json_name', 'default', 'omitempty', 'fdec', 'fenc') + + def __init__(self, json_name, default=None, omitempty=False, + decoder=None, encoder=None): + # pylint: disable=too-many-arguments + self.json_name = json_name + self.default = default + self.omitempty = omitempty + + self.fdec = self.default_decoder if decoder is None else decoder + self.fenc = self.default_encoder if encoder is None else encoder + + @classmethod + def _empty(cls, value): + """Is the provided value considered "empty" for this field? + + This is useful for subclasses that might want to override the + definition of being empty, e.g. for some more exotic data types. + + """ + return not isinstance(value, bool) and not value + + def omit(self, value): + """Omit the value in output?""" + return self._empty(value) and self.omitempty + + def _update_params(self, **kwargs): + current = dict(json_name=self.json_name, default=self.default, + omitempty=self.omitempty, + decoder=self.fdec, encoder=self.fenc) + current.update(kwargs) + return type(self)(**current) # pylint: disable=star-args + + def decoder(self, fdec): + """Descriptor to change the decoder on JSON object field.""" + return self._update_params(decoder=fdec) + + def encoder(self, fenc): + """Descriptor to change the encoder on JSON object field.""" + return self._update_params(encoder=fenc) + + def decode(self, value): + """Decode a value, optionally with context JSON object.""" + return self.fdec(value) + + def encode(self, value): + """Encode a value, optionally with context JSON object.""" + return self.fenc(value) + + @classmethod + def default_decoder(cls, value): + """Default decoder. + + Recursively deserialize into immutable types ( + :class:`acme.jose.util.frozendict` instead of + :func:`dict`, :func:`tuple` instead of :func:`list`). + + """ + # bases cases for different types returned by json.loads + if isinstance(value, list): + return tuple(cls.default_decoder(subvalue) for subvalue in value) + elif isinstance(value, dict): + return util.frozendict( + dict((cls.default_decoder(key), cls.default_decoder(value)) + for key, value in six.iteritems(value))) + else: # integer or string + return value + + @classmethod + def default_encoder(cls, value): + """Default (passthrough) encoder.""" + # field.to_partial_json() is no good as encoder has to do partial + # serialization only + return value + + +class JSONObjectWithFieldsMeta(abc.ABCMeta): + """Metaclass for :class:`JSONObjectWithFields` and its subclasses. + + It makes sure that, for any class ``cls`` with ``__metaclass__`` + set to ``JSONObjectWithFieldsMeta``: + + 1. All fields (attributes of type :class:`Field`) in the class + definition are moved to the ``cls._fields`` dictionary, where + keys are field attribute names and values are fields themselves. + + 2. ``cls.__slots__`` is extended by all field attribute names + (i.e. not :attr:`Field.json_name`). Original ``cls.__slots__`` + are stored in ``cls._orig_slots``. + + In a consequence, for a field attribute name ``some_field``, + ``cls.some_field`` will be a slot descriptor and not an instance + of :class:`Field`. For example:: + + some_field = Field('someField', default=()) + + class Foo(object): + __metaclass__ = JSONObjectWithFieldsMeta + __slots__ = ('baz',) + some_field = some_field + + assert Foo.__slots__ == ('some_field', 'baz') + assert Foo._orig_slots == () + assert Foo.some_field is not Field + + assert Foo._fields.keys() == ['some_field'] + assert Foo._fields['some_field'] is some_field + + As an implementation note, this metaclass inherits from + :class:`abc.ABCMeta` (and not the usual :class:`type`) to mitigate + the metaclass conflict (:class:`ImmutableMap` and + :class:`JSONDeSerializable`, parents of :class:`JSONObjectWithFields`, + use :class:`abc.ABCMeta` as its metaclass). + + """ + + def __new__(mcs, name, bases, dikt): + fields = {} + + for base in bases: + fields.update(getattr(base, '_fields', {})) + # Do not reorder, this class might override fields from base classes! + for key, value in tuple(six.iteritems(dikt)): + # not six.iterkeys() (in-place edit!) + if isinstance(value, Field): + fields[key] = dikt.pop(key) + + dikt['_orig_slots'] = dikt.get('__slots__', ()) + dikt['__slots__'] = tuple( + list(dikt['_orig_slots']) + list(six.iterkeys(fields))) + dikt['_fields'] = fields + + return abc.ABCMeta.__new__(mcs, name, bases, dikt) + + +@six.add_metaclass(JSONObjectWithFieldsMeta) +class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable): + # pylint: disable=too-few-public-methods + """JSON object with fields. + + Example:: + + class Foo(JSONObjectWithFields): + bar = Field('Bar') + empty = Field('Empty', omitempty=True) + + @bar.encoder + def bar(value): + return value + 'bar' + + @bar.decoder + def bar(value): + if not value.endswith('bar'): + raise errors.DeserializationError('No bar suffix!') + return value[:-3] + + assert Foo(bar='baz').to_partial_json() == {'Bar': 'bazbar'} + assert Foo.from_json({'Bar': 'bazbar'}) == Foo(bar='baz') + assert (Foo.from_json({'Bar': 'bazbar', 'Empty': '!'}) + == Foo(bar='baz', empty='!')) + assert Foo(bar='baz').bar == 'baz' + + """ + + @classmethod + def _defaults(cls): + """Get default fields values.""" + return dict([(slot, field.default) for slot, field + in six.iteritems(cls._fields)]) + + def __init__(self, **kwargs): + # pylint: disable=star-args + super(JSONObjectWithFields, self).__init__( + **(dict(self._defaults(), **kwargs))) + + def encode(self, name): + """Encode a single field. + + :param str name: Name of the field to be encoded. + + :raises errors.SerializationError: if field cannot be serialized + :raises errors.Error: if field could not be found + + """ + try: + field = self._fields[name] + except KeyError: + raise errors.Error("Field not found: {0}".format(name)) + + return field.encode(getattr(self, name)) + + def fields_to_partial_json(self): + """Serialize fields to JSON.""" + jobj = {} + omitted = set() + for slot, field in six.iteritems(self._fields): + value = getattr(self, slot) + + if field.omit(value): + omitted.add((slot, value)) + else: + try: + jobj[field.json_name] = field.encode(value) + except errors.SerializationError as error: + raise errors.SerializationError( + 'Could not encode {0} ({1}): {2}'.format( + slot, value, error)) + return jobj + + def to_partial_json(self): + return self.fields_to_partial_json() + + @classmethod + def _check_required(cls, jobj): + missing = set() + for _, field in six.iteritems(cls._fields): + if not field.omitempty and field.json_name not in jobj: + missing.add(field.json_name) + + if missing: + raise errors.DeserializationError( + 'The following fields are required: {0}'.format( + ','.join(missing))) + + @classmethod + def fields_from_json(cls, jobj): + """Deserialize fields from JSON.""" + cls._check_required(jobj) + fields = {} + for slot, field in six.iteritems(cls._fields): + if field.json_name not in jobj and field.omitempty: + fields[slot] = field.default + else: + value = jobj[field.json_name] + try: + fields[slot] = field.decode(value) + except errors.DeserializationError as error: + raise errors.DeserializationError( + 'Could not decode {0!r} ({1!r}): {2}'.format( + slot, value, error)) + return fields + + @classmethod + def from_json(cls, jobj): + return cls(**cls.fields_from_json(jobj)) + + +def encode_b64jose(data): + """Encode JOSE Base-64 field. + + :param bytes data: + :rtype: `unicode` + + """ + # b64encode produces ASCII characters only + return b64.b64encode(data).decode('ascii') + + +def decode_b64jose(data, size=None, minimum=False): + """Decode JOSE Base-64 field. + + :param unicode data: + :param int size: Required length (after decoding). + :param bool minimum: If ``True``, then `size` will be treated as + minimum required length, as opposed to exact equality. + + :rtype: bytes + + """ + error_cls = TypeError if six.PY2 else binascii.Error + try: + decoded = b64.b64decode(data.encode()) + except error_cls as error: + raise errors.DeserializationError(error) + + if size is not None and ((not minimum and len(decoded) != size) or + (minimum and len(decoded) < size)): + raise errors.DeserializationError( + "Expected at least or exactly {0} bytes".format(size)) + + return decoded + + +def encode_hex16(value): + """Hexlify. + + :param bytes value: + :rtype: unicode + + """ + return binascii.hexlify(value).decode() + + +def decode_hex16(value, size=None, minimum=False): + """Decode hexlified field. + + :param unicode value: + :param int size: Required length (after decoding). + :param bool minimum: If ``True``, then `size` will be treated as + minimum required length, as opposed to exact equality. + + :rtype: bytes + + """ + value = value.encode() + if size is not None and ((not minimum and len(value) != size * 2) or + (minimum and len(value) < size * 2)): + raise errors.DeserializationError() + error_cls = TypeError if six.PY2 else binascii.Error + try: + return binascii.unhexlify(value) + except error_cls as error: + raise errors.DeserializationError(error) + + +def encode_cert(cert): + """Encode certificate as JOSE Base-64 DER. + + :type cert: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` + :rtype: unicode + + """ + return encode_b64jose(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) + + +def decode_cert(b64der): + """Decode JOSE Base-64 DER-encoded certificate. + + :param unicode b64der: + :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509` + + """ + try: + return util.ComparableX509(OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +def encode_csr(csr): + """Encode CSR as JOSE Base-64 DER. + + :type csr: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` + :rtype: unicode + + """ + return encode_b64jose(OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_ASN1, csr.wrapped)) + + +def decode_csr(b64der): + """Decode JOSE Base-64 DER-encoded CSR. + + :param unicode b64der: + :rtype: `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509` + + """ + try: + return util.ComparableX509(OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_ASN1, decode_b64jose(b64der))) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +class TypedJSONObjectWithFields(JSONObjectWithFields): + """JSON object with type.""" + + typ = NotImplemented + """Type of the object. Subclasses must override.""" + + type_field_name = "type" + """Field name used to distinguish different object types. + + Subclasses will probably have to override this. + + """ + + TYPES = NotImplemented + """Types registered for JSON deserialization""" + + @classmethod + def register(cls, type_cls, typ=None): + """Register class for JSON deserialization.""" + typ = type_cls.typ if typ is None else typ + cls.TYPES[typ] = type_cls + return type_cls + + @classmethod + def get_type_cls(cls, jobj): + """Get the registered class for ``jobj``.""" + if cls in six.itervalues(cls.TYPES): + if cls.type_field_name not in jobj: + raise errors.DeserializationError( + "Missing type field ({0})".format(cls.type_field_name)) + # cls is already registered type_cls, force to use it + # so that, e.g Revocation.from_json(jobj) fails if + # jobj["type"] != "revocation". + return cls + + if not isinstance(jobj, dict): + raise errors.DeserializationError( + "{0} is not a dictionary object".format(jobj)) + try: + typ = jobj[cls.type_field_name] + except KeyError: + raise errors.DeserializationError("missing type field") + + try: + return cls.TYPES[typ] + except KeyError: + raise errors.UnrecognizedTypeError(typ, jobj) + + def to_partial_json(self): + """Get JSON serializable object. + + :returns: Serializable JSON object representing ACME typed object. + :meth:`validate` will almost certainly not work, due to reasons + explained in :class:`acme.interfaces.IJSONSerializable`. + :rtype: dict + + """ + jobj = self.fields_to_partial_json() + jobj[self.type_field_name] = self.typ + return jobj + + @classmethod + def from_json(cls, jobj): + """Deserialize ACME object from valid JSON object. + + :raises acme.errors.UnrecognizedTypeError: if type + of the ACME object has not been registered. + + """ + # make sure subclasses don't cause infinite recursive from_json calls + type_cls = cls.get_type_cls(jobj) + return type_cls(**type_cls.fields_from_json(jobj)) diff --git a/acme/acme/jose/json_util_test.py b/acme/acme/jose/json_util_test.py new file mode 100644 index 000000000..25e36211e --- /dev/null +++ b/acme/acme/jose/json_util_test.py @@ -0,0 +1,381 @@ +"""Tests for acme.jose.json_util.""" +import itertools +import unittest + +import mock +import six + +from acme import test_util + +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import util + + +CERT = test_util.load_comparable_cert('cert.pem') +CSR = test_util.load_comparable_csr('csr.pem') + + +class FieldTest(unittest.TestCase): + """Tests for acme.jose.json_util.Field.""" + + def test_no_omit_boolean(self): + from acme.jose.json_util import Field + for default, omitempty, value in itertools.product( + [True, False], [True, False], [True, False]): + self.assertFalse( + Field("foo", default=default, omitempty=omitempty).omit(value)) + + def test_descriptors(self): + mock_value = mock.MagicMock() + + # pylint: disable=missing-docstring + + def decoder(unused_value): + return 'd' + + def encoder(unused_value): + return 'e' + + from acme.jose.json_util import Field + field = Field('foo') + + field = field.encoder(encoder) + self.assertEqual('e', field.encode(mock_value)) + + field = field.decoder(decoder) + self.assertEqual('e', field.encode(mock_value)) + self.assertEqual('d', field.decode(mock_value)) + + def test_default_encoder_is_partial(self): + class MockField(interfaces.JSONDeSerializable): + # pylint: disable=missing-docstring + def to_partial_json(self): + return 'foo' # pragma: no cover + + @classmethod + def from_json(cls, jobj): + pass # pragma: no cover + mock_field = MockField() + + from acme.jose.json_util import Field + self.assertTrue(Field.default_encoder(mock_field) is mock_field) + # in particular... + self.assertNotEqual('foo', Field.default_encoder(mock_field)) + + def test_default_encoder_passthrough(self): + mock_value = mock.MagicMock() + from acme.jose.json_util import Field + self.assertTrue(Field.default_encoder(mock_value) is mock_value) + + def test_default_decoder_list_to_tuple(self): + from acme.jose.json_util import Field + self.assertEqual((1, 2, 3), Field.default_decoder([1, 2, 3])) + + def test_default_decoder_dict_to_frozendict(self): + from acme.jose.json_util import Field + obj = Field.default_decoder({'x': 2}) + self.assertTrue(isinstance(obj, util.frozendict)) + self.assertEqual(obj, util.frozendict(x=2)) + + def test_default_decoder_passthrough(self): + mock_value = mock.MagicMock() + from acme.jose.json_util import Field + self.assertTrue(Field.default_decoder(mock_value) is mock_value) + + +class JSONObjectWithFieldsMetaTest(unittest.TestCase): + """Tests for acme.jose.json_util.JSONObjectWithFieldsMeta.""" + + def setUp(self): + from acme.jose.json_util import Field + from acme.jose.json_util import JSONObjectWithFieldsMeta + self.field = Field('Baz') + self.field2 = Field('Baz2') + # pylint: disable=invalid-name,missing-docstring,too-few-public-methods + # pylint: disable=blacklisted-name + + @six.add_metaclass(JSONObjectWithFieldsMeta) + class A(object): + __slots__ = ('bar',) + baz = self.field + + class B(A): + pass + + class C(A): + baz = self.field2 + + self.a_cls = A + self.b_cls = B + self.c_cls = C + + def test_fields(self): + # pylint: disable=protected-access,no-member + self.assertEqual({'baz': self.field}, self.a_cls._fields) + self.assertEqual({'baz': self.field}, self.b_cls._fields) + + def test_fields_inheritance(self): + # pylint: disable=protected-access,no-member + self.assertEqual({'baz': self.field2}, self.c_cls._fields) + + def test_slots(self): + self.assertEqual(('bar', 'baz'), self.a_cls.__slots__) + self.assertEqual(('baz',), self.b_cls.__slots__) + + def test_orig_slots(self): + # pylint: disable=protected-access,no-member + self.assertEqual(('bar',), self.a_cls._orig_slots) + self.assertEqual((), self.b_cls._orig_slots) + + +class JSONObjectWithFieldsTest(unittest.TestCase): + """Tests for acme.jose.json_util.JSONObjectWithFields.""" + # pylint: disable=protected-access + + def setUp(self): + from acme.jose.json_util import JSONObjectWithFields + from acme.jose.json_util import Field + + class MockJSONObjectWithFields(JSONObjectWithFields): + # pylint: disable=invalid-name,missing-docstring,no-self-argument + # pylint: disable=too-few-public-methods + x = Field('x', omitempty=True, + encoder=(lambda x: x * 2), + decoder=(lambda x: x / 2)) + y = Field('y') + z = Field('Z') # on purpose uppercase + + @y.encoder + def y(value): + if value == 500: + raise errors.SerializationError() + return value + + @y.decoder + def y(value): + if value == 500: + raise errors.DeserializationError() + return value + + # pylint: disable=invalid-name + self.MockJSONObjectWithFields = MockJSONObjectWithFields + self.mock = MockJSONObjectWithFields(x=None, y=2, z=3) + + def test_init_defaults(self): + self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3)) + + def test_encode(self): + self.assertEqual(10, self.MockJSONObjectWithFields( + x=5, y=0, z=0).encode("x")) + + def test_encode_wrong_field(self): + self.assertRaises(errors.Error, self.mock.encode, 'foo') + + def test_encode_serialization_error_passthrough(self): + self.assertRaises( + errors.SerializationError, + self.MockJSONObjectWithFields(y=500, z=None).encode, "y") + + def test_fields_to_partial_json_omits_empty(self): + self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3}) + + def test_fields_from_json_fills_default_for_empty(self): + self.assertEqual( + {'x': None, 'y': 2, 'z': 3}, + self.MockJSONObjectWithFields.fields_from_json({'y': 2, 'Z': 3})) + + def test_fields_from_json_fails_on_missing(self): + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'y': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'Z': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'y': 0}) + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.fields_from_json, {'x': 0, 'Z': 0}) + + def test_fields_to_partial_json_encoder(self): + self.assertEqual( + self.MockJSONObjectWithFields(x=1, y=2, z=3).to_partial_json(), + {'x': 2, 'y': 2, 'Z': 3}) + + def test_fields_from_json_decoder(self): + self.assertEqual( + {'x': 2, 'y': 2, 'z': 3}, + self.MockJSONObjectWithFields.fields_from_json( + {'x': 4, 'y': 2, 'Z': 3})) + + def test_fields_to_partial_json_error_passthrough(self): + self.assertRaises( + errors.SerializationError, self.MockJSONObjectWithFields( + x=1, y=500, z=3).to_partial_json) + + def test_fields_from_json_error_passthrough(self): + self.assertRaises( + errors.DeserializationError, + self.MockJSONObjectWithFields.from_json, + {'x': 4, 'y': 500, 'Z': 3}) + + +class DeEncodersTest(unittest.TestCase): + def setUp(self): + self.b64_cert = ( + u'MIIB3jCCAYigAwIBAgICBTkwDQYJKoZIhvcNAQELBQAwdzELMAkGA1UEBhM' + u'CVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRIwEAYDVQQHDAlBbm4gQXJib3IxKz' + u'ApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTWljaGlnYW4gYW5kIHRoZSBFRkYxF' + u'DASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTE0MTIxMTIyMzQ0NVoXDTE0MTIx' + u'ODIyMzQ0NVowdzELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2FuMRI' + u'wEAYDVQQHDAlBbm4gQXJib3IxKzApBgNVBAoMIlVuaXZlcnNpdHkgb2YgTW' + u'ljaGlnYW4gYW5kIHRoZSBFRkYxFDASBgNVBAMMC2V4YW1wbGUuY29tMFwwD' + u'QYJKoZIhvcNAQEBBQADSwAwSAJBAKx1c7RR7R_drnBSQ_zfx1vQLHUbFLh1' + u'AQQQ5R8DZUXd36efNK79vukFhN9HFoHZiUvOjm0c-pVE6K-EdE_twuUCAwE' + u'AATANBgkqhkiG9w0BAQsFAANBAC24z0IdwIVKSlntksllvr6zJepBH5fMnd' + u'fk3XJp10jT6VE-14KNtjh02a56GoraAvJAT5_H67E8GvJ_ocNnB_o' + ) + self.b64_csr = ( + u'MIIBXTCCAQcCAQAweTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE1pY2hpZ2F' + u'uMRIwEAYDVQQHDAlBbm4gQXJib3IxDDAKBgNVBAoMA0VGRjEfMB0GA1UECw' + u'wWVW5pdmVyc2l0eSBvZiBNaWNoaWdhbjEUMBIGA1UEAwwLZXhhbXBsZS5jb' + u'20wXDANBgkqhkiG9w0BAQEFAANLADBIAkEArHVztFHtH92ucFJD_N_HW9As' + u'dRsUuHUBBBDlHwNlRd3fp580rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3' + u'C5QIDAQABoCkwJwYJKoZIhvcNAQkOMRowGDAWBgNVHREEDzANggtleGFtcG' + u'xlLmNvbTANBgkqhkiG9w0BAQsFAANBAHJH_O6BtC9aGzEVCMGOZ7z9iIRHW' + u'Szr9x_bOzn7hLwsbXPAgO1QxEwL-X-4g20Gn9XBE1N9W6HCIEut2d8wACg' + ) + + def test_encode_b64jose(self): + from acme.jose.json_util import encode_b64jose + encoded = encode_b64jose(b'x') + self.assertTrue(isinstance(encoded, six.string_types)) + self.assertEqual(u'eA', encoded) + + def test_decode_b64jose(self): + from acme.jose.json_util import decode_b64jose + decoded = decode_b64jose(u'eA') + self.assertTrue(isinstance(decoded, six.binary_type)) + self.assertEqual(b'x', decoded) + + def test_decode_b64jose_padding_error(self): + from acme.jose.json_util import decode_b64jose + self.assertRaises(errors.DeserializationError, decode_b64jose, u'x') + + def test_decode_b64jose_size(self): + from acme.jose.json_util import decode_b64jose + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3)) + self.assertRaises( + errors.DeserializationError, decode_b64jose, u'Zm9v', size=2) + self.assertRaises( + errors.DeserializationError, decode_b64jose, u'Zm9v', size=4) + + def test_decode_b64jose_minimum_size(self): + from acme.jose.json_util import decode_b64jose + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=3, minimum=True)) + self.assertEqual(b'foo', decode_b64jose(u'Zm9v', size=2, minimum=True)) + self.assertRaises(errors.DeserializationError, decode_b64jose, + u'Zm9v', size=4, minimum=True) + + def test_encode_hex16(self): + from acme.jose.json_util import encode_hex16 + encoded = encode_hex16(b'foo') + self.assertEqual(u'666f6f', encoded) + self.assertTrue(isinstance(encoded, six.string_types)) + + def test_decode_hex16(self): + from acme.jose.json_util import decode_hex16 + decoded = decode_hex16(u'666f6f') + self.assertEqual(b'foo', decoded) + self.assertTrue(isinstance(decoded, six.binary_type)) + + def test_decode_hex16_minimum_size(self): + from acme.jose.json_util import decode_hex16 + self.assertEqual(b'foo', decode_hex16(u'666f6f', size=3, minimum=True)) + self.assertEqual(b'foo', decode_hex16(u'666f6f', size=2, minimum=True)) + self.assertRaises(errors.DeserializationError, decode_hex16, + u'666f6f', size=4, minimum=True) + + def test_decode_hex16_odd_length(self): + from acme.jose.json_util import decode_hex16 + self.assertRaises(errors.DeserializationError, decode_hex16, u'x') + + def test_encode_cert(self): + from acme.jose.json_util import encode_cert + self.assertEqual(self.b64_cert, encode_cert(CERT)) + + def test_decode_cert(self): + from acme.jose.json_util import decode_cert + cert = decode_cert(self.b64_cert) + self.assertTrue(isinstance(cert, util.ComparableX509)) + self.assertEqual(cert, CERT) + self.assertRaises(errors.DeserializationError, decode_cert, u'') + + def test_encode_csr(self): + from acme.jose.json_util import encode_csr + self.assertEqual(self.b64_csr, encode_csr(CSR)) + + def test_decode_csr(self): + from acme.jose.json_util import decode_csr + csr = decode_csr(self.b64_csr) + self.assertTrue(isinstance(csr, util.ComparableX509)) + self.assertEqual(csr, CSR) + self.assertRaises(errors.DeserializationError, decode_csr, u'') + + +class TypedJSONObjectWithFieldsTest(unittest.TestCase): + + def setUp(self): + from acme.jose.json_util import TypedJSONObjectWithFields + + # pylint: disable=missing-docstring,abstract-method + # pylint: disable=too-few-public-methods + + class MockParentTypedJSONObjectWithFields(TypedJSONObjectWithFields): + TYPES = {} + type_field_name = 'type' + + @MockParentTypedJSONObjectWithFields.register + class MockTypedJSONObjectWithFields( + MockParentTypedJSONObjectWithFields): + typ = 'test' + __slots__ = ('foo',) + + @classmethod + def fields_from_json(cls, jobj): + return {'foo': jobj['foo']} + + def fields_to_partial_json(self): + return {'foo': self.foo} + + self.parent_cls = MockParentTypedJSONObjectWithFields + self.msg = MockTypedJSONObjectWithFields(foo='bar') + + def test_to_partial_json(self): + self.assertEqual(self.msg.to_partial_json(), { + 'type': 'test', + 'foo': 'bar', + }) + + def test_from_json_non_dict_fails(self): + for value in [[], (), 5, "asd"]: # all possible input types + self.assertRaises( + errors.DeserializationError, self.parent_cls.from_json, value) + + def test_from_json_dict_no_type_fails(self): + self.assertRaises( + errors.DeserializationError, self.parent_cls.from_json, {}) + + def test_from_json_unknown_type_fails(self): + self.assertRaises(errors.UnrecognizedTypeError, + self.parent_cls.from_json, {'type': 'bar'}) + + def test_from_json_returns_obj(self): + self.assertEqual({'foo': 'bar'}, self.parent_cls.from_json( + {'type': 'test', 'foo': 'bar'})) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py new file mode 100644 index 000000000..9b682ecab --- /dev/null +++ b/acme/acme/jose/jwa.py @@ -0,0 +1,180 @@ +"""JSON Web Algorithm. + +https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-40 + +""" +import abc +import collections +import logging + +import cryptography.exceptions +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import hmac # type: ignore +from cryptography.hazmat.primitives.asymmetric import padding # type: ignore + +from acme.jose import errors +from acme.jose import interfaces +from acme.jose import jwk + + +logger = logging.getLogger(__name__) + + +class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method + # pylint: disable=too-few-public-methods + # for some reason disable=abstract-method has to be on the line + # above... + """JSON Web Algorithm.""" + + +class JWASignature(JWA, collections.Hashable): # type: ignore + """JSON Web Signature Algorithm.""" + SIGNATURES = {} # type: dict + + def __init__(self, name): + self.name = name + + def __eq__(self, other): + if not isinstance(other, JWASignature): + return NotImplemented + return self.name == other.name + + def __hash__(self): + return hash((self.__class__, self.name)) + + def __ne__(self, other): + return not self == other + + @classmethod + def register(cls, signature_cls): + """Register class for JSON deserialization.""" + cls.SIGNATURES[signature_cls.name] = signature_cls + return signature_cls + + def to_partial_json(self): + return self.name + + @classmethod + def from_json(cls, jobj): + return cls.SIGNATURES[jobj] + + @abc.abstractmethod + def sign(self, key, msg): # pragma: no cover + """Sign the ``msg`` using ``key``.""" + raise NotImplementedError() + + @abc.abstractmethod + def verify(self, key, msg, sig): # pragma: no cover + """Verify the ``msg` and ``sig`` using ``key``.""" + raise NotImplementedError() + + def __repr__(self): + return self.name + + +class _JWAHS(JWASignature): + + kty = jwk.JWKOct + + def __init__(self, name, hash_): + super(_JWAHS, self).__init__(name) + self.hash = hash_() + + def sign(self, key, msg): + signer = hmac.HMAC(key, self.hash, backend=default_backend()) + signer.update(msg) + return signer.finalize() + + def verify(self, key, msg, sig): + verifier = hmac.HMAC(key, self.hash, backend=default_backend()) + verifier.update(msg) + try: + verifier.verify(sig) + except cryptography.exceptions.InvalidSignature as error: + logger.debug(error, exc_info=True) + return False + else: + return True + + +class _JWARSA(object): + + kty = jwk.JWKRSA + padding = NotImplemented + hash = NotImplemented + + def sign(self, key, msg): + """Sign the ``msg`` using ``key``.""" + try: + signer = key.signer(self.padding, self.hash) + except AttributeError as error: + logger.debug(error, exc_info=True) + raise errors.Error("Public key cannot be used for signing") + except ValueError as error: # digest too large + logger.debug(error, exc_info=True) + raise errors.Error(str(error)) + signer.update(msg) + try: + return signer.finalize() + except ValueError as error: + logger.debug(error, exc_info=True) + raise errors.Error(str(error)) + + def verify(self, key, msg, sig): + """Verify the ``msg` and ``sig`` using ``key``.""" + verifier = key.verifier(sig, self.padding, self.hash) + verifier.update(msg) + try: + verifier.verify() + except cryptography.exceptions.InvalidSignature as error: + logger.debug(error, exc_info=True) + return False + else: + return True + + +class _JWARS(_JWARSA, JWASignature): + + def __init__(self, name, hash_): + super(_JWARS, self).__init__(name) + self.padding = padding.PKCS1v15() + self.hash = hash_() + + +class _JWAPS(_JWARSA, JWASignature): + + def __init__(self, name, hash_): + super(_JWAPS, self).__init__(name) + self.padding = padding.PSS( + mgf=padding.MGF1(hash_()), + salt_length=padding.PSS.MAX_LENGTH) + self.hash = hash_() + + +class _JWAES(JWASignature): # pylint: disable=abstract-class-not-used + + # TODO: implement ES signatures + + def sign(self, key, msg): # pragma: no cover + raise NotImplementedError() + + def verify(self, key, msg, sig): # pragma: no cover + raise NotImplementedError() + + +HS256 = JWASignature.register(_JWAHS('HS256', hashes.SHA256)) +HS384 = JWASignature.register(_JWAHS('HS384', hashes.SHA384)) +HS512 = JWASignature.register(_JWAHS('HS512', hashes.SHA512)) + +RS256 = JWASignature.register(_JWARS('RS256', hashes.SHA256)) +RS384 = JWASignature.register(_JWARS('RS384', hashes.SHA384)) +RS512 = JWASignature.register(_JWARS('RS512', hashes.SHA512)) + +PS256 = JWASignature.register(_JWAPS('PS256', hashes.SHA256)) +PS384 = JWASignature.register(_JWAPS('PS384', hashes.SHA384)) +PS512 = JWASignature.register(_JWAPS('PS512', hashes.SHA512)) + +ES256 = JWASignature.register(_JWAES('ES256')) +ES384 = JWASignature.register(_JWAES('ES384')) +ES512 = JWASignature.register(_JWAES('ES512')) diff --git a/acme/acme/jose/jwa_test.py b/acme/acme/jose/jwa_test.py new file mode 100644 index 000000000..3328d083a --- /dev/null +++ b/acme/acme/jose/jwa_test.py @@ -0,0 +1,104 @@ +"""Tests for acme.jose.jwa.""" +import unittest + +from acme import test_util + +from acme.jose import errors + + +RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') +RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') +RSA1024_KEY = test_util.load_rsa_private_key('rsa1024_key.pem') + + +class JWASignatureTest(unittest.TestCase): + """Tests for acme.jose.jwa.JWASignature.""" + + def setUp(self): + from acme.jose.jwa import JWASignature + + class MockSig(JWASignature): + # pylint: disable=missing-docstring,too-few-public-methods + # pylint: disable=abstract-class-not-used + def sign(self, key, msg): + raise NotImplementedError() # pragma: no cover + + def verify(self, key, msg, sig): + raise NotImplementedError() # pragma: no cover + + # pylint: disable=invalid-name + self.Sig1 = MockSig('Sig1') + self.Sig2 = MockSig('Sig2') + + def test_eq(self): + self.assertEqual(self.Sig1, self.Sig1) + + def test_ne(self): + self.assertNotEqual(self.Sig1, self.Sig2) + + def test_ne_other_type(self): + self.assertNotEqual(self.Sig1, 5) + + def test_repr(self): + self.assertEqual('Sig1', repr(self.Sig1)) + self.assertEqual('Sig2', repr(self.Sig2)) + + def test_to_partial_json(self): + self.assertEqual(self.Sig1.to_partial_json(), 'Sig1') + self.assertEqual(self.Sig2.to_partial_json(), 'Sig2') + + def test_from_json(self): + from acme.jose.jwa import JWASignature + from acme.jose.jwa import RS256 + self.assertTrue(JWASignature.from_json('RS256') is RS256) + + +class JWAHSTest(unittest.TestCase): # pylint: disable=too-few-public-methods + + def test_it(self): + from acme.jose.jwa import HS256 + sig = ( + b"\xceR\xea\xcd\x94\xab\xcf\xfb\xe0\xacA.:\x1a'\x08i\xe2\xc4" + b"\r\x85+\x0e\x85\xaeUZ\xd4\xb3\x97zO" + ) + self.assertEqual(HS256.sign(b'some key', b'foo'), sig) + self.assertTrue(HS256.verify(b'some key', b'foo', sig) is True) + self.assertTrue(HS256.verify(b'some key', b'foo', sig + b'!') is False) + + +class JWARSTest(unittest.TestCase): + + def test_sign_no_private_part(self): + from acme.jose.jwa import RS256 + self.assertRaises( + errors.Error, RS256.sign, RSA512_KEY.public_key(), b'foo') + + def test_sign_key_too_small(self): + from acme.jose.jwa import RS256 + from acme.jose.jwa import PS256 + self.assertRaises(errors.Error, RS256.sign, RSA256_KEY, b'foo') + self.assertRaises(errors.Error, PS256.sign, RSA256_KEY, b'foo') + + def test_rs(self): + from acme.jose.jwa import RS256 + sig = ( + b'|\xc6\xb2\xa4\xab(\x87\x99\xfa*:\xea\xf8\xa0N&}\x9f\x0f\xc0O' + b'\xc6t\xa3\xe6\xfa\xbb"\x15Y\x80Y\xe0\x81\xb8\x88)\xba\x0c\x9c' + b'\xa4\x99\x1e\x19&\xd8\xc7\x99S\x97\xfc\x85\x0cOV\xe6\x07\x99' + b'\xd2\xb9.>}\xfd' + ) + self.assertEqual(RS256.sign(RSA512_KEY, b'foo'), sig) + self.assertTrue(RS256.verify(RSA512_KEY.public_key(), b'foo', sig)) + self.assertFalse(RS256.verify( + RSA512_KEY.public_key(), b'foo', sig + b'!')) + + def test_ps(self): + from acme.jose.jwa import PS256 + sig = PS256.sign(RSA1024_KEY, b'foo') + self.assertTrue(PS256.verify(RSA1024_KEY.public_key(), b'foo', sig)) + self.assertFalse(PS256.verify( + RSA1024_KEY.public_key(), b'foo', sig + b'!')) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py new file mode 100644 index 000000000..54423f670 --- /dev/null +++ b/acme/acme/jose/jwk.py @@ -0,0 +1,281 @@ +"""JSON Web Key.""" +import abc +import binascii +import json +import logging + +import cryptography.exceptions +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec # type: ignore +from cryptography.hazmat.primitives.asymmetric import rsa + +import six + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import util + + +logger = logging.getLogger(__name__) + + +class JWK(json_util.TypedJSONObjectWithFields): + # pylint: disable=too-few-public-methods + """JSON Web Key.""" + type_field_name = 'kty' + TYPES = {} # type: dict + cryptography_key_types = () # type: tuple + """Subclasses should override.""" + + required = NotImplemented + """Required members of public key's representation as defined by JWK/JWA.""" + + _thumbprint_json_dumps_params = { + # "no whitespace or line breaks before or after any syntactic + # elements" + 'indent': None, + 'separators': (',', ':'), + # "members ordered lexicographically by the Unicode [UNICODE] + # code points of the member names" + 'sort_keys': True, + } + + def thumbprint(self, hash_function=hashes.SHA256): + """Compute JWK Thumbprint. + + https://tools.ietf.org/html/rfc7638 + + :returns bytes: + + """ + digest = hashes.Hash(hash_function(), backend=default_backend()) + digest.update(json.dumps( + dict((k, v) for k, v in six.iteritems(self.to_json()) + if k in self.required), + **self._thumbprint_json_dumps_params).encode()) + return digest.finalize() + + @abc.abstractmethod + def public_key(self): # pragma: no cover + """Generate JWK with public key. + + For symmetric cryptosystems, this would return ``self``. + + """ + raise NotImplementedError() + + @classmethod + def _load_cryptography_key(cls, data, password=None, backend=None): + backend = default_backend() if backend is None else backend + exceptions = {} + + # private key? + for loader in (serialization.load_pem_private_key, + serialization.load_der_private_key): + try: + return loader(data, password, backend) + except (ValueError, TypeError, + cryptography.exceptions.UnsupportedAlgorithm) as error: + exceptions[loader] = error + + # public key? + for loader in (serialization.load_pem_public_key, + serialization.load_der_public_key): + try: + return loader(data, backend) + except (ValueError, + cryptography.exceptions.UnsupportedAlgorithm) as error: + exceptions[loader] = error + + # no luck + raise errors.Error('Unable to deserialize key: {0}'.format(exceptions)) + + @classmethod + def load(cls, data, password=None, backend=None): + """Load serialized key as JWK. + + :param str data: Public or private key serialized as PEM or DER. + :param str password: Optional password. + :param backend: A `.PEMSerializationBackend` and + `.DERSerializationBackend` provider. + + :raises errors.Error: if unable to deserialize, or unsupported + JWK algorithm + + :returns: JWK of an appropriate type. + :rtype: `JWK` + + """ + try: + key = cls._load_cryptography_key(data, password, backend) + except errors.Error as error: + logger.debug('Loading symmetric key, asymmetric failed: %s', error) + return JWKOct(key=data) + + if cls.typ is not NotImplemented and not isinstance( + key, cls.cryptography_key_types): + raise errors.Error('Unable to deserialize {0} into {1}'.format( + key.__class__, cls.__class__)) + for jwk_cls in six.itervalues(cls.TYPES): + if isinstance(key, jwk_cls.cryptography_key_types): + return jwk_cls(key=key) + raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__)) + + +@JWK.register +class JWKES(JWK): # pragma: no cover + # pylint: disable=abstract-class-not-used + """ES JWK. + + .. warning:: This is not yet implemented! + + """ + typ = 'ES' + cryptography_key_types = ( + ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey) + required = ('crv', JWK.type_field_name, 'x', 'y') + + def fields_to_partial_json(self): + raise NotImplementedError() + + @classmethod + def fields_from_json(cls, jobj): + raise NotImplementedError() + + def public_key(self): + raise NotImplementedError() + + +@JWK.register +class JWKOct(JWK): + """Symmetric JWK.""" + typ = 'oct' + __slots__ = ('key',) + required = ('k', JWK.type_field_name) + + def fields_to_partial_json(self): + # TODO: An "alg" member SHOULD also be present to identify the + # algorithm intended to be used with the key, unless the + # application uses another means or convention to determine + # the algorithm used. + return {'k': json_util.encode_b64jose(self.key)} + + @classmethod + def fields_from_json(cls, jobj): + return cls(key=json_util.decode_b64jose(jobj['k'])) + + def public_key(self): + return self + + +@JWK.register +class JWKRSA(JWK): + """RSA JWK. + + :ivar key: `cryptography.hazmat.primitives.rsa.RSAPrivateKey` + or `cryptography.hazmat.primitives.rsa.RSAPublicKey` wrapped + in `.ComparableRSAKey` + + """ + typ = 'RSA' + cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey) + __slots__ = ('key',) + required = ('e', JWK.type_field_name, 'n') + + def __init__(self, *args, **kwargs): + if 'key' in kwargs and not isinstance( + kwargs['key'], util.ComparableRSAKey): + kwargs['key'] = util.ComparableRSAKey(kwargs['key']) + super(JWKRSA, self).__init__(*args, **kwargs) + + @classmethod + def _encode_param(cls, data): + """Encode Base64urlUInt. + + :type data: long + :rtype: unicode + + """ + def _leading_zeros(arg): + if len(arg) % 2: + return '0' + arg + return arg + + return json_util.encode_b64jose(binascii.unhexlify( + _leading_zeros(hex(data)[2:].rstrip('L')))) + + @classmethod + def _decode_param(cls, data): + """Decode Base64urlUInt.""" + try: + return int(binascii.hexlify(json_util.decode_b64jose(data)), 16) + except ValueError: # invalid literal for long() with base 16 + raise errors.DeserializationError() + + def public_key(self): + return type(self)(key=self.key.public_key()) + + @classmethod + def fields_from_json(cls, jobj): + # pylint: disable=invalid-name + n, e = (cls._decode_param(jobj[x]) for x in ('n', 'e')) + public_numbers = rsa.RSAPublicNumbers(e=e, n=n) + if 'd' not in jobj: # public key + key = public_numbers.public_key(default_backend()) + else: # private key + d = cls._decode_param(jobj['d']) + if ('p' in jobj or 'q' in jobj or 'dp' in jobj or + 'dq' in jobj or 'qi' in jobj or 'oth' in jobj): + # "If the producer includes any of the other private + # key parameters, then all of the others MUST be + # present, with the exception of "oth", which MUST + # only be present when more than two prime factors + # were used." + p, q, dp, dq, qi, = all_params = tuple( + jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi')) + if tuple(param for param in all_params if param is None): + raise errors.Error( + 'Some private parameters are missing: {0}'.format( + all_params)) + p, q, dp, dq, qi = tuple( + cls._decode_param(x) for x in all_params) + + # TODO: check for oth + else: + # cryptography>=0.8 + p, q = rsa.rsa_recover_prime_factors(n, e, d) + dp = rsa.rsa_crt_dmp1(d, p) + dq = rsa.rsa_crt_dmq1(d, q) + qi = rsa.rsa_crt_iqmp(p, q) + + key = rsa.RSAPrivateNumbers( + p, q, d, dp, dq, qi, public_numbers).private_key( + default_backend()) + + return cls(key=key) + + def fields_to_partial_json(self): + # pylint: disable=protected-access + if isinstance(self.key._wrapped, rsa.RSAPublicKey): + numbers = self.key.public_numbers() + params = { + 'n': numbers.n, + 'e': numbers.e, + } + else: # rsa.RSAPrivateKey + private = self.key.private_numbers() + public = self.key.public_key().public_numbers() + params = { + 'n': public.n, + 'e': public.e, + 'd': private.d, + 'p': private.p, + 'q': private.q, + 'dp': private.dmp1, + 'dq': private.dmq1, + 'qi': private.iqmp, + } + return dict((key, self._encode_param(value)) + for key, value in six.iteritems(params)) diff --git a/acme/acme/jose/jwk_test.py b/acme/acme/jose/jwk_test.py new file mode 100644 index 000000000..eea5793bf --- /dev/null +++ b/acme/acme/jose/jwk_test.py @@ -0,0 +1,191 @@ +"""Tests for acme.jose.jwk.""" +import binascii +import unittest + +from acme import test_util + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import util + + +DSA_PEM = test_util.load_vector('dsa512_key.pem') +RSA256_KEY = test_util.load_rsa_private_key('rsa256_key.pem') +RSA512_KEY = test_util.load_rsa_private_key('rsa512_key.pem') + + +class JWKTest(unittest.TestCase): + """Tests for acme.jose.jwk.JWK.""" + + def test_load(self): + from acme.jose.jwk import JWK + self.assertRaises(errors.Error, JWK.load, DSA_PEM) + + def test_load_subclass_wrong_type(self): + from acme.jose.jwk import JWKRSA + self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM) + + +class JWKTestBaseMixin(object): + """Mixin test for JWK subclass tests.""" + + thumbprint = NotImplemented + + def test_thumbprint_private(self): + self.assertEqual(self.thumbprint, self.jwk.thumbprint()) + + def test_thumbprint_public(self): + self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint()) + + +class JWKOctTest(unittest.TestCase, JWKTestBaseMixin): + """Tests for acme.jose.jwk.JWKOct.""" + + thumbprint = (b"\xf3\xe7\xbe\xa8`\xd2\xdap\xe9}\x9c\xce>" + b"\xd0\xfcI\xbe\xcd\x92'\xd4o\x0e\xf41\xea" + b"\x8e(\x8a\xb2i\x1c") + + def setUp(self): + from acme.jose.jwk import JWKOct + self.jwk = JWKOct(key=b'foo') + self.jobj = {'kty': 'oct', 'k': json_util.encode_b64jose(b'foo')} + + def test_to_partial_json(self): + self.assertEqual(self.jwk.to_partial_json(), self.jobj) + + def test_from_json(self): + from acme.jose.jwk import JWKOct + self.assertEqual(self.jwk, JWKOct.from_json(self.jobj)) + + def test_from_json_hashable(self): + from acme.jose.jwk import JWKOct + hash(JWKOct.from_json(self.jobj)) + + def test_load(self): + from acme.jose.jwk import JWKOct + self.assertEqual(self.jwk, JWKOct.load(b'foo')) + + def test_public_key(self): + self.assertTrue(self.jwk.public_key() is self.jwk) + + +class JWKRSATest(unittest.TestCase, JWKTestBaseMixin): + """Tests for acme.jose.jwk.JWKRSA.""" + # pylint: disable=too-many-instance-attributes + + thumbprint = (b'\x83K\xdc#3\x98\xca\x98\xed\xcb\x80\x80<\x0c' + b'\xf0\x95\xb9H\xb2*l\xbd$\xe5&|O\x91\xd4 \xb0Y') + + def setUp(self): + from acme.jose.jwk import JWKRSA + self.jwk256 = JWKRSA(key=RSA256_KEY.public_key()) + self.jwk256json = { + 'kty': 'RSA', + 'e': 'AQAB', + 'n': 'm2Fylv-Uz7trgTW8EBHP3FQSMeZs2GNQ6VRo1sIVJEk', + } + # pylint: disable=protected-access + self.jwk256_not_comparable = JWKRSA( + key=RSA256_KEY.public_key()._wrapped) + self.jwk512 = JWKRSA(key=RSA512_KEY.public_key()) + self.jwk512json = { + 'kty': 'RSA', + 'e': 'AQAB', + 'n': 'rHVztFHtH92ucFJD_N_HW9AsdRsUuHUBBBDlHwNlRd3fp5' + '80rv2-6QWE30cWgdmJS86ObRz6lUTor4R0T-3C5Q', + } + self.private = JWKRSA(key=RSA256_KEY) + self.private_json_small = self.jwk256json.copy() + self.private_json_small['d'] = ( + 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE') + self.private_json = self.jwk256json.copy() + self.private_json.update({ + 'd': 'lPQED_EPTV0UIBfNI3KP2d9Jlrc2mrMllmf946bu-CE', + 'p': 'zUVNZn4lLLBD1R6NE8TKNQ', + 'q': 'wcfKfc7kl5jfqXArCRSURQ', + 'dp': 'CWJFq43QvT5Bm5iN8n1okQ', + 'dq': 'bHh2u7etM8LKKCF2pY2UdQ', + 'qi': 'oi45cEkbVoJjAbnQpFY87Q', + }) + self.jwk = self.private + + def test_init_auto_comparable(self): + self.assertTrue(isinstance( + self.jwk256_not_comparable.key, util.ComparableRSAKey)) + self.assertEqual(self.jwk256, self.jwk256_not_comparable) + + def test_encode_param_zero(self): + from acme.jose.jwk import JWKRSA + # pylint: disable=protected-access + # TODO: move encode/decode _param to separate class + self.assertEqual('AA', JWKRSA._encode_param(0)) + + def test_equals(self): + self.assertEqual(self.jwk256, self.jwk256) + self.assertEqual(self.jwk512, self.jwk512) + + def test_not_equals(self): + self.assertNotEqual(self.jwk256, self.jwk512) + self.assertNotEqual(self.jwk512, self.jwk256) + + def test_load(self): + from acme.jose.jwk import JWKRSA + self.assertEqual(self.private, JWKRSA.load( + test_util.load_vector('rsa256_key.pem'))) + + def test_public_key(self): + self.assertEqual(self.jwk256, self.private.public_key()) + + def test_to_partial_json(self): + self.assertEqual(self.jwk256.to_partial_json(), self.jwk256json) + self.assertEqual(self.jwk512.to_partial_json(), self.jwk512json) + self.assertEqual(self.private.to_partial_json(), self.private_json) + + def test_from_json(self): + from acme.jose.jwk import JWK + self.assertEqual( + self.jwk256, JWK.from_json(self.jwk256json)) + self.assertEqual( + self.jwk512, JWK.from_json(self.jwk512json)) + self.assertEqual(self.private, JWK.from_json(self.private_json)) + + def test_from_json_private_small(self): + from acme.jose.jwk import JWK + self.assertEqual(self.private, JWK.from_json(self.private_json_small)) + + def test_from_json_missing_one_additional(self): + from acme.jose.jwk import JWK + del self.private_json['q'] + self.assertRaises(errors.Error, JWK.from_json, self.private_json) + + def test_from_json_hashable(self): + from acme.jose.jwk import JWK + hash(JWK.from_json(self.jwk256json)) + + def test_from_json_non_schema_errors(self): + # valid against schema, but still failing + from acme.jose.jwk import JWK + self.assertRaises(errors.DeserializationError, JWK.from_json, + {'kty': 'RSA', 'e': 'AQAB', 'n': ''}) + self.assertRaises(errors.DeserializationError, JWK.from_json, + {'kty': 'RSA', 'e': 'AQAB', 'n': '1'}) + + def test_thumbprint_go_jose(self): + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk.go#L155 + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L331-L344 + # https://github.com/square/go-jose/blob/4ddd71883fa547d37fbf598071f04512d8bafee3/jwk_test.go#L384 + from acme.jose.jwk import JWKRSA + key = JWKRSA.json_loads("""{ + "kty": "RSA", + "kid": "bilbo.baggins@hobbiton.example", + "use": "sig", + "n": "n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw", + "e": "AQAB" +}""") + self.assertEqual( + binascii.hexlify(key.thumbprint()), + b"f63838e96077ad1fc01c3f8405774dedc0641f558ebb4b40dccf5f9b6d66a932") + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py new file mode 100644 index 000000000..5f446e4b1 --- /dev/null +++ b/acme/acme/jose/jws.py @@ -0,0 +1,433 @@ +"""JOSE Web Signature.""" +import argparse +import base64 +import sys + +import OpenSSL +import six + +from acme.jose import b64 +from acme.jose import errors +from acme.jose import json_util +from acme.jose import jwa +from acme.jose import jwk +from acme.jose import util + + +class MediaType(object): + """MediaType field encoder/decoder.""" + + PREFIX = 'application/' + """MIME Media Type and Content Type prefix.""" + + @classmethod + def decode(cls, value): + """Decoder.""" + # 4.1.10 + if '/' not in value: + if ';' in value: + raise errors.DeserializationError('Unexpected semi-colon') + return cls.PREFIX + value + return value + + @classmethod + def encode(cls, value): + """Encoder.""" + # 4.1.10 + if ';' not in value: + assert value.startswith(cls.PREFIX) + return value[len(cls.PREFIX):] + return value + + +class Header(json_util.JSONObjectWithFields): + """JOSE Header. + + .. warning:: This class supports **only** Registered Header + Parameter Names (as defined in section 4.1 of the + protocol). If you need Public Header Parameter Names (4.2) + or Private Header Parameter Names (4.3), you must subclass + and override :meth:`from_json` and :meth:`to_partial_json` + appropriately. + + .. warning:: This class does not support any extensions through + the "crit" (Critical) Header Parameter (4.1.11) and as a + conforming implementation, :meth:`from_json` treats its + occurrence as an error. Please subclass if you seek for + a different behaviour. + + :ivar x5tS256: "x5t#S256" + :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`. + :ivar str cty: Content-Type, inc. :const:`MediaType.PREFIX`. + + """ + alg = json_util.Field( + 'alg', decoder=jwa.JWASignature.from_json, omitempty=True) + jku = json_util.Field('jku', omitempty=True) + jwk = json_util.Field('jwk', decoder=jwk.JWK.from_json, omitempty=True) + kid = json_util.Field('kid', omitempty=True) + x5u = json_util.Field('x5u', omitempty=True) + x5c = json_util.Field('x5c', omitempty=True, default=()) + x5t = json_util.Field( + 'x5t', decoder=json_util.decode_b64jose, omitempty=True) + x5tS256 = json_util.Field( + 'x5t#S256', decoder=json_util.decode_b64jose, omitempty=True) + typ = json_util.Field('typ', encoder=MediaType.encode, + decoder=MediaType.decode, omitempty=True) + cty = json_util.Field('cty', encoder=MediaType.encode, + decoder=MediaType.decode, omitempty=True) + crit = json_util.Field('crit', omitempty=True, default=()) + + def not_omitted(self): + """Fields that would not be omitted in the JSON object.""" + return dict((name, getattr(self, name)) + for name, field in six.iteritems(self._fields) + if not field.omit(getattr(self, name))) + + def __add__(self, other): + if not isinstance(other, type(self)): + raise TypeError('Header cannot be added to: {0}'.format( + type(other))) + + not_omitted_self = self.not_omitted() + not_omitted_other = other.not_omitted() + + if set(not_omitted_self).intersection(not_omitted_other): + raise TypeError('Addition of overlapping headers not defined') + + not_omitted_self.update(not_omitted_other) + return type(self)(**not_omitted_self) # pylint: disable=star-args + + def find_key(self): + """Find key based on header. + + .. todo:: Supports only "jwk" header parameter lookup. + + :returns: (Public) key found in the header. + :rtype: .JWK + + :raises acme.jose.errors.Error: if key could not be found + + """ + if self.jwk is None: + raise errors.Error('No key found') + return self.jwk + + @crit.decoder + def crit(unused_value): + # pylint: disable=missing-docstring,no-self-argument,no-self-use + raise errors.DeserializationError( + '"crit" is not supported, please subclass') + + # x5c does NOT use JOSE Base64 (4.1.6) + + @x5c.encoder # type: ignore + def x5c(value): # pylint: disable=missing-docstring,no-self-argument + return [base64.b64encode(OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] + + @x5c.decoder # type: ignore + def x5c(value): # pylint: disable=missing-docstring,no-self-argument + try: + return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_ASN1, + base64.b64decode(cert))) for cert in value) + except OpenSSL.crypto.Error as error: + raise errors.DeserializationError(error) + + +class Signature(json_util.JSONObjectWithFields): + """JWS Signature. + + :ivar combined: Combined Header (protected and unprotected, + :class:`Header`). + :ivar unicode protected: JWS protected header (Jose Base-64 decoded). + :ivar header: JWS Unprotected Header (:class:`Header`). + :ivar str signature: The signature. + + """ + header_cls = Header + + __slots__ = ('combined',) + protected = json_util.Field('protected', omitempty=True, default='') + header = json_util.Field( + 'header', omitempty=True, default=header_cls(), + decoder=header_cls.from_json) + signature = json_util.Field( + 'signature', decoder=json_util.decode_b64jose, + encoder=json_util.encode_b64jose) + + @protected.encoder # type: ignore + def protected(value): # pylint: disable=missing-docstring,no-self-argument + # wrong type guess (Signature, not bytes) | pylint: disable=no-member + return json_util.encode_b64jose(value.encode('utf-8')) + + @protected.decoder # type: ignore + def protected(value): # pylint: disable=missing-docstring,no-self-argument + return json_util.decode_b64jose(value).decode('utf-8') + + def __init__(self, **kwargs): + if 'combined' not in kwargs: + kwargs = self._with_combined(kwargs) + super(Signature, self).__init__(**kwargs) + assert self.combined.alg is not None + + @classmethod + def _with_combined(cls, kwargs): + assert 'combined' not in kwargs + header = kwargs.get('header', cls._fields['header'].default) + protected = kwargs.get('protected', cls._fields['protected'].default) + + if protected: + combined = header + cls.header_cls.json_loads(protected) + else: + combined = header + + kwargs['combined'] = combined + return kwargs + + @classmethod + def _msg(cls, protected, payload): + return (b64.b64encode(protected.encode('utf-8')) + b'.' + + b64.b64encode(payload)) + + def verify(self, payload, key=None): + """Verify. + + :param JWK key: Key used for verification. + + """ + key = self.combined.find_key() if key is None else key + return self.combined.alg.verify( + key=key.key, sig=self.signature, + msg=self._msg(self.protected, payload)) + + @classmethod + def sign(cls, payload, key, alg, include_jwk=True, + protect=frozenset(), **kwargs): + """Sign. + + :param JWK key: Key for signature. + + """ + assert isinstance(key, alg.kty) + + header_params = kwargs + header_params['alg'] = alg + if include_jwk: + header_params['jwk'] = key.public_key() + + assert set(header_params).issubset(cls.header_cls._fields) + assert protect.issubset(cls.header_cls._fields) + + protected_params = {} + for header in protect: + if header in header_params: + protected_params[header] = header_params.pop(header) + if protected_params: + # pylint: disable=star-args + protected = cls.header_cls(**protected_params).json_dumps() + else: + protected = '' + + header = cls.header_cls(**header_params) # pylint: disable=star-args + signature = alg.sign(key.key, cls._msg(protected, payload)) + + return cls(protected=protected, header=header, signature=signature) + + def fields_to_partial_json(self): + fields = super(Signature, self).fields_to_partial_json() + if not fields['header'].not_omitted(): + del fields['header'] + return fields + + @classmethod + def fields_from_json(cls, jobj): + fields = super(Signature, cls).fields_from_json(jobj) + fields_with_combined = cls._with_combined(fields) + if 'alg' not in fields_with_combined['combined'].not_omitted(): + raise errors.DeserializationError('alg not present') + return fields_with_combined + + +class JWS(json_util.JSONObjectWithFields): + """JSON Web Signature. + + :ivar str payload: JWS Payload. + :ivar str signature: JWS Signatures. + + """ + __slots__ = ('payload', 'signatures') + + signature_cls = Signature + + def verify(self, key=None): + """Verify.""" + return all(sig.verify(self.payload, key) for sig in self.signatures) + + @classmethod + def sign(cls, payload, **kwargs): + """Sign.""" + return cls(payload=payload, signatures=( + cls.signature_cls.sign(payload=payload, **kwargs),)) + + @property + def signature(self): + """Get a singleton signature. + + :rtype: `signature_cls` + + """ + assert len(self.signatures) == 1 + return self.signatures[0] + + def to_compact(self): + """Compact serialization. + + :rtype: bytes + + """ + assert len(self.signatures) == 1 + + assert 'alg' not in self.signature.header.not_omitted() + # ... it must be in protected + + return ( + b64.b64encode(self.signature.protected.encode('utf-8')) + + b'.' + + b64.b64encode(self.payload) + + b'.' + + b64.b64encode(self.signature.signature)) + + @classmethod + def from_compact(cls, compact): + """Compact deserialization. + + :param bytes compact: + + """ + try: + protected, payload, signature = compact.split(b'.') + except ValueError: + raise errors.DeserializationError( + 'Compact JWS serialization should comprise of exactly' + ' 3 dot-separated components') + + sig = cls.signature_cls( + protected=b64.b64decode(protected).decode('utf-8'), + signature=b64.b64decode(signature)) + return cls(payload=b64.b64decode(payload), signatures=(sig,)) + + def to_partial_json(self, flat=True): # pylint: disable=arguments-differ + assert self.signatures + payload = json_util.encode_b64jose(self.payload) + + if flat and len(self.signatures) == 1: + ret = self.signatures[0].to_partial_json() + ret['payload'] = payload + return ret + else: + return { + 'payload': payload, + 'signatures': self.signatures, + } + + @classmethod + def from_json(cls, jobj): + if 'signature' in jobj and 'signatures' in jobj: + raise errors.DeserializationError('Flat mixed with non-flat') + elif 'signature' in jobj: # flat + return cls(payload=json_util.decode_b64jose(jobj.pop('payload')), + signatures=(cls.signature_cls.from_json(jobj),)) + else: + return cls(payload=json_util.decode_b64jose(jobj['payload']), + signatures=tuple(cls.signature_cls.from_json(sig) + for sig in jobj['signatures'])) + + +class CLI(object): + """JWS CLI.""" + + @classmethod + def sign(cls, args): + """Sign.""" + key = args.alg.kty.load(args.key.read()) + args.key.close() + if args.protect is None: + args.protect = [] + if args.compact: + args.protect.append('alg') + + sig = JWS.sign(payload=sys.stdin.read().encode(), key=key, alg=args.alg, + protect=set(args.protect)) + + if args.compact: + six.print_(sig.to_compact().decode('utf-8')) + else: # JSON + six.print_(sig.json_dumps_pretty()) + + @classmethod + def verify(cls, args): + """Verify.""" + if args.compact: + sig = JWS.from_compact(sys.stdin.read().encode()) + else: # JSON + try: + sig = JWS.json_loads(sys.stdin.read()) + except errors.Error as error: + six.print_(error) + return -1 + + if args.key is not None: + assert args.kty is not None + key = args.kty.load(args.key.read()).public_key() + args.key.close() + else: + key = None + + sys.stdout.write(sig.payload) + return not sig.verify(key=key) + + @classmethod + def _alg_type(cls, arg): + return jwa.JWASignature.from_json(arg) + + @classmethod + def _header_type(cls, arg): + assert arg in Signature.header_cls._fields + return arg + + @classmethod + def _kty_type(cls, arg): + assert arg in jwk.JWK.TYPES + return jwk.JWK.TYPES[arg] + + @classmethod + def run(cls, args=sys.argv[1:]): + """Parse arguments and sign/verify.""" + parser = argparse.ArgumentParser() + parser.add_argument('--compact', action='store_true') + + subparsers = parser.add_subparsers() + parser_sign = subparsers.add_parser('sign') + parser_sign.set_defaults(func=cls.sign) + parser_sign.add_argument( + '-k', '--key', type=argparse.FileType('rb'), required=True) + parser_sign.add_argument( + '-a', '--alg', type=cls._alg_type, default=jwa.RS256) + parser_sign.add_argument( + '-p', '--protect', action='append', type=cls._header_type) + + parser_verify = subparsers.add_parser('verify') + parser_verify.set_defaults(func=cls.verify) + parser_verify.add_argument( + '-k', '--key', type=argparse.FileType('rb'), required=False) + parser_verify.add_argument( + '--kty', type=cls._kty_type, required=False) + + parsed = parser.parse_args(args) + return parsed.func(parsed) + + +if __name__ == '__main__': + exit(CLI.run()) # pragma: no cover diff --git a/acme/acme/jose/jws_test.py b/acme/acme/jose/jws_test.py new file mode 100644 index 000000000..ec91f6a1b --- /dev/null +++ b/acme/acme/jose/jws_test.py @@ -0,0 +1,239 @@ +"""Tests for acme.jose.jws.""" +import base64 +import unittest + +import mock +import OpenSSL + +from acme import test_util + +from acme.jose import errors +from acme.jose import json_util +from acme.jose import jwa +from acme.jose import jwk + + +CERT = test_util.load_comparable_cert('cert.pem') +KEY = jwk.JWKRSA.load(test_util.load_vector('rsa512_key.pem')) + + +class MediaTypeTest(unittest.TestCase): + """Tests for acme.jose.jws.MediaType.""" + + def test_decode(self): + from acme.jose.jws import MediaType + self.assertEqual('application/app', MediaType.decode('application/app')) + self.assertEqual('application/app', MediaType.decode('app')) + self.assertRaises( + errors.DeserializationError, MediaType.decode, 'app;foo') + + def test_encode(self): + from acme.jose.jws import MediaType + self.assertEqual('app', MediaType.encode('application/app')) + self.assertEqual('application/app;foo', + MediaType.encode('application/app;foo')) + + +class HeaderTest(unittest.TestCase): + """Tests for acme.jose.jws.Header.""" + + def setUp(self): + from acme.jose.jws import Header + self.header1 = Header(jwk='foo') + self.header2 = Header(jwk='bar') + self.crit = Header(crit=('a', 'b')) + self.empty = Header() + + def test_add_non_empty(self): + from acme.jose.jws import Header + self.assertEqual(Header(jwk='foo', crit=('a', 'b')), + self.header1 + self.crit) + + def test_add_empty(self): + self.assertEqual(self.header1, self.header1 + self.empty) + self.assertEqual(self.header1, self.empty + self.header1) + + def test_add_overlapping_error(self): + self.assertRaises(TypeError, self.header1.__add__, self.header2) + + def test_add_wrong_type_error(self): + self.assertRaises(TypeError, self.header1.__add__, 'xxx') + + def test_crit_decode_always_errors(self): + from acme.jose.jws import Header + self.assertRaises(errors.DeserializationError, Header.from_json, + {'crit': ['a', 'b']}) + + def test_x5c_decoding(self): + from acme.jose.jws import Header + header = Header(x5c=(CERT, CERT)) + jobj = header.to_partial_json() + cert_asn1 = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, CERT.wrapped) + cert_b64 = base64.b64encode(cert_asn1) + self.assertEqual(jobj, {'x5c': [cert_b64, cert_b64]}) + self.assertEqual(header, Header.from_json(jobj)) + jobj['x5c'][0] = base64.b64encode(b'xxx' + cert_asn1) + self.assertRaises(errors.DeserializationError, Header.from_json, jobj) + + def test_find_key(self): + self.assertEqual('foo', self.header1.find_key()) + self.assertEqual('bar', self.header2.find_key()) + self.assertRaises(errors.Error, self.crit.find_key) + + +class SignatureTest(unittest.TestCase): + """Tests for acme.jose.jws.Signature.""" + + def test_from_json(self): + from acme.jose.jws import Header + from acme.jose.jws import Signature + self.assertEqual( + Signature(signature=b'foo', header=Header(alg=jwa.RS256)), + Signature.from_json( + {'signature': 'Zm9v', 'header': {'alg': 'RS256'}})) + + def test_from_json_no_alg_error(self): + from acme.jose.jws import Signature + self.assertRaises(errors.DeserializationError, + Signature.from_json, {'signature': 'foo'}) + + +class JWSTest(unittest.TestCase): + """Tests for acme.jose.jws.JWS.""" + + def setUp(self): + self.privkey = KEY + self.pubkey = self.privkey.public_key() + + from acme.jose.jws import JWS + self.unprotected = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256) + self.protected = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256, + protect=frozenset(['jwk', 'alg'])) + self.mixed = JWS.sign( + payload=b'foo', key=self.privkey, alg=jwa.RS256, + protect=frozenset(['alg'])) + + def test_pubkey_jwk(self): + self.assertEqual(self.unprotected.signature.combined.jwk, self.pubkey) + self.assertEqual(self.protected.signature.combined.jwk, self.pubkey) + self.assertEqual(self.mixed.signature.combined.jwk, self.pubkey) + + def test_sign_unprotected(self): + self.assertTrue(self.unprotected.verify()) + + def test_sign_protected(self): + self.assertTrue(self.protected.verify()) + + def test_sign_mixed(self): + self.assertTrue(self.mixed.verify()) + + def test_compact_lost_unprotected(self): + compact = self.mixed.to_compact() + self.assertEqual( + b'eyJhbGciOiAiUlMyNTYifQ.Zm9v.OHdxFVj73l5LpxbFp1AmYX4yJM0Pyb' + b'_893n1zQjpim_eLS5J1F61lkvrCrCDErTEJnBGOGesJ72M7b6Ve1cAJA', + compact) + + from acme.jose.jws import JWS + mixed = JWS.from_compact(compact) + + self.assertNotEqual(self.mixed, mixed) + self.assertEqual( + set(['alg']), set(mixed.signature.combined.not_omitted())) + + def test_from_compact_missing_components(self): + from acme.jose.jws import JWS + self.assertRaises(errors.DeserializationError, JWS.from_compact, b'.') + + def test_json_omitempty(self): + protected_jobj = self.protected.to_partial_json(flat=True) + unprotected_jobj = self.unprotected.to_partial_json(flat=True) + + self.assertTrue('protected' not in unprotected_jobj) + self.assertTrue('header' not in protected_jobj) + + unprotected_jobj['header'] = unprotected_jobj['header'].to_json() + + from acme.jose.jws import JWS + self.assertEqual(JWS.from_json(protected_jobj), self.protected) + self.assertEqual(JWS.from_json(unprotected_jobj), self.unprotected) + + def test_json_flat(self): + jobj_to = { + 'signature': json_util.encode_b64jose( + self.mixed.signature.signature), + 'payload': json_util.encode_b64jose(b'foo'), + 'header': self.mixed.signature.header, + 'protected': json_util.encode_b64jose( + self.mixed.signature.protected.encode('utf-8')), + } + jobj_from = jobj_to.copy() + jobj_from['header'] = jobj_from['header'].to_json() + + self.assertEqual(self.mixed.to_partial_json(flat=True), jobj_to) + from acme.jose.jws import JWS + self.assertEqual(self.mixed, JWS.from_json(jobj_from)) + + def test_json_not_flat(self): + jobj_to = { + 'signatures': (self.mixed.signature,), + 'payload': json_util.encode_b64jose(b'foo'), + } + jobj_from = jobj_to.copy() + jobj_from['signatures'] = [jobj_to['signatures'][0].to_json()] + + self.assertEqual(self.mixed.to_partial_json(flat=False), jobj_to) + from acme.jose.jws import JWS + self.assertEqual(self.mixed, JWS.from_json(jobj_from)) + + def test_from_json_mixed_flat(self): + from acme.jose.jws import JWS + self.assertRaises(errors.DeserializationError, JWS.from_json, + {'signatures': (), 'signature': 'foo'}) + + def test_from_json_hashable(self): + from acme.jose.jws import JWS + hash(JWS.from_json(self.mixed.to_json())) + + +class CLITest(unittest.TestCase): + + def setUp(self): + self.key_path = test_util.vector_path('rsa512_key.pem') + + def test_unverified(self): + from acme.jose.jws import CLI + with mock.patch('sys.stdin') as sin: + sin.read.return_value = '{"payload": "foo", "signature": "xxx"}' + with mock.patch('sys.stdout'): + self.assertEqual(-1, CLI.run(['verify'])) + + def test_json(self): + from acme.jose.jws import CLI + + with mock.patch('sys.stdin') as sin: + sin.read.return_value = 'foo' + with mock.patch('sys.stdout') as sout: + CLI.run(['sign', '-k', self.key_path, '-a', 'RS256', + '-p', 'jwk']) + sin.read.return_value = sout.write.mock_calls[0][1][0] + self.assertEqual(0, CLI.run(['verify'])) + + def test_compact(self): + from acme.jose.jws import CLI + + with mock.patch('sys.stdin') as sin: + sin.read.return_value = 'foo' + with mock.patch('sys.stdout') as sout: + CLI.run(['--compact', 'sign', '-k', self.key_path]) + sin.read.return_value = sout.write.mock_calls[0][1][0] + self.assertEqual(0, CLI.run([ + '--compact', 'verify', '--kty', 'RSA', + '-k', self.key_path])) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py new file mode 100644 index 000000000..26b7e0c5a --- /dev/null +++ b/acme/acme/jose/util.py @@ -0,0 +1,226 @@ +"""JOSE utilities.""" +import collections + +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL +import six + + +class abstractclassmethod(classmethod): + # pylint: disable=invalid-name,too-few-public-methods + """Descriptor for an abstract classmethod. + + It augments the :mod:`abc` framework with an abstract + classmethod. This is implemented as :class:`abc.abstractclassmethod` + in the standard Python library starting with version 3.2. + + This particular implementation, allegedly based on Python 3.3 source + code, is stolen from + http://stackoverflow.com/questions/11217878/python-2-7-combine-abc-abstractmethod-and-classmethod. + + """ + __isabstractmethod__ = True + + def __init__(self, target): + target.__isabstractmethod__ = True + super(abstractclassmethod, self).__init__(target) + + +class ComparableX509(object): # pylint: disable=too-few-public-methods + """Wrapper for OpenSSL.crypto.X509** objects that supports __eq__. + + :ivar wrapped: Wrapped certificate or certificate request. + :type wrapped: `OpenSSL.crypto.X509` or `OpenSSL.crypto.X509Req`. + + """ + def __init__(self, wrapped): + assert isinstance(wrapped, OpenSSL.crypto.X509) or isinstance( + wrapped, OpenSSL.crypto.X509Req) + self.wrapped = wrapped + + def __getattr__(self, name): + return getattr(self.wrapped, name) + + def _dump(self, filetype=OpenSSL.crypto.FILETYPE_ASN1): + """Dumps the object into a buffer with the specified encoding. + + :param int filetype: The desired encoding. Should be one of + `OpenSSL.crypto.FILETYPE_ASN1`, + `OpenSSL.crypto.FILETYPE_PEM`, or + `OpenSSL.crypto.FILETYPE_TEXT`. + + :returns: Encoded X509 object. + :rtype: str + + """ + if isinstance(self.wrapped, OpenSSL.crypto.X509): + func = OpenSSL.crypto.dump_certificate + else: # assert in __init__ makes sure this is X509Req + func = OpenSSL.crypto.dump_certificate_request + return func(filetype, self.wrapped) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + # pylint: disable=protected-access + return self._dump() == other._dump() + + def __hash__(self): + return hash((self.__class__, self._dump())) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '<{0}({1!r})>'.format(self.__class__.__name__, self.wrapped) + + +class ComparableKey(object): # pylint: disable=too-few-public-methods + """Comparable wrapper for `cryptography` keys. + + See https://github.com/pyca/cryptography/issues/2122. + + """ + __hash__ = NotImplemented + + def __init__(self, wrapped): + self._wrapped = wrapped + + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def __eq__(self, other): + # pylint: disable=protected-access + if (not isinstance(other, self.__class__) or + self._wrapped.__class__ is not other._wrapped.__class__): + return NotImplemented + elif hasattr(self._wrapped, 'private_numbers'): + return self.private_numbers() == other.private_numbers() + elif hasattr(self._wrapped, 'public_numbers'): + return self.public_numbers() == other.public_numbers() + else: + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return '<{0}({1!r})>'.format(self.__class__.__name__, self._wrapped) + + def public_key(self): + """Get wrapped public key.""" + return self.__class__(self._wrapped.public_key()) + + +class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods + """Wrapper for `cryptography` RSA keys. + + Wraps around: + - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey` + - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey` + + """ + + def __hash__(self): + # public_numbers() hasn't got stable hash! + # https://github.com/pyca/cryptography/issues/2143 + if isinstance(self._wrapped, rsa.RSAPrivateKeyWithSerialization): + priv = self.private_numbers() + pub = priv.public_numbers + return hash((self.__class__, priv.p, priv.q, priv.dmp1, + priv.dmq1, priv.iqmp, pub.n, pub.e)) + elif isinstance(self._wrapped, rsa.RSAPublicKeyWithSerialization): + pub = self.public_numbers() + return hash((self.__class__, pub.n, pub.e)) + + +class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore + # pylint: disable=too-few-public-methods + """Immutable key to value mapping with attribute access.""" + + __slots__ = () + """Must be overridden in subclasses.""" + + def __init__(self, **kwargs): + if set(kwargs) != set(self.__slots__): + raise TypeError( + '__init__() takes exactly the following arguments: {0} ' + '({1} given)'.format(', '.join(self.__slots__), + ', '.join(kwargs) if kwargs else 'none')) + for slot in self.__slots__: + object.__setattr__(self, slot, kwargs.pop(slot)) + + def update(self, **kwargs): + """Return updated map.""" + items = dict(self) + items.update(kwargs) + return type(self)(**items) # pylint: disable=star-args + + def __getitem__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) + + def __iter__(self): + return iter(self.__slots__) + + def __len__(self): + return len(self.__slots__) + + def __hash__(self): + return hash(tuple(getattr(self, slot) for slot in self.__slots__)) + + def __setattr__(self, name, value): + raise AttributeError("can't set attribute") + + def __repr__(self): + return '{0}({1})'.format(self.__class__.__name__, ', '.join( + '{0}={1!r}'.format(key, value) + for key, value in six.iteritems(self))) + + +class frozendict(collections.Mapping, collections.Hashable): # type: ignore + # pylint: disable=invalid-name,too-few-public-methods + """Frozen dictionary.""" + __slots__ = ('_items', '_keys') + + def __init__(self, *args, **kwargs): + if kwargs and not args: + items = dict(kwargs) + elif len(args) == 1 and isinstance(args[0], collections.Mapping): + items = args[0] + else: + raise TypeError() + # TODO: support generators/iterators + + object.__setattr__(self, '_items', items) + object.__setattr__(self, '_keys', tuple(sorted(six.iterkeys(items)))) + + def __getitem__(self, key): + return self._items[key] + + def __iter__(self): + return iter(self._keys) + + def __len__(self): + return len(self._items) + + def _sorted_items(self): + return tuple((key, self[key]) for key in self._keys) + + def __hash__(self): + return hash(self._sorted_items()) + + def __getattr__(self, name): + try: + return self._items[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + raise AttributeError("can't set attribute") + + def __repr__(self): + return 'frozendict({0})'.format(', '.join('{0}={1!r}'.format( + key, value) for key, value in self._sorted_items())) diff --git a/acme/acme/jose/util_test.py b/acme/acme/jose/util_test.py new file mode 100644 index 000000000..0038a6cc1 --- /dev/null +++ b/acme/acme/jose/util_test.py @@ -0,0 +1,199 @@ +"""Tests for acme.jose.util.""" +import functools +import unittest + +import six + +from acme import test_util + + +class ComparableX509Test(unittest.TestCase): + """Tests for acme.jose.util.ComparableX509.""" + + def setUp(self): + # test_util.load_comparable_{csr,cert} return ComparableX509 + self.req1 = test_util.load_comparable_csr('csr.pem') + self.req2 = test_util.load_comparable_csr('csr.pem') + self.req_other = test_util.load_comparable_csr('csr-san.pem') + + self.cert1 = test_util.load_comparable_cert('cert.pem') + self.cert2 = test_util.load_comparable_cert('cert.pem') + self.cert_other = test_util.load_comparable_cert('cert-san.pem') + + def test_getattr_proxy(self): + self.assertTrue(self.cert1.has_expired()) + + def test_eq(self): + self.assertEqual(self.req1, self.req2) + self.assertEqual(self.cert1, self.cert2) + + def test_ne(self): + self.assertNotEqual(self.req1, self.req_other) + self.assertNotEqual(self.cert1, self.cert_other) + + def test_ne_wrong_types(self): + self.assertNotEqual(self.req1, 5) + self.assertNotEqual(self.cert1, 5) + + def test_hash(self): + self.assertEqual(hash(self.req1), hash(self.req2)) + self.assertNotEqual(hash(self.req1), hash(self.req_other)) + + self.assertEqual(hash(self.cert1), hash(self.cert2)) + self.assertNotEqual(hash(self.cert1), hash(self.cert_other)) + + def test_repr(self): + for x509 in self.req1, self.cert1: + self.assertEqual(repr(x509), + ''.format(x509.wrapped)) + + +class ComparableRSAKeyTest(unittest.TestCase): + """Tests for acme.jose.util.ComparableRSAKey.""" + + def setUp(self): + # test_utl.load_rsa_private_key return ComparableRSAKey + self.key = test_util.load_rsa_private_key('rsa256_key.pem') + self.key_same = test_util.load_rsa_private_key('rsa256_key.pem') + self.key2 = test_util.load_rsa_private_key('rsa512_key.pem') + + def test_getattr_proxy(self): + self.assertEqual(256, self.key.key_size) + + def test_eq(self): + self.assertEqual(self.key, self.key_same) + + def test_ne(self): + self.assertNotEqual(self.key, self.key2) + + def test_ne_different_types(self): + self.assertNotEqual(self.key, 5) + + def test_ne_not_wrapped(self): + # pylint: disable=protected-access + self.assertNotEqual(self.key, self.key_same._wrapped) + + def test_ne_no_serialization(self): + from acme.jose.util import ComparableRSAKey + self.assertNotEqual(ComparableRSAKey(5), ComparableRSAKey(5)) + + def test_hash(self): + self.assertTrue(isinstance(hash(self.key), int)) + self.assertEqual(hash(self.key), hash(self.key_same)) + self.assertNotEqual(hash(self.key), hash(self.key2)) + + def test_repr(self): + self.assertTrue(repr(self.key).startswith( + '=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', - # formerly known as acme.jose: - 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', 'PyOpenSSL>=0.13', @@ -76,5 +74,10 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, + entry_points={ + 'console_scripts': [ + 'jws = acme.jose.jws:CLI.run', + ], + }, test_suite='acme', ) diff --git a/certbot-apache/certbot_apache/apache_util.py b/certbot-apache/certbot_apache/apache_util.py index f03c9da87..b4a24f137 100644 --- a/certbot-apache/certbot_apache/apache_util.py +++ b/certbot-apache/certbot_apache/apache_util.py @@ -93,8 +93,4 @@ def parse_define_file(filepath, varname): if v == "-D" and len(a_opts) >= i+2: var_parts = a_opts[i+1].partition("=") return_vars[var_parts[0]] = var_parts[2] - elif len(v) > 2 and v.startswith("-D"): - # Found var with no whitespace separator - var_parts = v[2:].partition("=") - return_vars[var_parts[0]] = var_parts[2] return return_vars diff --git a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf index 56c946a4e..17ae1be76 100644 --- a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA SSLHonorCipherOrder on SSLOptions +StrictRequire diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index fd6a9eb11..a13ca04a6 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -16,8 +16,6 @@ ALL_SSL_OPTIONS_HASHES = [ '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27', 'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88', 'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b', - '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791', - 'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/options-ssl-apache.conf b/certbot-apache/certbot_apache/options-ssl-apache.conf index 8113ee81e..950a02a8b 100644 --- a/certbot-apache/certbot_apache/options-ssl-apache.conf +++ b/certbot-apache/certbot_apache/options-ssl-apache.conf @@ -8,7 +8,7 @@ SSLEngine on # Intermediate configuration, tweak to your needs SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA SSLHonorCipherOrder on SSLCompression off diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index 92f1d4a20..d4d4e96b9 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,7 +49,6 @@ class GentooParser(parser.ApacheParser): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() - self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -57,10 +56,3 @@ class GentooParser(parser.ApacheParser): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] - - def update_modules(self): - """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] - matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") - for mod in matches: - self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/centos_test.py b/certbot-apache/certbot_apache/tests/centos_test.py index d7a2a2fd9..7ca47a4d5 100644 --- a/certbot-apache/certbot_apache/tests/centos_test.py +++ b/certbot-apache/certbot_apache/tests/centos_test.py @@ -118,8 +118,6 @@ class MultipleVhostsTestCentOS(util.ApacheTest): self.assertTrue("mock_define_too" in self.config.parser.variables.keys()) self.assertTrue("mock_value" in self.config.parser.variables.keys()) self.assertEqual("TRUE", self.config.parser.variables["mock_value"]) - self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys()) - self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index cfbaffac7..0f2b96818 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,8 +2,6 @@ import os import unittest -import mock - from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -48,10 +46,9 @@ class MultipleVhostsTestGentoo(util.ApacheTest): config_root=config_root, vhost_root=vhost_root) - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -81,47 +78,9 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): - self.config.parser.update_runtime_variables() + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) - @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") - def test_no_binary_configdump(self, mock_subprocess): - """Make sure we don't call binary dumps other than modules from Apache - as this is not supported in Gentoo currently""" - - with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): - self.config.parser.update_runtime_variables() - self.config.parser.reset_modules() - self.assertFalse(mock_subprocess.called) - - self.config.parser.update_runtime_variables() - self.config.parser.reset_modules() - self.assertTrue(mock_subprocess.called) - - @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") - def test_opportunistic_httpd_runtime_parsing(self, mock_get): - mod_val = ( - 'Loaded Modules:\n' - ' mock_module (static)\n' - ' another_module (static)\n' - ) - def mock_get_cfg(command): - """Mock httpd process stdout""" - if command == ['apache2ctl', 'modules']: - return mod_val - mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() - - with mock.patch("certbot.util.get_os_info") as mock_osi: - # Make sure we have the have the CentOS httpd constants - mock_osi.return_value = ("gentoo", "123") - self.config.parser.update_runtime_variables() - - self.assertEquals(mock_get.call_count, 1) - self.assertEquals(len(self.config.parser.modules), 4) - self.assertTrue("mod_another.c" in self.config.parser.modules) - if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd index 4bcb300c2..0bf6b176c 100644 --- a/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd +++ b/certbot-apache/certbot_apache/tests/testdata/centos7_apache/apache/sysconfig/httpd @@ -14,7 +14,7 @@ # To pass additional options (for instance, -D definitions) to the # httpd binary at startup, set OPTIONS here. # -OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE -DMOCK_NOSEP -DNOSEP_TWO=NOSEP_VAL" +OPTIONS="-D mock_define -D mock_define_too -D mock_value=TRUE" # # This setting ensures the httpd process is started in the "C" locale diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index ca667465c..2405110c5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -5,10 +5,11 @@ import sys import unittest import augeas -import josepy as jose import mock import zope.component +from acme import jose + from certbot.display import util as display_util from certbot.plugins import common diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index 4155944bd..af951aa6a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -6,8 +6,7 @@ import re import shutil import tarfile -import josepy as jose - +from acme import jose from acme import test_util from certbot import constants diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8af474c5e..e9d4e36d4 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -182,7 +182,7 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, ", ".join(vhost.names)) + vhost.filep, vhost.names) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 7b32d8e82..6e1b0d8ff 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -5,10 +5,11 @@ import pkg_resources import tempfile import unittest -import josepy as jose import mock import zope.component +from acme import jose + from certbot import configuration from certbot.tests import util as test_util diff --git a/certbot/account.py b/certbot/account.py index 41e980097..389f96791 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -7,13 +7,13 @@ import shutil import socket from cryptography.hazmat.primitives import serialization -import josepy as jose import pyrfc3339 import pytz import six import zope.component from acme import fields as acme_fields +from acme import jose from acme import messages from certbot import errors diff --git a/certbot/achallenges.py b/certbot/achallenges.py index 6535a6b63..f39bb4cec 100644 --- a/certbot/achallenges.py +++ b/certbot/achallenges.py @@ -19,9 +19,8 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging -import josepy as jose - from acme import challenges +from acme import jose logger = logging.getLogger(__name__) diff --git a/certbot/cli.py b/certbot/cli.py index f0fa7eb7e..622462278 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1220,18 +1220,6 @@ def _create_subparsers(helpful): key=constants.REVOCATION_REASONS.get)), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") - helpful.add("revoke", - "--delete-after-revoke", action="store_true", - default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them.") - helpful.add("revoke", - "--no-delete-after-revoke", action="store_false", - dest="delete_after_revoke", - default=flag_default("delete_after_revoke"), - help="Do not delete certificates after revoking them. This " - "option should be used with caution because the 'renew' " - "subcommand will attempt to renew undeleted revoked " - "certificates.") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), diff --git a/certbot/client.py b/certbot/client.py index b735421f5..ed70fda71 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -5,13 +5,13 @@ import platform from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa -import josepy as jose import OpenSSL import zope.component from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors +from acme import jose from acme import messages import certbot diff --git a/certbot/constants.py b/certbot/constants.py index a6878824b..0ac82dafe 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -71,7 +71,6 @@ CLI_DEFAULTS = dict( user_agent_comment=None, csr=None, reason=0, - delete_after_revoke=None, rollback_checkpoints=1, init=False, prepare=False, diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 3ae16529d..112ef7c85 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -14,9 +14,9 @@ import six import zope.component from cryptography.hazmat.backends import default_backend from cryptography import x509 -import josepy as jose from acme import crypto_util as acme_crypto_util +from acme import jose from certbot import errors from certbot import interfaces @@ -368,7 +368,7 @@ def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in - :class:`josepy.util.ComparableX509`). + `acme.jose.ComparableX509`). """ # XXX: returns empty string when no chain is available, which diff --git a/certbot/main.py b/certbot/main.py index e25e030aa..72af7fbba 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -6,9 +6,9 @@ import os import sys import configobj -import josepy as jose import zope.component +from acme import jose from acme import errors as acme_errors import certbot @@ -536,11 +536,9 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b display = zope.component.getUtility(interfaces.IDisplay) reporter_util = zope.component.getUtility(interfaces.IReporter) - attempt_deletion = config.delete_after_revoke - if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked?") - attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", - force_interactive=True, default=True) + msg = ("Would you like to delete the cert(s) you just revoked?") + attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", + force_interactive=True, default=True) if not attempt_deletion: reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 002d2f225..420d15679 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -9,7 +9,7 @@ import OpenSSL import pkg_resources import zope.interface -from josepy import util as jose_util +from acme.jose import util as jose_util from certbot import constants from certbot import crypto_util diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 1a1ca7dcb..8ce68bbb5 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -5,11 +5,11 @@ import shutil import tempfile import unittest -import josepy as jose import mock import OpenSSL from acme import challenges +from acme import jose from certbot import achallenges from certbot import crypto_util diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py index 54b656b20..d8cd29404 100644 --- a/certbot/plugins/dns_test_common.py +++ b/certbot/plugins/dns_test_common.py @@ -3,10 +3,10 @@ import os import configobj -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot.tests import acme_util diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py index a221cf1bf..f9c5735e8 100644 --- a/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,7 @@ """Base test class for DNS authenticators built on Lexicon.""" -import josepy as jose import mock +from acme import jose from requests.exceptions import HTTPError, RequestException from certbot import errors diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 5227bc59e..1ae731e42 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -3,11 +3,11 @@ import argparse import socket import unittest -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 36e2ffba6..92160bdfa 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -10,11 +10,11 @@ import stat import tempfile import unittest -import josepy as jose import mock import six from acme import challenges +from acme import jose from certbot import achallenges from certbot import errors diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 8ebda56af..7245ad6a1 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -6,10 +6,10 @@ import shutil import stat import unittest -import josepy as jose import mock import pytz +from acme import jose from acme import messages from certbot import errors diff --git a/certbot/tests/acme_util.py b/certbot/tests/acme_util.py index 53a2f214a..f0549666a 100644 --- a/certbot/tests/acme_util.py +++ b/certbot/tests/acme_util.py @@ -1,10 +1,10 @@ """ACME utilities for testing.""" import datetime -import josepy as jose import six from acme import challenges +from acme import jose from acme import messages from certbot import auth_handler diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index c5935d722..2fce412e2 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -164,8 +164,6 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue("--cert-path" in out) self.assertTrue("--key-path" in out) self.assertTrue("--reason" in out) - self.assertTrue("--delete-after-revoke" in out) - self.assertTrue("--no-delete-after-revoke" in out) out = self._help_output(['-h', 'config_changes']) self.assertTrue("--cert-path" not in out) @@ -414,18 +412,6 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods def test_no_directory_hooks_unset(self): self.assertTrue(self.parse([]).directory_hooks) - def test_delete_after_revoke(self): - namespace = self.parse(["--delete-after-revoke"]) - self.assertTrue(namespace.delete_after_revoke) - - def test_delete_after_revoke_default(self): - namespace = self.parse([]) - self.assertEqual(namespace.delete_after_revoke, None) - - def test_no_delete_after_revoke(self): - namespace = self.parse(["--no-delete-after-revoke"]) - self.assertFalse(namespace.delete_after_revoke) - class DefaultTest(unittest.TestCase): """Tests for certbot.cli._Default.""" diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 204f46323..09c4a50ca 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -4,11 +4,11 @@ import shutil import tempfile import unittest -import josepy as jose import OpenSSL import mock from acme import errors as acme_errors +from acme import jose from certbot import account from certbot import errors diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 57d82f839..cb0fb32e3 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,10 +4,10 @@ import os import sys import unittest -import josepy as jose import mock import zope.component +from acme import jose from acme import messages from certbot import account diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index b1d58542f..1f690df26 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,10 +11,11 @@ import unittest import datetime import pytz -import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme import jose + from certbot import account from certbot import cli from certbot import constants @@ -298,29 +299,25 @@ class RevokeTest(test_util.TempDirTestCase): self._call() self.assertFalse(mock_delete.called) -class DeleteIfAppropriateTest(test_util.ConfigTestCase): +class DeleteIfAppropriateTest(unittest.TestCase): """Tests for certbot.main._delete_if_appropriate """ + def setUp(self): + self.config = mock.Mock() + self.config.namespace = mock.Mock() + self.config.namespace.noninteractive_mode = False + def _call(self, mock_config): from certbot.main import _delete_if_appropriate _delete_if_appropriate(mock_config) - def _test_delete_opt_out_common(self, mock_get_utility): - with mock.patch('certbot.cert_manager.delete') as mock_delete: - self._call(self.config) - mock_delete.assert_not_called() - self.assertTrue(mock_get_utility().add_message.called) - + @mock.patch('certbot.cert_manager.delete') @test_util.patch_get_utility() - def test_delete_flag_opt_out(self, mock_get_utility): - self.config.delete_after_revoke = False - self._test_delete_opt_out_common(mock_get_utility) - - @test_util.patch_get_utility() - def test_delete_prompt_opt_out(self, mock_get_utility): + def test_delete_opt_out(self, mock_get_utility, mock_delete): util_mock = mock_get_utility() util_mock.yesno.return_value = False - self._test_delete_opt_out_common(mock_get_utility) + self._call(self.config) + mock_delete.assert_not_called() # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @@ -401,28 +398,6 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase): self._call(config) self.assertEqual(mock_delete.call_count, 1) - # pylint: disable=too-many-arguments - @mock.patch('certbot.storage.renewal_file_for_certname') - @mock.patch('certbot.cert_manager.match_and_check_overlaps') - @mock.patch('certbot.storage.full_archive_path') - @mock.patch('certbot.cert_manager.cert_path_to_lineage') - @mock.patch('certbot.cert_manager.delete') - @test_util.patch_get_utility() - def test_opt_in_deletion(self, mock_get_utility, mock_delete, - mock_cert_path_to_lineage, mock_full_archive_dir, - mock_match_and_check_overlaps, mock_renewal_file_for_certname): - # pylint: disable = unused-argument - config = self.config - config.namespace.delete_after_revoke = True - config.cert_path = "/some/reasonable/path" - config.certname = "" - mock_cert_path_to_lineage.return_value = "example.com" - mock_full_archive_dir.return_value = "" - mock_match_and_check_overlaps.return_value = "" - self._call(config) - self.assertEqual(mock_delete.call_count, 1) - self.assertFalse(mock_get_utility().yesno.called) - # pylint: disable=too-many-arguments @mock.patch('certbot.storage.renewal_file_for_certname') @mock.patch('certbot.cert_manager.match_and_check_overlaps') diff --git a/certbot/tests/util.py b/certbot/tests/util.py index ddd4a1aec..c43b44522 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -14,10 +14,11 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import mock import OpenSSL -import josepy as jose import six from six.moves import reload_module # pylint: disable=import-error +from acme import jose + from certbot import constants from certbot import interfaces from certbot import storage diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6 index 47eb48f50..8c1a4b353 100644 --- a/letsencrypt-auto-source/Dockerfile.centos6 +++ b/letsencrypt-auto-source/Dockerfile.centos6 @@ -33,5 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source USER lea WORKDIR /home/lea -RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh -CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh +CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index f1361d8ea..8d2e8a6b6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -68,12 +68,10 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; + --noninteractive|--non-interactive|renew) + ASSUME_YES=1;; --quiet) QUIET=1;; - renew) - ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -95,7 +93,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 + ASSUME_YES=1 HELP=0 fi @@ -246,29 +244,15 @@ DeprecationBootstrap() { fi } -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version + DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - if [ -n "$USE_PYTHON_3" ]; then - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi + error "Cannot find any Pythons; please install one!" + exit 1 fi export LE_PYTHON @@ -400,19 +384,23 @@ BootstrapDebCommon() { fi } -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. +# If new packages are installed by BootstrapRpmCommon below, this version +# number must be increased. +BOOTSTRAP_RPM_COMMON_VERSION=1 + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Enables EPEL if applicable and possible. -InitializeRPMCommonBase() { if type dnf 2>/dev/null then - TOOL=dnf + tool=dnf elif type yum 2>/dev/null then - TOOL=yum + tool=yum else error "Neither yum nor dnf found. Aborting bootstrap!" @@ -420,15 +408,15 @@ InitializeRPMCommonBase() { fi if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" + yes_flag="-y" fi if [ "$QUIET" = 1 ]; then QUIET_FLAG='--quiet' fi - if ! $TOOL list *virtualenv >/dev/null 2>&1; then + if ! $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $TOOL list epel-release >/dev/null 2>&1; then + if ! $tool list epel-release >/dev/null 2>&1; then error "Enable the EPEL repository and try running Certbot again." exit 1 fi @@ -440,17 +428,11 @@ InitializeRPMCommonBase() { /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." sleep 1s fi - if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then + if ! $tool install $yes_flag $QUIET_FLAG epel-release; then error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice pkgs=" gcc @@ -462,39 +444,10 @@ BootstrapRpmCommonBase() { ca-certificates " - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} - -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 - - InitializeRPMCommonBase - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python + if $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python python-devel python-virtualenv python-tools @@ -502,8 +455,9 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 + elif $tool list python2 >/dev/null 2>&1; then + pkgs="$pkgs + python2 python2-libs python2-setuptools python2-devel @@ -514,7 +468,8 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - python_pkgs="$python27 + pkgs="$pkgs + python27 python27-devel python27-virtualenv python27-tools @@ -522,31 +477,16 @@ BootstrapRpmCommon() { " fi - BootstrapRpmCommonBase "$python_pkgs" -} - -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then - python_pkgs="python34 - python34-devel - python34-tools + if $tool list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 fi - BootstrapRpmCommonBase "$python_pkgs" + if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi } # If new packages are installed by BootstrapSuseCommon below, this version @@ -775,24 +715,11 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi - export LE_PYTHON="$prev_le_python" + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -931,18 +858,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then @@ -1064,16 +983,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1150,6 +1062,10 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # @@ -1437,22 +1353,17 @@ On failure, return non-zero. """ -from __future__ import print_function, unicode_literals +from __future__ import print_function from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re -import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError +from urllib2 import build_opener, HTTPHandler, HTTPSHandler +from urllib2 import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -1474,11 +1385,8 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) - else: - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -1500,7 +1408,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: + with open(join(dir, filename), 'w') as file: file.write(contents) @@ -1508,13 +1416,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].iterkeys() if re.match('^[0-9.]+$', r))) @@ -1531,7 +1439,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -1546,14 +1454,6 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f4c1b202f..4eef10c80 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -68,12 +68,10 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; + --noninteractive|--non-interactive|renew) + ASSUME_YES=1;; --quiet) QUIET=1;; - renew) - ASSUME_YES=1;; --verbose) VERBOSE=1;; -[!-]*) @@ -95,7 +93,7 @@ done if [ $BASENAME = "letsencrypt-auto" ]; then # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 + ASSUME_YES=1 HELP=0 fi @@ -246,29 +244,15 @@ DeprecationBootstrap() { fi } -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version + DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - if [ -n "$USE_PYTHON_3" ]; then - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi + error "Cannot find any Pythons; please install one!" + exit 1 fi export LE_PYTHON @@ -281,9 +265,7 @@ DeterminePythonVersion() { } {{ bootstrappers/deb_common.sh }} -{{ bootstrappers/rpm_common_base.sh }} {{ bootstrappers/rpm_common.sh }} -{{ bootstrappers/rpm_python3.sh }} {{ bootstrappers/suse_common.sh }} {{ bootstrappers/arch_common.sh }} {{ bootstrappers/gentoo_common.sh }} @@ -314,24 +296,11 @@ elif [ -f /etc/mageia-release ]; then } BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION" elif [ -f /etc/redhat-release ]; then - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then - Bootstrap() { - BootstrapMessage "RedHat-based OSes that will use Python3" - BootstrapRpmPython3 - } - USE_PYTHON_3=1 - BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION" - else - Bootstrap() { - BootstrapMessage "RedHat-based OSes" - BootstrapRpmCommon - } - BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" - fi - export LE_PYTHON="$prev_le_python" + Bootstrap() { + BootstrapMessage "RedHat-based OSes" + BootstrapRpmCommon + } + BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION" elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then Bootstrap() { BootstrapMessage "openSUSE-based OSes" @@ -470,18 +439,10 @@ if [ "$1" = "--le-auto-phase2" ]; then say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" - if [ "$PYVER" -le 27 ]; then - if [ "$VERBOSE" = 1 ]; then - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" - else - virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null - fi + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" else - if [ "$VERBOSE" = 1 ]; then - "$LE_PYTHON" -m venv "$VENV_PATH" - else - "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null - fi + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi if [ -n "$BOOTSTRAP_VERSION" ]; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 80d55a393..5b120a9e6 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -7,13 +7,61 @@ BootstrapRpmCommon() { # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 + # - CentOS 6 (EPEL must be installed manually) - InitializeRPMCommonBase + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + error "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + if [ "$QUIET" = 1 ]; then + QUIET_FLAG='--quiet' + fi + + if ! $tool list *virtualenv >/dev/null 2>&1; then + echo "To use Certbot, packages from the EPEL repository need to be installed." + if ! $tool list epel-release >/dev/null 2>&1; then + error "Enable the EPEL repository and try running Certbot again." + exit 1 + fi + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Enabling the EPEL repository in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." + sleep 1s + fi + if ! $tool install $yes_flag $QUIET_FLAG epel-release; then + error "Could not enable EPEL. Aborting bootstrap!" + exit 1 + fi + fi + + pkgs=" + gcc + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python + if $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python python-devel python-virtualenv python-tools @@ -21,8 +69,9 @@ BootstrapRpmCommon() { " # Fedora 26 starts to use the prefix python2 for python2 based packages. # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 + elif $tool list python2 >/dev/null 2>&1; then + pkgs="$pkgs + python2 python2-libs python2-setuptools python2-devel @@ -33,7 +82,8 @@ BootstrapRpmCommon() { # Some distros and older versions of current distros use a "python27" # instead of the "python" or "python-" naming convention. else - python_pkgs="$python27 + pkgs="$pkgs + python27 python27-devel python27-virtualenv python27-tools @@ -41,5 +91,14 @@ BootstrapRpmCommon() { " fi - BootstrapRpmCommonBase "$python_pkgs" + if $tool list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh deleted file mode 100644 index d7a9f3133..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ /dev/null @@ -1,78 +0,0 @@ -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. - -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Enables EPEL if applicable and possible. -InitializeRPMCommonBase() { - if type dnf 2>/dev/null - then - TOOL=dnf - elif type yum 2>/dev/null - then - TOOL=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! $TOOL list *virtualenv >/dev/null 2>&1; then - echo "To use Certbot, packages from the EPEL repository need to be installed." - if ! $TOOL list epel-release >/dev/null 2>&1; then - error "Enable the EPEL repository and try running Certbot again." - exit 1 - fi - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Enabling the EPEL repository in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..." - sleep 1s - fi - if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then - error "Could not enable EPEL. Aborting bootstrap!" - exit 1 - fi - fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " - - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh deleted file mode 100644 index b011a7235..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ /dev/null @@ -1,23 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then - python_pkgs="python34 - python34-devel - python34-tools - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 0e2cec984..dec7ae7d0 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -105,16 +105,9 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 -josepy==1.0.1 \ - --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ - --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -191,3 +184,7 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index ae72a299b..8f34351c9 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -11,22 +11,17 @@ On failure, return non-zero. """ -from __future__ import print_function, unicode_literals +from __future__ import print_function from distutils.version import LooseVersion from json import loads from os import devnull, environ from os.path import dirname, join import re -import ssl from subprocess import check_call, CalledProcessError from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError +from urllib2 import build_opener, HTTPHandler, HTTPSHandler +from urllib2 import HTTPError, URLError PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq @@ -48,11 +43,8 @@ class HttpsGetter(object): def __init__(self): """Build an HTTPS opener.""" # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context())) - else: - self._opener = build_opener(HTTPSHandler()) + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) # Strip out HTTPHandler to prevent MITM spoof: for handler in self._opener.handlers: if isinstance(handler, HTTPHandler): @@ -74,7 +66,7 @@ class HttpsGetter(object): def write(contents, dir, filename): """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: + with open(join(dir, filename), 'w') as file: file.write(contents) @@ -82,13 +74,13 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most # packages, LE included. return str(max(LooseVersion(r) for r - in iter(metadata['releases'].keys()) + in metadata['releases'].iterkeys() if re.match('^[0-9.]+$', r))) @@ -105,7 +97,7 @@ def verified_new_le_auto(get, tag, temp_dir): 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') try: with open(devnull, 'w') as dev_null: check_call(['openssl', 'dgst', '-sha256', '-verify', @@ -120,14 +112,6 @@ def verified_new_le_auto(get, tag, temp_dir): "certbot-auto.", exc) -def create_CERT_NONE_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - def main(): get = HttpsGetter().get flag = argv[1] diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index d187452a1..2fa03105d 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -30,10 +30,6 @@ sys.path.insert(0, dirname(tests_dir())) from build import build as build_le_auto -BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' -"""Name of the file where certbot-auto saves its bootstrap version.""" - - class RequestHandler(BaseHTTPRequestHandler): """An HTTPS request handler which is quiet and serves a specific folder.""" @@ -202,7 +198,6 @@ LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 iQIDAQAB -----END PUBLIC KEY-----""", - NO_CERT_VERIFY='1', **kwargs) env.update(d) return out_and_err( @@ -301,31 +296,17 @@ class AutoTests(TestCase): def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') - with temp_paths() as (le_auto_path, venv_dir): - install_le_auto(self.NEW_LE_AUTO, le_auto_path) - - # Create venv saving the correct bootstrap script version - out, err = run_le_auto(le_auto_path, venv_dir, base_url, - PIP_FIND_LINKS=pip_find_links) - self.assertFalse('Upgrading certbot-auto ' in out) - self.assertTrue('Creating virtual environment...' in out) - with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: - bootstrap_version = f.read() - - # Create a new venv with an old letsencrypt version - with temp_paths() as (le_auto_path, venv_dir): + with temp_paths() as (le_auto_path, venv_dir): + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} + with serving(resources) as base_url: venv_bin = join(venv_dir, 'bin') makedirs(venv_bin) set_le_script_version(venv_dir, '0.0.1') - with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: - f.write(bootstrap_version) install_le_auto(self.NEW_LE_AUTO, le_auto_path) + pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) @@ -350,7 +331,6 @@ class AutoTests(TestCase): self.assertTrue("Couldn't verify signature of downloaded " "certbot-auto." in exc.output) else: - print(out) self.fail('Signature check on certbot-auto erroneously passed.') def test_pip_failure(self): diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh deleted file mode 100644 index e3ebbaec5..000000000 --- a/letsencrypt-auto-source/tests/centos6_tests.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Start by making sure your system is up-to-date: -yum update > /dev/null -yum install -y centos-release-scl > /dev/null -yum install -y python27 > /dev/null 2> /dev/null - -# we're going to modify env variables, so do this in a subshell -( -source /opt/rh/python27/enable - -# ensure python 3 isn't installed -python3 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "Python3 is already installed." - exit 1 -fi - -# ensure python2.7 is available -python2.7 --version 2> /dev/null -RESULT=$? -if [ $RESULT -ne 0 ]; then - error "Python3 is not available." - exit 1 -fi - -# bootstrap, but don't install python 3. -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null - -# ensure python 3 isn't installed -python3 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "letsencrypt-auto installed Python3 even though Python2.7 is present." - exit 1 -fi - -echo "" -echo "PASSED: Did not upgrade to Python3 when Python2.7 is present." -) - -# ensure python2.7 isn't available -python2.7 --version 2> /dev/null -RESULT=$? -if [ $RESULT -eq 0 ]; then - error "Python2.7 is still available." - exit 1 -fi - -# bootstrap, this time installing python3 -certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null - -# ensure python 3 is installed -python3 --version > /dev/null -RESULT=$? -if [ $RESULT -ne 0 ]; then - error "letsencrypt-auto failed to install Python3 when only Python2.6 is present." - exit 1 -fi - -echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present." -echo "" - -# test using python3 -pytest -v -s certbot/letsencrypt-auto-source/tests diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b64550cb7..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --quiet diff --git a/setup.py b/setup.py index ce505a62e..ee108c514 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,10 @@ readme = read_file(os.path.join(here, 'README.rst')) changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] -# This package relies on PyOpenSSL, requests, and six, however, it isn't -# specified here to avoid masking the more specific request requirements in -# acme. See https://github.com/pypa/pip/issues/988 for more info. +# Please update tox.ini when modifying dependency version requirements +# This package relies on requests, however, it isn't specified here to avoid +# masking the more specific request requirements in acme. See +# https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but @@ -43,11 +44,13 @@ install_requires = [ 'cryptography>=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT + 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', + 'six', 'zope.component', 'zope.interface', ] diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index e1aad4336..1e0b7754b 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -345,14 +345,9 @@ common auth --must-staple --domains "must-staple.le.wtf" openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' # revoke by account key -common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke +common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" # revoke renewed -common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke -if [ ! -d "$root/conf/live/le1.wtf" ]; then - echo "cert deleted when --no-delete-after-revoke was used!" - exit 1 -fi -common delete --cert-name le1.wtf +common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" # revoke by cert key common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ --key-path "$root/conf/live/le2.wtf/privkey.pem" diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index a83cbd826..cb659786e 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -15,56 +15,19 @@ if ! command -v git ; then exit 1 fi fi +BRANCH=`git rev-parse --abbrev-ref HEAD` # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's # the first version that pins package versions, properly supports # --no-self-upgrade, and works with newer versions of pip. -git checkout -f v0.5.0 letsencrypt-auto +git checkout -f v0.5.0 if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then echo initial installation appeared to fail exit 1 fi -# Now that python and openssl have been installed, we can set up a fake server -# to provide a new version of letsencrypt-auto. First, we start the server and -# directory to be served. -MY_TEMP_DIR=$(mktemp -d) -PORT_FILE="$MY_TEMP_DIR/port" -SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py) -cd "$MY_TEMP_DIR" -"$SERVER_PATH" 0 > $PORT_FILE & -SERVER_PID=$! -trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT -cd ~- - -# Then, we set up the files to be served. -FAKE_VERSION_NUM="99.99.99" -echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" -LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" -NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -mkdir "$LE_AUTO_SOURCE_DIR" -cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" -openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" - -# Next, we wait for the server to start and get the port number. -sleep 5s -SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") - -# Finally, we set the necessary certbot-auto environment variables. -export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" -export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" -export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY----- -" - -if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then +git checkout -f "$BRANCH" +EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then echo upgrade appeared to fail exit 1 fi diff --git a/tools/deactivate.py b/tools/deactivate.py index d43b84552..5facc8436 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -18,10 +18,10 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization -import josepy as jose from acme import client as acme_client from acme import errors as acme_errors +from acme import jose from acme import messages DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 0d39e0594..d57f0974e 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,9 +2,8 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. +# set to 1, packages are installed using certbot-auto's requirements file as +# constraints. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" @@ -23,5 +22,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - "$(dirname $0)/pytest.sh" --pyargs $pkg + pytest --numprocesses auto --quiet --pyargs $pkg done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py deleted file mode 100755 index c8fb95351..000000000 --- a/tools/merge_requirements.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -"""Merges multiple Python requirements files into one file. - -Requirements files specified later take precedence over earlier ones. Only -simple SomeProject==1.2.3 format is currently supported. - -""" - -from __future__ import print_function - -import sys - - -def read_file(file_path): - """Reads in a Python requirements file. - - :param str file_path: path to requirements file - - :returns: mapping from a project to its pinned version - :rtype: dict - - """ - d = {} - with open(file_path) as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d - - -def print_requirements(requirements): - """Prints requirements to stdout. - - :param dict requirements: mapping from a project to its pinned version - - """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) - - -def merge_requirements_files(*files): - """Merges multiple requirements files together and prints the result. - - Requirement files specified later in the list take precedence over earlier - files. - - :param tuple files: paths to requirements files - - """ - d = {} - for f in files: - d.update(read_file(f)) - print_requirements(d) - - -if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt deleted file mode 100644 index de2b83ad8..000000000 --- a/tools/oldest_constraints.txt +++ /dev/null @@ -1,51 +0,0 @@ -# This file contains the oldest versions of our dependencies we say we require -# in our packages or versions we need to support to maintain compatibility with -# the versions included in the various Linux distros where we are packaged. - -# CentOS/RHEL 7 EPEL constraints -cffi==1.6.0 -chardet==2.2.1 -configobj==4.7.2 -ipaddress==1.0.16 -mock==1.0.1 -ndg-httpsclient==0.3.2 -ply==3.4 -pyasn1==0.1.9 -pycparser==2.14 -pyOpenSSL==0.13.1 -pyparsing==1.5.6 -pyRFC3339==1.0 -python-augeas==0.5.0 -six==1.9.0 -# setuptools 0.9.8 is the actual version packaged, but some other dependencies -# in this file require setuptools>=1.0 and there are no relevant changes for us -# between these versions. -setuptools==1.0.0 -urllib3==1.10.2 -zope.component==4.1.0 -zope.event==4.0.3 -zope.interface==4.0.5 - -# Debian Jessie Backports constraints -PyICU==1.8 -colorama==0.3.2 -enum34==1.0.3 -html5lib==0.999 -idna==2.0 -pbr==1.8.0 -pytz==2012rc0 - -# Our setup.py constraints -cloudflare==1.5.1 -cryptography==1.2.0 -google-api-python-client==1.5 -oauth2client==2.0 -parsedatetime==1.3 -pyparsing==1.5.5 -python-digitalocean==1.11 -requests[security]==2.4.1 - -# Ubuntu Xenial constraints -ConfigArgParse==0.10.0 -funcsigs==0.4 -zope.hookable==4.0.4 diff --git a/tools/dev_constraints.txt b/tools/pip_constraints.txt similarity index 71% rename from tools/dev_constraints.txt rename to tools/pip_constraints.txt index afc362ff8..cacec37d6 100644 --- a/tools/dev_constraints.txt +++ b/tools/pip_constraints.txt @@ -1,15 +1,16 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# letsencrypt-auto's requirements file. We should avoid listing packages in +# both places because if both files are used as constraints for the same pip +# invocation, some constraints may be ignored due to pip's lack of dependency +# resolution. alabaster==0.7.10 apipkg==1.4 -asn1crypto==0.22.0 astroid==1.3.5 -attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.5.1 +cloudflare==1.8.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -18,7 +19,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.5 +google-api-python-client==1.6.4 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -26,22 +27,20 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 -josepy==1.0.1 -logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -ndg-httpsclient==0.3.2 -oauth2client==2.0.0 +oauth2client==4.1.2 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 +pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.1.9 -pyasn1-modules==0.0.10 +pyasn1==0.3.7 +pyasn1-modules==0.1.5 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -49,7 +48,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.11 +python-digitalocean==1.12 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -66,6 +65,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==0.6 +uritemplate==3.0.0 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index d2aae4a43..fafd58e54 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,26 +1,17 @@ -#!/bin/bash -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's -# requirements file and tools/dev_constraints.txt is used. The other file -# always takes precedence over tools/dev_constraints.txt. +#!/bin/sh -e +# pip installs packages using pinned package versions # get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -dev_constraints="$tools_dir/dev_constraints.txt" -merge_reqs="$tools_dir/merge_requirements.py" -test_constraints=$(mktemp) -trap "rm -f $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi +my_path=$("$(dirname $0)/readlink.py" $0) +repo_root=$(dirname $(dirname $my_path)) +requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" +certbot_auto_constraints=$(mktemp) +trap "rm -f $certbot_auto_constraints" EXIT +# extract pinned requirements without hashes +sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints +dev_constraints="$(dirname $my_path)/pip_constraints.txt" set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" +pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" diff --git a/tools/pytest.sh b/tools/pytest.sh deleted file mode 100755 index 8e3619d5d..000000000 --- a/tools/pytest.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Runs pytest with the provided arguments, adding --numprocesses to the command -# line. This argument is set to "auto" if the environmnent variable TRAVIS is -# not set, otherwise, it is set to 2. This works around -# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis -# environnment provides two cores. See -# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. - -if ${TRAVIS:-false}; then - NUMPROCESSES="2" -else - NUMPROCESSES="auto" -fi - -pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py deleted file mode 100755 index 26bf231b7..000000000 --- a/tools/simple_http_server.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" -from BaseHTTPServer import HTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler -import sys - -def serve_forever(port=0): - """Spins up an HTTP server on all interfaces and the given port. - - A message is printed to stdout specifying the address and port being used - by the server. - - :param int port: port number to use. - - """ - server = HTTPServer(('', port), SimpleHTTPRequestHandler) - print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address) - sys.stdout.flush() - server.serve_forever() - - -if __name__ == '__main__': - kwargs = {} - if len(sys.argv) > 1: - kwargs['port'] = int(sys.argv[1]) - serve_forever(**kwargs) diff --git a/tox.cover.sh b/tox.cover.sh index bc0e5a8bf..2b5a3cf19 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,8 +51,7 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest="$(dirname $0)/tools/pytest.sh" - "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" + pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 20f5cda32..bb421daa5 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,9 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. All dependencies are pinned -# to a specific version for increased stability for developers. +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using certbot-auto's requirements file as +# constraints. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -61,7 +62,6 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 -passenv = TRAVIS [testenv] commands = @@ -70,25 +70,48 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 -passenv = - {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 -passenv = - {[testenv]passenv} [testenv:py27-oldest] commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_OLDEST=1 -passenv = - {[testenv]passenv} + CERTBOT_NO_PIN=1 +deps = + PyOpenSSL==0.13 + cffi==1.5.2 + configargparse==0.10.0 + configargparse==0.10.0 + configobj==4.7.2 + cryptography==1.2.3 + enum34==0.9.23 + google-api-python-client==1.5 + idna==2.0 + ipaddress==1.0.16 + mock==1.0.1 + ndg-httpsclient==0.3.2 + oauth2client==2.0 + parsedatetime==1.4 + pyasn1-modules==0.0.5 + pyasn1==0.1.9 + pyparsing==1.5.6 + pyrfc3339==1.0 + pytest==3.2.5 + python-augeas==0.4.1 + pytz==2012c + requests[security]==2.6.0 + setuptools==0.9.8 + six==1.9.0 + urllib3==1.10 + zope.component==4.0.2 + zope.event==4.0.1 + zope.interface==4.0.5 [testenv:py27_install] basepython = python2.7 @@ -100,8 +123,6 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh -passenv = - {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187)