diff --git a/.gitignore b/.gitignore index 1becea3b4..341843f98 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ letsencrypt.log # letstest tests/letstest/letest-*/ tests/letstest/*.pem +tests/letstest/venv/ diff --git a/.travis.yml b/.travis.yml index d9b4cb5ea..0e4c8c0f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python +cache: + directories: + - $HOME/.cache/pip + services: - rabbitmq - mariadb @@ -19,23 +23,35 @@ env: global: - GOPATH=/tmp/go - PATH=$GOPATH/bin:$PATH - matrix: - - TOXENV=py26 BOULDER_INTEGRATION=1 - - TOXENV=py27 BOULDER_INTEGRATION=1 - - TOXENV=py26-oldest BOULDER_INTEGRATION=1 - - TOXENV=py27-oldest BOULDER_INTEGRATION=1 - - TOXENV=py33 - - TOXENV=py34 - - TOXENV=lint - - TOXENV=cover -# Disabled for now due to requiring sudo -> causing more boulder integration -# DNS timeouts :( -# - TOXENV=apacheconftest + matrix: include: - - env: TOXENV=py35 - python: 3.5 - + - python: "2.6" + env: TOXENV=py26 BOULDER_INTEGRATION=1 + - python: "2.6" + env: TOXENV=py26-oldest BOULDER_INTEGRATION=1 +# Disabled for now due to requiring sudo -> causing more boulder integration +# DNS timeouts :( +# - python: "2.7" +# env: TOXENV=apacheconftest + - python: "2.7" + env: TOXENV=py27 BOULDER_INTEGRATION=1 + - python: "2.7" + env: TOXENV=py27-oldest BOULDER_INTEGRATION=1 + - python: "2.7" + env: TOXENV=lint + - sudo: required + env: TOXENV=le_auto + services: docker + before_install: + - python: "2.7" + env: TOXENV=cover + - python: "3.3" + env: TOXENV=py33 + - python: "3.4" + env: TOXENV=py34 + - python: "3.5" + env: TOXENV=py35 # Only build pushes to the master branch, PRs, and branches beginning with # `test-`. This reduces the number of simultaneous Travis runs, which speeds @@ -49,15 +65,19 @@ branches: sudo: false addons: - # make sure simplehttp simple verification works (custom /etc/hosts) + # Custom /etc/hosts required for simple verification of http-01 + # and tls-sni-01, and for letsencrypt_test_nginx hosts: - le.wtf + - le1.wtf + - le2.wtf + - le3.wtf + - nginx.wtf mariadb: "10.0" apt: sources: - augeas - packages: # keep in sync with bootstrap/ubuntu.sh and Boulder - - python + packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder. - python-dev - python-virtualenv - gcc diff --git a/CHANGES.rst b/CHANGES.rst index 3ed13041b..55e4bd796 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,23 +5,7 @@ Please note: the change log will only get updated after first release - for now please use the `commit log `_. +To see the changes in a given release, inspect the github milestone for the +release. For instance: -Release 0.1.0 (not released yet) --------------------------------- - -New Features: - -* ... - -Fixes: - -* ... - -Other changes: - -* ... - -Release 0.0.0 (not released yet) --------------------------------- - -Initial release. +https://github.com/letsencrypt/letsencrypt/issues?utf8=%E2%9C%93&q=milestone%3A0.3.0 diff --git a/Dockerfile b/Dockerfile index da0110604..71e217659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # directories in its path. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ diff --git a/Dockerfile-dev b/Dockerfile-dev index 3c5b53966..56e2ec05b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt # TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. -COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh -RUN /opt/letsencrypt/src/ubuntu.sh && \ +COPY letsencrypt-auto-source/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/letsencrypt/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ @@ -58,7 +58,7 @@ RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \ -e /opt/letsencrypt/src/letsencrypt-nginx \ -e /opt/letsencrypt/src/letshelp-letsencrypt \ -e /opt/letsencrypt/src/letsencrypt-compatibility-test \ - -e /opt/letsencrypt/src[dev,docs,testing] + -e /opt/letsencrypt/src[dev,docs] # install in editable mode (-e) to save space: it's not possible to # "rm -rf /opt/letsencrypt/src" (it's stays in the underlaying image); diff --git a/README.rst b/README.rst index 57908e90f..522400a1f 100644 --- a/README.rst +++ b/README.rst @@ -51,11 +51,11 @@ client will guide you through the process of obtaining and installing certs interactively. You can also tell it exactly what you want it to do from the command line. -For instance, if you want to obtain a cert for ``thing.com``, -``www.thing.com``, and ``otherthing.net``, using the Apache plugin to both +For instance, if you want to obtain a cert for ``example.com``, +``www.example.com``, and ``other.example.net``, using the Apache plugin to both obtain and install the certs, you could do this:: - ./letsencrypt-auto --apache -d thing.com -d www.thing.com -d otherthing.net + ./letsencrypt-auto --apache -d example.com -d www.example.com -d other.example.net (The first time you run the command, it will make an account, and ask for an email and agreement to the Let's Encrypt Subscriber Agreement; you can @@ -64,7 +64,7 @@ automate those with ``--email`` and ``--agree-tos``) If you want to use a webserver that doesn't have full plugin support yet, you can still use "standalone" or "webroot" plugins to obtain a certificate:: - ./letsencrypt-auto certonly --standalone --email admin@thing.com -d thing.com -d www.thing.com -d otherthing.net + ./letsencrypt-auto certonly --standalone --email admin@example.com -d example.com -d www.example.com -d other.example.net Understanding the client in more depth diff --git a/Vagrantfile b/Vagrantfile index a2759440c..e5975442f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,10 +5,19 @@ VAGRANTFILE_API_VERSION = "2" # Setup instructions from docs/contributing.rst +# Script installs dependencies for tox and boulder integration $ubuntu_setup_script = <> /home/vagrant/.profile; fi +if ! grep -Fxq "export PATH=\\$GOROOT/bin:\\$PATH" /home/vagrant/.profile ; then echo "export PATH=\\$GOROOT/bin:\\$PATH" >> /home/vagrant/.profile; fi +if ! grep -Fxq "export GOPATH=\\$HOME/go" /home/vagrant/.profile ; then echo "export GOPATH=\\$HOME/go" >> /home/vagrant/.profile; fi +if ! grep -Fxq "cd /vagrant/; ./tests/boulder-start.sh &" /etc/rc.local ; then sed -i -e '$i \cd /vagrant/; ./tests/boulder-start.sh &\n' /etc/rc.local; fi +export DEBIAN_FRONTEND=noninteractive +sudo -E apt-get -q -y install git make libltdl-dev mariadb-server rabbitmq-server nginx-light SETUP_SCRIPT Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| @@ -21,6 +30,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Cannot allocate memory" when running # letsencrypt.client.tests.display.util_test.NcursesDisplayTest v.memory = 1024 + + # Handle cases when the host is behind a private network by making the + # NAT engine use the host's resolver mechanisms to handle DNS requests. + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] end end diff --git a/acme/.pylintrc b/acme/.pylintrc index 33650310d..d0d150631 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -100,7 +100,7 @@ comment=no [FORMAT] # Maximum number of characters on a single line. -max-line-length=100 +max-line-length=80 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 68bf3fce4..13d19d3c4 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -236,10 +236,8 @@ class HTTP01Response(KeyAuthorizationChallengeResponse): :param challenges.SimpleHTTP chall: Corresponding challenge. :param unicode domain: Domain name being verified. - :param account_public_key: Public key for the key pair - being authorized. If ``None`` key verification is not - performed! - :param JWK account_public_key: + :param JWK account_public_key: Public key for the key pair + being authorized. :param int port: Port used in the validation. :returns: ``True`` iff validation is successful, ``False`` diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 4f2d06167..ef78e1eba 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -73,7 +73,8 @@ class KeyAuthorizationChallengeResponseTest(unittest.TestCase): def test_verify_wrong_form(self): from acme.challenges import KeyAuthorizationChallengeResponse response = KeyAuthorizationChallengeResponse( - key_authorization='.foo.oKGqedy-b-acd5eoybm2f-NVFxvyOoET5CNy3xnv8WY') + key_authorization='.foo.oKGqedy-b-acd5eoybm2f-' + 'NVFxvyOoET5CNy3xnv8WY') self.assertFalse(response.verify(self.chall, KEY.public_key())) @@ -273,10 +274,12 @@ class TLSSNI01ResponseTest(unittest.TestCase): @mock.patch('acme.challenges.TLSSNI01Response.verify_cert', autospec=True) def test_simple_verify(self, mock_verify_cert): mock_verify_cert.return_value = mock.sentinel.verification - self.assertEqual(mock.sentinel.verification, self.response.simple_verify( - self.chall, self.domain, KEY.public_key(), - cert=mock.sentinel.cert)) - mock_verify_cert.assert_called_once_with(self.response, mock.sentinel.cert) + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, mock.sentinel.cert) @mock.patch('acme.challenges.TLSSNI01Response.probe_cert') def test_simple_verify_false_on_probe_error(self, mock_probe_cert): @@ -590,7 +593,8 @@ class DNSTest(unittest.TestCase): def test_check_validation_wrong_fields(self): bad_validation = jose.JWS.sign( - payload=self.msg.update(token=b'x' * 20).json_dumps().encode('utf-8'), + payload=self.msg.update( + token=b'x' * 20).json_dumps().encode('utf-8'), alg=jose.RS256, key=KEY) self.assertFalse(self.msg.check_validation( bad_validation, KEY.public_key())) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3e28ef47..0c0d75b05 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -1,4 +1,5 @@ """ACME client API.""" +import collections import datetime import heapq import logging @@ -66,15 +67,13 @@ class Client(object): # pylint: disable=too-many-instance-attributes @classmethod def _regr_from_response(cls, response, uri=None, new_authzr_uri=None, terms_of_service=None): - terms_of_service = ( - response.links['terms-of-service']['url'] - if 'terms-of-service' in response.links else terms_of_service) + if 'terms-of-service' in response.links: + terms_of_service = response.links['terms-of-service']['url'] + if 'next' in response.links: + new_authzr_uri = response.links['next']['url'] if new_authzr_uri is None: - try: - new_authzr_uri = response.links['next']['url'] - except KeyError: - raise errors.ClientError('"next" link missing') + raise errors.ClientError('"next" link missing') return messages.RegistrationResource( body=messages.Registration.from_json(response.json()), @@ -181,40 +180,41 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier, new_authzr_uri): + def request_challenges(self, identifier, new_authzr_uri=None): """Request challenges. - :param identifier: Identifier to be challenged. - :type identifier: `.messages.Identifier` - - :param str new_authzr_uri: new-authorization URI + :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: ``new-authorization`` URI. If omitted, + will default to value found in ``directory``. :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ new_authz = messages.NewAuthorization(identifier=identifier) - response = self.net.post(new_authzr_uri, new_authz) + response = self.net.post(self.directory.new_authz + if new_authzr_uri is None else new_authzr_uri, + new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) - def request_domain_challenges(self, domain, new_authz_uri): + def request_domain_challenges(self, domain, new_authzr_uri=None): """Request challenges for domain names. This is simply a convenience function that wraps around `request_challenges`, but works with domain names instead of - generic identifiers. + generic identifiers. See ``request_challenges`` for more + documentation. :param str domain: Domain name to be challenged. - :param str new_authzr_uri: new-authorization URI :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain), new_authz_uri) + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) def answer_challenge(self, challb, response): """Answer challenge. @@ -336,8 +336,9 @@ class Client(object): # pylint: disable=too-many-instance-attributes :param authzrs: `list` of `.AuthorizationResource` :param int mintime: Minimum time before next attempt, used if ``Retry-After`` is not present in the response. - :param int max_attempts: Maximum number of attempts before - `PollError` with non-empty ``waiting`` is raised. + :param int max_attempts: Maximum number of attempts (per + authorization) before `PollError` with non-empty ``waiting`` + is raised. :returns: ``(cert, updated_authzrs)`` `tuple` where ``cert`` is the issued certificate (`.messages.CertificateResource`), @@ -351,6 +352,11 @@ class Client(object): # pylint: disable=too-many-instance-attributes was marked by the CA as invalid """ + # pylint: disable=too-many-locals + assert max_attempts > 0 + attempts = collections.defaultdict(int) + exhausted = set() + # priority queue with datetime (based on Retry-After) as key, # and original Authorization Resource as value waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] @@ -358,8 +364,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) - while waiting and max_attempts: - max_attempts -= 1 + while waiting: # find the smallest Retry-After, and sleep if necessary when, authzr = heapq.heappop(waiting) now = datetime.datetime.now() @@ -373,16 +378,20 @@ class Client(object): # pylint: disable=too-many-instance-attributes updated_authzr, response = self.poll(updated[authzr]) updated[authzr] = updated_authzr + attempts[authzr] += 1 # pylint: disable=no-member if updated_authzr.body.status not in ( messages.STATUS_VALID, messages.STATUS_INVALID): - # push back to the priority queue, with updated retry_after - heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + if attempts[authzr] < max_attempts: + # push back to the priority queue, with updated retry_after + heapq.heappush(waiting, (self.retry_after( + response, default=mintime), authzr)) + else: + exhausted.add(authzr) - if not max_attempts or any(authzr.body.status == messages.STATUS_INVALID - for authzr in six.itervalues(updated)): - raise errors.PollError(waiting, updated) + if exhausted or any(authzr.body.status == messages.STATUS_INVALID + for authzr in six.itervalues(updated)): + raise errors.PollError(exhausted, updated) updated_authzrs = tuple(updated[authzr] for authzr in authzrs) return self.request_issuance(csr, updated_authzrs), updated_authzrs @@ -483,7 +492,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes 'Successful revocation must return HTTP OK status') -class ClientNetwork(object): +class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """Client network.""" JSON_CONTENT_TYPE = 'application/json' JSON_ERROR_CONTENT_TYPE = 'application/problem+json' @@ -539,7 +548,7 @@ class ClientNetwork(object): # TODO: response.json() is called twice, once here, and # once in _get and _post clients jobj = response.json() - except ValueError as error: + except ValueError: jobj = None if not response.ok: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 58f55b293..822a1be5f 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -34,8 +34,12 @@ class ClientTest(unittest.TestCase): self.net.get.return_value = self.response self.directory = messages.Directory({ - messages.NewRegistration: 'https://www.letsencrypt-demo.org/acme/new-reg', - messages.Revocation: 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewRegistration: + 'https://www.letsencrypt-demo.org/acme/new-reg', + messages.Revocation: + 'https://www.letsencrypt-demo.org/acme/revoke-cert', + messages.NewAuthorization: + 'https://www.letsencrypt-demo.org/acme/new-authz', }) from acme.client import Client @@ -127,13 +131,20 @@ class ClientTest(unittest.TestCase): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) + def test_query_registration_updates_new_authzr_uri(self): + self.response.json.return_value = self.regr.body.to_json() + self.response.links = {'next': {'url': 'UPDATED'}} + self.assertEqual( + 'UPDATED', + self.client.query_registration(self.regr).new_authzr_uri) + def test_agree_to_tos(self): self.client.update_registration = mock.Mock() self.client.agree_to_tos(self.regr) regr = self.client.update_registration.call_args[0][0] self.assertEqual(self.regr.terms_of_service, regr.body.agreement) - def test_request_challenges(self): + def _prepare_response_for_request_challenges(self): self.response.status_code = http_client.CREATED self.response.headers['Location'] = self.authzr.uri self.response.json.return_value = self.authz.to_json() @@ -141,10 +152,20 @@ class ClientTest(unittest.TestCase): 'next': {'url': self.authzr.new_cert_uri}, } - self.client.request_challenges(self.identifier, self.authzr.uri) - # TODO: test POST call arguments + def test_request_challenges(self): + self._prepare_response_for_request_challenges() + self.client.request_challenges(self.identifier) + self.net.post.assert_called_once_with( + self.directory.new_authz, + messages.NewAuthorization(identifier=self.identifier)) - # TODO: split here and separate test + def test_requets_challenges_custom_uri(self): + self._prepare_response_for_request_challenges() + self.client.request_challenges(self.identifier, 'URI') + self.net.post.assert_called_once_with('URI', mock.ANY) + + def test_request_challenges_unexpected_update(self): + self._prepare_response_for_request_challenges() self.response.json.return_value = self.authz.update( identifier=self.identifier.update(value='foo')).to_json() self.assertRaises( @@ -153,15 +174,20 @@ class ClientTest(unittest.TestCase): def test_request_challenges_missing_next(self): self.response.status_code = http_client.CREATED - self.assertRaises( - errors.ClientError, self.client.request_challenges, - self.identifier, self.regr) + self.assertRaises(errors.ClientError, self.client.request_challenges, + self.identifier) def test_request_domain_challenges(self): self.client.request_challenges = mock.MagicMock() self.assertEqual( self.client.request_challenges(self.identifier), - self.client.request_domain_challenges('example.com', self.regr)) + self.client.request_domain_challenges('example.com')) + + def test_request_domain_challenges_custom_uri(self): + self.client.request_challenges = mock.MagicMock() + self.assertEqual( + self.client.request_challenges(self.identifier, 'URI'), + self.client.request_domain_challenges('example.com', 'URI')) def test_answer_challenge(self): self.response.links['up'] = {'url': self.challr.authzr_uri} @@ -310,7 +336,10 @@ class ClientTest(unittest.TestCase): ) cert, updated_authzrs = self.client.poll_and_request_issuance( - csr, authzrs, mintime=mintime) + csr, authzrs, mintime=mintime, + # make sure that max_attempts is per-authorization, rather + # than global + max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) self.assertTrue(cert[0] is csr) self.assertTrue(cert[1] is updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') @@ -331,7 +360,8 @@ class ClientTest(unittest.TestCase): self.assertEqual(clock.dt, datetime.datetime(2015, 3, 27, 0, 1, 7)) # CA sets invalid | TODO: move to a separate test - invalid_authzr = mock.MagicMock(times=[], retries=[messages.STATUS_INVALID]) + invalid_authzr = mock.MagicMock( + times=[], retries=[messages.STATUS_INVALID]) self.assertRaises( errors.PollError, self.client.poll_and_request_issuance, csr, authzrs=(invalid_authzr,), mintime=mintime) diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 0385667c7..77d47c522 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -56,26 +56,25 @@ class MissingNonce(NonceError): class PollError(ClientError): """Generic error when polling for authorization fails. - This might be caused by either timeout (`waiting` will be non-empty) + This might be caused by either timeout (`exhausted` will be non-empty) or by some authorization being invalid. - :ivar waiting: Priority queue with `datetime.datatime` (based on - ``Retry-After``) as key, and original `.AuthorizationResource` - as value. + :ivar exhausted: Set of `.AuthorizationResource` that didn't finish + within max allowed attempts. :ivar updated: Mapping from original `.AuthorizationResource` to the most recently updated one """ - def __init__(self, waiting, updated): - self.waiting = waiting + def __init__(self, exhausted, updated): + self.exhausted = exhausted self.updated = updated super(PollError, self).__init__() @property def timeout(self): """Was the error caused by timeout?""" - return bool(self.waiting) + return bool(self.exhausted) def __repr__(self): - return '{0}(waiting={1!r}, updated={2!r})'.format( - self.__class__.__name__, self.waiting, self.updated) + return '{0}(exhausted={1!r}, updated={2!r})'.format( + self.__class__.__name__, self.exhausted, self.updated) diff --git a/acme/acme/errors_test.py b/acme/acme/errors_test.py index 45b269a0b..1e5f3d479 100644 --- a/acme/acme/errors_test.py +++ b/acme/acme/errors_test.py @@ -1,5 +1,4 @@ """Tests for acme.errors.""" -import datetime import unittest import mock @@ -36,9 +35,9 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - waiting=[(datetime.datetime(2015, 11, 29), mock.sentinel.AR)], + exhausted=set([mock.sentinel.AR]), updated={}) - self.invalid = PollError(waiting=[], updated={ + self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) def test_timeout(self): @@ -46,8 +45,8 @@ class PollErrorTest(unittest.TestCase): self.assertFalse(self.invalid.timeout) def test_repr(self): - self.assertEqual('PollError(waiting=[], updated={sentinel.AR: ' - 'sentinel.AR2})', repr(self.invalid)) + self.assertEqual('PollError(exhausted=%s, updated={sentinel.AR: ' + 'sentinel.AR2})' % repr(set()), repr(self.invalid)) if __name__ == "__main__": diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 9c6a5f7b9..06b4492d6 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -130,8 +130,9 @@ class Directory(jose.JSONDeSerializable): @classmethod def register(cls, resource_body_cls): """Register resource.""" - assert resource_body_cls.resource_type not in cls._REGISTERED_TYPES - cls._REGISTERED_TYPES[resource_body_cls.resource_type] = resource_body_cls + resource_type = resource_body_cls.resource_type + assert resource_type not in cls._REGISTERED_TYPES + cls._REGISTERED_TYPES[resource_type] = resource_body_cls return resource_body_cls def __init__(self, jobj): diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 2778635f5..85cd9d11d 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -32,11 +32,10 @@ class TLSSNI01ServerTest(unittest.TestCase): """Test for acme.standalone.TLSSNI01Server.""" def setUp(self): - self.certs = { - b'localhost': (test_util.load_pyopenssl_private_key('rsa512_key.pem'), - # pylint: disable=protected-access - test_util.load_cert('cert.pem')), - } + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa512_key.pem'), + test_util.load_cert('cert.pem'), + )} from acme.standalone import TLSSNI01Server self.server = TLSSNI01Server(("", 0), certs=self.certs) # pylint: disable=no-member @@ -49,7 +48,8 @@ class TLSSNI01ServerTest(unittest.TestCase): def test_it(self): host, port = self.server.socket.getsockname()[:2] - cert = crypto_util.probe_sni(b'localhost', host=host, port=port, timeout=1) + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1) self.assertEqual(jose.ComparableX509(cert), jose.ComparableX509(self.certs[b'localhost'][1])) @@ -140,7 +140,8 @@ class TestSimpleTLSSNI01Server(unittest.TestCase): while max_attempts: max_attempts -= 1 try: - cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', self.port) + cert = crypto_util.probe_sni( + b'localhost', b'0.0.0.0', self.port) except errors.Error: self.assertTrue(max_attempts > 0, "Timeout!") time.sleep(1) # wait until thread starts diff --git a/acme/setup.cfg b/acme/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/acme/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/acme/setup.py b/acme/setup.py index 372c05b13..0e1af6ad3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,13 +4,15 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', + 'ndg-httpsclient', # urllib3 InsecurePlatformWarning (#304) + 'pyasn1', # urllib3 InsecurePlatformWarning (#304) # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13', 'pyrfc3339', @@ -22,6 +24,7 @@ install_requires = [ ] # env markers in extras_require cause problems with older pip: #517 +# Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying @@ -31,10 +34,11 @@ if sys.version_info < (2, 7): else: install_requires.append('mock') -if sys.version_info < (2, 7, 9): - # For secure SSL connection with Python 2.7 (InsecurePlatformWarning) - install_requires.append('ndg-httpsclient') - install_requires.append('pyasn1') +dev_extras = [ + 'nose', + 'pep8', + 'tox', +] docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags @@ -42,11 +46,6 @@ docs_extras = [ 'sphinxcontrib-programoutput', ] -testing_extras = [ - 'nose', - 'tox', -] - setup( name='acme', @@ -76,8 +75,8 @@ setup( include_package_data=True, install_requires=install_requires, extras_require={ + 'dev': dev_extras, 'docs': docs_extras, - 'testing': testing_extras, }, entry_points={ 'console_scripts': [ diff --git a/bootstrap/README b/bootstrap/README deleted file mode 100644 index d91780903..000000000 --- a/bootstrap/README +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains scripts that install necessary OS-specific -prerequisite dependencies (see docs/using.rst). - -General dependencies: -- ca-certificates: communication with demo ACMO server at - https://www.letsencrypt-demo.org diff --git a/bootstrap/_arch_common.sh b/bootstrap/_arch_common.sh deleted file mode 100755 index 2b512792f..000000000 --- a/bootstrap/_arch_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Tested with: -# - ArchLinux (x86_64) -# -# "python-virtualenv" is Python3, but "python2-virtualenv" provides -# only "virtualenv2" binary, not "virtualenv" necessary in -# ./bootstrap/dev/_common_venv.sh - -deps=" - python2 - python-virtualenv - gcc - dialog - augeas - openssl - libffi - ca-certificates - pkg-config -" - -missing=$(pacman -T $deps) - -if [ "$missing" ]; then - pacman -S --needed $missing -fi diff --git a/bootstrap/_deb_common.sh b/bootstrap/_deb_common.sh deleted file mode 100755 index 58ea67a45..000000000 --- a/bootstrap/_deb_common.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -# Current version tested with: -# -# - Ubuntu -# - 14.04 (x64) -# - 15.04 (x64) -# - Debian -# - 7.9 "wheezy" (x64) -# - sid (2015-10-21) (x64) - -# Past versions tested with: -# -# - Debian 8.0 "jessie" (x64) -# - Raspbian 7.8 (armhf) - -# Believed not to work: -# -# - Debian 6.0.10 "squeeze" (x64) - -apt-get update - -# virtualenv binary can be found in different packages depending on -# distro version (#346) - -virtualenv= -if apt-cache show virtualenv > /dev/null 2>&1; then - virtualenv="virtualenv" -fi - -if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" -fi - -augeas_pkg="libaugeas0 augeas-lenses" -AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` - -AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' - fi - - echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/"$BACKPORT_NAME".list - apt-get update - apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - fi - -} - - -if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Let's Encrypt apache plugin..." - fi - # XXX add a case for ubuntu PPAs -fi - -apt-get install -y --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - dialog \ - $augeas_pkg \ - libssl-dev \ - libffi-dev \ - ca-certificates \ - - - -if ! command -v virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting - exit 1 -fi diff --git a/bootstrap/_gentoo_common.sh b/bootstrap/_gentoo_common.sh deleted file mode 100755 index f49dc00f0..000000000 --- a/bootstrap/_gentoo_common.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - dev-util/dialog - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - -case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace $PACKAGES - ;; - (portage|*) - emerge --noreplace $PACKAGES - ;; -esac diff --git a/bootstrap/_rpm_common.sh b/bootstrap/_rpm_common.sh deleted file mode 100755 index db1665268..000000000 --- a/bootstrap/_rpm_common.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh - -# Tested with: -# - Fedora 22, 23 (x64) -# - Centos 7 (x64: on DigitalOcean droplet) - -if type dnf 2>/dev/null -then - tool=dnf -elif type yum 2>/dev/null -then - tool=yum - -else - echo "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 -fi - -# Some distros and older versions of current distros use a "python27" -# instead of "python" naming convention. Try both conventions. -if ! $tool install -y \ - python \ - python-devel \ - python-virtualenv -then - if ! $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi -fi - -if ! $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates -then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 -fi - - -if $tool list installed "httpd" >/dev/null 2>&1; then - if ! $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi -fi diff --git a/bootstrap/_suse_common.sh b/bootstrap/_suse_common.sh deleted file mode 100755 index efeebe4f8..000000000 --- a/bootstrap/_suse_common.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# SLE12 don't have python-virtualenv - -zypper -nq in -l \ - python \ - python-devel \ - python-virtualenv \ - gcc \ - dialog \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates \ diff --git a/bootstrap/archlinux.sh b/bootstrap/archlinux.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/archlinux.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/centos.sh b/bootstrap/centos.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/centos.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/debian.sh b/bootstrap/debian.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/debian.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/dev/README b/bootstrap/dev/README deleted file mode 100644 index 759496187..000000000 --- a/bootstrap/dev/README +++ /dev/null @@ -1 +0,0 @@ -This directory contains developer setup. diff --git a/bootstrap/fedora.sh b/bootstrap/fedora.sh deleted file mode 120000 index a0db46d70..000000000 --- a/bootstrap/fedora.sh +++ /dev/null @@ -1 +0,0 @@ -_rpm_common.sh \ No newline at end of file diff --git a/bootstrap/freebsd.sh b/bootstrap/freebsd.sh deleted file mode 100755 index 4482c35cd..000000000 --- a/bootstrap/freebsd.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -xe - -pkg install -Ay \ - python \ - py27-virtualenv \ - augeas \ - libffi \ diff --git a/bootstrap/gentoo.sh b/bootstrap/gentoo.sh deleted file mode 120000 index 125d6a592..000000000 --- a/bootstrap/gentoo.sh +++ /dev/null @@ -1 +0,0 @@ -_gentoo_common.sh \ No newline at end of file diff --git a/bootstrap/install-deps.sh b/bootstrap/install-deps.sh deleted file mode 100755 index e907e7035..000000000 --- a/bootstrap/install-deps.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -e -# -# Install OS dependencies. In the glorious future, letsencrypt-auto will -# source this... - -if test "`id -u`" -ne "0" ; then - SUDO=sudo -else - SUDO= -fi - -BOOTSTRAP=`dirname $0` -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/arch-release ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif [ -f /etc/gentoo-release ] ; then - echo "Bootstrapping dependencies for Gentoo-based OSes..." - $SUDO $BOOTSTRAP/_gentoo_common.sh -elif uname | grep -iq FreeBSD ; then - echo "Bootstrapping dependencies for FreeBSD..." - $SUDO $BOOTSTRAP/freebsd.sh -elif `grep -qs openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE.." - $SUDO $BOOTSTRAP/suse.sh -elif uname | grep -iq Darwin ; then - echo "Bootstrapping dependencies for Mac OS X..." - echo "WARNING: Mac support is very experimental at present..." - $BOOTSTRAP/mac.sh -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - exit 1 -fi diff --git a/bootstrap/mac.sh b/bootstrap/mac.sh deleted file mode 100755 index 4d1fb8208..000000000 --- a/bootstrap/mac.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -e -if ! hash brew 2>/dev/null; then - echo "Homebrew Not Installed\nDownloading..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -fi - -brew install augeas -brew install dialog - -if ! hash pip 2>/dev/null; then - echo "pip Not Installed\nInstalling python from Homebrew..." - brew install python -fi - -if ! hash virtualenv 2>/dev/null; then - echo "virtualenv Not Installed\nInstalling with pip" - pip install virtualenv -fi diff --git a/bootstrap/manjaro.sh b/bootstrap/manjaro.sh deleted file mode 120000 index c5c9479f7..000000000 --- a/bootstrap/manjaro.sh +++ /dev/null @@ -1 +0,0 @@ -_arch_common.sh \ No newline at end of file diff --git a/bootstrap/suse.sh b/bootstrap/suse.sh deleted file mode 120000 index fc4c1dee4..000000000 --- a/bootstrap/suse.sh +++ /dev/null @@ -1 +0,0 @@ -_suse_common.sh \ No newline at end of file diff --git a/bootstrap/ubuntu.sh b/bootstrap/ubuntu.sh deleted file mode 120000 index 068a039cb..000000000 --- a/bootstrap/ubuntu.sh +++ /dev/null @@ -1 +0,0 @@ -_deb_common.sh \ No newline at end of file diff --git a/bootstrap/venv.sh b/bootstrap/venv.sh deleted file mode 100755 index 5042178d9..000000000 --- a/bootstrap/venv.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -e -# -# Installs and updates letencrypt virtualenv -# -# USAGE: source ./dev/venv.sh - - -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - virtualenv --no-site-packages --python ${LE_PYTHON:-python2} $VENV_PATH -fi - -. $VENV_PATH/bin/activate -pip install -U setuptools -pip install -U pip - -pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx - -echo -echo "Congratulations, Let's Encrypt has been successfully installed/updated!" -echo -printf "%s" "Your prompt should now be prepended with ($VENV_NAME). Next " -printf "time, if the prompt is different, 'source' this script again " -printf "before running 'letsencrypt'." -echo -echo -echo "You can now run 'letsencrypt --help'." diff --git a/docs/api/renewer.rst b/docs/api/renewer.rst deleted file mode 100644 index cc42c6eab..000000000 --- a/docs/api/renewer.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`letsencrypt.renewer` --------------------------- - -.. automodule:: letsencrypt.renewer - :members: diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 49c0824a3..ef644b7a0 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -107,7 +107,7 @@ and the version implemented by the Let's Encrypt client will be the version that was most current as of the release date of each client version. Mozilla offers three separate sets of cryptographic options, which trade off security and compatibility differently. These are -referred to as as the "Modern", "Intermediate", and "Old" configurations +referred to as the "Modern", "Intermediate", and "Old" configurations (in order from most secure to least secure, and least-backwards compatible to most-backwards compatible). The client will follow the Mozilla defaults for the *Intermediate* configuration by default, at least with regards to @@ -139,7 +139,7 @@ client's priorities. The Mozilla security team is likely to have more resources and expertise to bring to bear on evaluating reasons why its recommendations should be updated. -The Let's Encrpyt project will entertain proposals to create a *very* +The Let's Encrypt project will entertain proposals to create a *very* small number of alternative configurations (apart from Modern, Intermediate, and Old) that there's reason to believe would be widely used by sysadmins; this would usually be a preferable course to modifying @@ -170,7 +170,7 @@ Changing your settings This will probably look something like -..code-block: shell +.. code-block:: shell letsencrypt --cipher-recommendations mozilla-secure letsencrypt --cipher-recommendations mozilla-intermediate @@ -179,14 +179,14 @@ This will probably look something like to track Mozilla's *Secure*, *Intermediate*, or *Old* recommendations, and -..code-block: shell +.. code-block:: shell letsencrypt --update-ciphers on to enable updating ciphers with each new Let's Encrypt client release, or -..code-block: shell +.. code-block:: shell letsencrypt --update-ciphers off diff --git a/docs/conf.py b/docs/conf.py index 21bcc6817..739d6ee43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -281,8 +281,6 @@ man_pages = [ [project], 7), ('man/letsencrypt', 'letsencrypt', u'letsencrypt script documentation', [project], 1), - ('man/letsencrypt-renewer', 'letsencrypt-renewer', - u'letsencrypt-renewer script documentation', [project], 1), ] # If true, show URL addresses after external links. diff --git a/docs/contributing.rst b/docs/contributing.rst index 5ec44470d..69604780c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -22,8 +22,8 @@ once: git clone https://github.com/letsencrypt/letsencrypt cd letsencrypt - ./bootstrap/install-deps.sh - ./bootstrap/dev/venv.sh + ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only + ./tools/venv.sh Then in each shell where you're working on the client, do: @@ -96,19 +96,48 @@ Integration testing with the boulder CA Generally it is sufficient to open a pull request and let Github and Travis run integration tests for you. -Mac OS X users: Run `./tests/mac-bootstrap.sh` instead of `boulder-start.sh` to -install dependencies, configure the environment, and start boulder. +However, if you prefer to run tests, you can use Vagrant, using the Vagrantfile +in Let's Encrypt's repository. To execute the tests on a Vagrant box, the only +command you are required to run is:: -Otherwise, install `Go`_ 1.5, libtool-ltdl, mariadb-server and -rabbitmq-server and then start Boulder_, an ACME CA server:: + ./tests/boulder-integration.sh + +Otherwise, please follow the following instructions. + +Mac OS X users: Run ``./tests/mac-bootstrap.sh`` instead of +``boulder-start.sh`` to install dependencies, configure the +environment, and start boulder. + +Otherwise, install `Go`_ 1.5, ``libtool-ltdl``, ``mariadb-server`` and +``rabbitmq-server`` and then start Boulder_, an ACME CA server. + +If you can't get packages of Go 1.5 for your Linux system, +you can execute the following commands to install it: + +.. code-block:: shell + + wget https://storage.googleapis.com/golang/go1.5.3.linux-amd64.tar.gz -P /tmp/ + sudo tar -C /usr/local -xzf /tmp/go1.5.3.linux-amd64.tar.gz + if ! grep -Fxq "export GOROOT=/usr/local/go" ~/.profile ; then echo "export GOROOT=/usr/local/go" >> ~/.profile; fi + if ! grep -Fxq "export PATH=\\$GOROOT/bin:\\$PATH" ~/.profile ; then echo "export PATH=\\$GOROOT/bin:\\$PATH" >> ~/.profile; fi + +These commands download `Go`_ 1.5.3 to ``/tmp/``, extracts to ``/usr/local``, +and then adds the export lines required to execute ``boulder-start.sh`` to +``~/.profile`` if they were not previously added + +Make sure you execute the following command after `Go`_ finishes installing:: + + if ! grep -Fxq "export GOPATH=\\$HOME/go" ~/.profile ; then echo "export GOPATH=\\$HOME/go" >> ~/.profile; fi + +Afterwards, you'd be able to start Boulder_ using the following command:: ./tests/boulder-start.sh The script will download, compile and run the executable; please be patient - it will take some time... Once its ready, you will see -``Server running, listening on 127.0.0.1:4000...``. Add an -``/etc/hosts`` entry pointing ``le.wtf`` to 127.0.0.1. You may now -run (in a separate terminal):: +``Server running, listening on 127.0.0.1:4000...``. Add ``/etc/hosts`` +entries pointing ``le.wtf``, ``le1.wtf``, ``le2.wtf``, ``le3.wtf`` +and ``nginx.wtf`` to 127.0.0.1. You may now run (in a separate terminal):: ./tests/boulder-integration.sh && echo OK || echo FAIL @@ -136,7 +165,7 @@ Plugin-architecture Let's Encrypt has a plugin architecture to facilitate support for different webservers, other TLS servers, and operating systems. The interfaces available for plugins to implement are defined in -`interfaces.py`_. +`interfaces.py`_ and `plugins/common.py`_. The most common kind of plugin is a "Configurator", which is likely to implement the `~letsencrypt.interfaces.IAuthenticator` and @@ -147,6 +176,7 @@ There are also `~letsencrypt.interfaces.IDisplay` plugins, which implement bindings to alternative UI libraries. .. _interfaces.py: https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py +.. _plugins/common.py: https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L34 Authenticators @@ -279,7 +309,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``. + virtualenv. You can do this by running ``./tools/venv.sh``. (this is a **very important** step) 3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. @@ -365,10 +395,13 @@ Now run tests inside the Docker image: Notes on OS dependencies ======================== -OS level dependencies are managed by scripts in ``bootstrap``. Some notes -are provided here mainly for the :ref:`developers ` reference. +OS-level dependencies can be installed like so: -In general: +.. code-block:: shell + + letsencrypt-auto-source/letsencrypt-auto --os-packages-only + +In general... * ``sudo`` is required as a suggested way of running privileged process * `Python`_ 2.6/2.7 is required @@ -380,62 +413,19 @@ In general: .. _Augeas: http://augeas.net/ .. _Virtualenv: https://virtualenv.pypa.io -Ubuntu ------- - -.. code-block:: shell - - sudo ./bootstrap/ubuntu.sh - Debian ------ -.. code-block:: shell - - sudo ./bootstrap/debian.sh - For squeeze you will need to: - Use ``virtualenv --no-site-packages -p python`` instead of ``-p python2``. -.. _`#280`: https://github.com/letsencrypt/letsencrypt/issues/280 - - -Mac OSX -------- - -.. code-block:: shell - - ./bootstrap/mac.sh - - -Fedora ------- - -.. code-block:: shell - - sudo ./bootstrap/fedora.sh - - -Centos 7 --------- - -.. code-block:: shell - - sudo ./bootstrap/centos.sh - - FreeBSD ------- -.. code-block:: shell - - sudo ./bootstrap/freebsd.sh - -Bootstrap script for FreeBSD uses ``pkg`` for package installation, -i.e. it does not use ports. +Package installation for FreeBSD uses ``pkg``, not ports. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see below), you will need a compatible shell, e.g. ``pkg install bash && diff --git a/docs/man/letsencrypt-renewer.rst b/docs/man/letsencrypt-renewer.rst deleted file mode 100644 index 8fd232fa8..000000000 --- a/docs/man/letsencrypt-renewer.rst +++ /dev/null @@ -1 +0,0 @@ -.. program-output:: letsencrypt-renewer --help diff --git a/docs/using.rst b/docs/using.rst index eb7c3962e..37fca2c57 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -16,27 +16,12 @@ letsencrypt-auto ---------------- ``letsencrypt-auto`` is a wrapper which installs some dependencies -from your OS standard package repositories (e.g using `apt-get` or +from your OS standard package repositories (e.g. using `apt-get` or `yum`), and for other dependencies it sets up a virtualized Python environment with packages downloaded from PyPI [#venv]_. It also provides automated updates. -Firstly, please `install Git`_ and run the following commands: - -.. code-block:: shell - - git clone https://github.com/letsencrypt/letsencrypt - cd letsencrypt - - -.. _`install Git`: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git - -.. note:: On RedHat/CentOS 6 you will need to enable the EPEL_ - repository before install. - -.. _EPEL: http://fedoraproject.org/wiki/EPEL - -To install and run the client you just need to type: +To install and run the client, just type... .. code-block:: shell @@ -64,7 +49,7 @@ or for full help, type: ``letsencrypt-auto`` is the recommended method of running the Let's Encrypt client beta releases on systems that don't have a packaged version. Debian, -Arch linux, FreeBSD, and OpenBSD now have native packages, so on those +Arch Linux, Gentoo, FreeBSD, and OpenBSD now have native packages, so on those systems you can just install ``letsencrypt`` (and perhaps ``letsencrypt-apache``). If you'd like to run the latest copy from Git, or run your own locally modified copy of the client, follow the instructions in @@ -86,7 +71,9 @@ Plugin Auth Inst Notes =========== ==== ==== =============================================================== apache_ Y Y Automates obtaining and installing a cert with Apache 2.4 on Debian-based distributions with ``libaugeas0`` 1.0+. -standalone_ Y N Uses a "standalone" webserver to obtain a cert. +standalone_ Y N Uses a "standalone" webserver to obtain a cert. This is useful + on systems with no webserver, or when direct integration with + the local webserver is not supported or not desired. webroot_ Y N Obtains a cert by writing to the webroot directory of an already running webserver. manual_ Y N Helps you obtain a cert by giving you instructions to perform @@ -106,6 +93,46 @@ This automates both obtaining *and* installing certs on an Apache webserver. To specify this plugin on the command line, simply include ``--apache``. +Webroot +------- + +If you're running a local webserver for which you have the ability +to modify the content being served, and you'd prefer not to stop the +webserver during the certificate issuance process, you can use the webroot +plugin to obtain a cert by including ``certonly`` and ``--webroot`` on +the command line. In addition, you'll need to specify ``--webroot-path`` +or ``-w`` with the top-level directory ("web root") containing the files +served by your webserver. For example, ``--webroot-path /var/www/html`` +or ``--webroot-path /usr/share/nginx/html`` are two common webroot paths. + +If you're getting a certificate for many domains at once, the plugin +needs to know where each domain's files are served from, which could +potentially be a separate directory for each domain. When requested a +certificate for multiple domains, each domain will use the most recently +specified ``--webroot-path``. So, for instance, + +``letsencrypt certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net`` + +would obtain a single certificate for all of those names, using the +``/var/www/example`` webroot directory for the first two, and +``/var/www/other`` for the second two. + +The webroot plugin works by creating a temporary file for each of your requested +domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's +Encrypt validation server makes HTTP requests to validate that the DNS for each +requested domain resolves to the server running letsencrypt. An example request +made to your web server would look like: + +:: + + 66.133.109.36 - - [05/Jan/2016:20:11:24 -0500] "GET /.well-known/acme-challenge/HGr8U1IeTW4kY_Z6UIyaakzOkyQgPr_7ArlLgtZE8SX HTTP/1.1" 200 87 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" + +Note that to use the webroot plugin, your server must be configured to serve +files from hidden directories. If ``/.well-known`` is treated specially by +your webserver configuration, you might need to modify the configuration +to ensure that files inside ``/.well-known/acme-challenge`` are served by +the webserver. + Standalone ---------- @@ -119,39 +146,10 @@ one of the options shown below on the command line. * ``--standalone-supported-challenges http-01`` to use port 80 * ``--standalone-supported-challenges tls-sni-01`` to use port 443 -Webroot -------- - -If you're running a webserver that you don't want to stop to use -standalone, you can use the webroot plugin to obtain a cert by -including ``certonly`` and ``--webroot`` on the command line. In -addition, you'll need to specify ``--webroot-path`` or ``-w`` with the root -directory of the files served by your webserver. For example, -``--webroot-path /var/www/html`` or -``--webroot-path /usr/share/nginx/html`` are two common webroot paths. - -If you're getting a certificate for many domains at once, each domain will use -the most recent ``--webroot-path``. So for instance: - -``letsencrypt certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/eg -d eg.is -d www.eg.is`` - -Would obtain a single certificate for all of those names, using the -``/var/www/example`` webroot directory for the first two, and -``/var/www/eg`` for the second two. - -The webroot plugin works by creating a temporary file for each of your requested -domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's -Encrypt validation server makes HTTP requests to validate that the DNS for each -requested domain resolves to the server running letsencrypt. An example request -made to your web server would look like: - -:: - - 66.133.109.36 - - [05/Jan/2016:20:11:24 -0500] "GET /.well-known/acme-challenge/HGr8U1IeTW4kY_Z6UIyaakzOkyQgPr_7ArlLgtZE8SX HTTP/1.1" 200 87 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" - -Note that to use the webroot plugin, your server must be configured to serve -files from hidden directories. - +The standalone plugin does not rely on any other server software running +on the machine where you obtain the certificate. It must still be possible +for that machine to accept inbound connections from the Internet on the +specified port using each requested domain name. Manual ------ @@ -161,7 +159,8 @@ other than your target webserver or perform the steps for domain validation yourself, you can use the manual plugin. While hidden from the UI, you can use the plugin to obtain a cert by specifying ``certonly`` and ``--manual`` on the command line. This requires you -to copy and paste commands into another terminal session. +to copy and paste commands into another terminal session, which may +be on a different computer. Nginx ----- @@ -172,7 +171,7 @@ is still experimental, however, and is not installed with letsencrypt-auto_. If installed, you can select this plugin on the command line by including ``--nginx``. -Third party plugins +Third-party plugins ------------------- These plugins are listed at @@ -182,24 +181,65 @@ interested, you can also :ref:`write your own plugin `. Renewal ======= -.. note:: Let's Encrypt CA issues short lived certificates (90 +.. note:: Let's Encrypt CA issues short-lived certificates (90 days). Make sure you renew the certificates at least once in 3 months. -In order to renew certificates simply call the ``letsencrypt`` (or -letsencrypt-auto_) again, and use the same values when prompted. You -can automate it slightly by passing necessary flags on the CLI (see -`--help all`), or even further using the :ref:`config-file`. The -``--renew-by-default`` flag may be helpful for automating renewal. If -you're sure that UI doesn't prompt for any details you can add the -command to ``crontab`` (make it less than every 90 days to avoid -problems, say every month). +The ``letsencrypt`` client now supports a ``renew`` action to check +all installed certificates for impending expiry and attempt to renew +them. The simplest form is simply + +``letsencrypt renew`` + +This will attempt to renew any previously-obtained certificates that +expire in less than 30 days. The same plugin and options that were used +at the time the certificate was originally issued will be used for the +renewal attempt, unless you specify other plugins or options. + +If you're sure that this command executes successfully without human +intervention, you can add the command to ``crontab`` (since certificates +are only renewed when they're determined to be near expiry, the command +can run on a regular basis, like every week or every day); note that +the current version provides detailed output describing either renewal +success or failure. + +The ``--force-renew`` flag may be helpful for automating renewal; +it causes the expiration time of the certificate(s) to be ignored when +considering renewal, and attempts to renew each and every installed +certificate regardless of its age. (This form is not appropriate to run +daily because each certificate will be renewed every day, which will +quickly run into the certificate authority rate limit.) + +Note that options provided to ``letsencrypt renew`` will apply to +*every* certificate for which renewal is attempted; for example, +``letsencrypt renew --rsa-key-size 4096`` would try to replace every +near-expiry certificate with an equivalent certificate using a 4096-bit +RSA public key. If a certificate is successfully renewed using +specified options, those options will be saved and used for future +renewals of that certificate. + + +An alternative form that provides for more fine-grained control over the +renewal process (while renewing specified certificates one at a time), +is ``letsencrypt certonly`` with the complete set of subject domains of +a specific certificate specified via `-d` flags, like + +``letsencrypt certonly -d example.com -d www.example.com`` + +(All of the domains covered by the certificate must be specified in +this case in order to renew and replace the old certificate rather +than obtaining a new one; don't forget any `www.` domains! Specifying +a subset of the domains creates a new, separate certificate containing +only those domains, rather than replacing the original certificate.) +The ``certonly`` form attempts to renew one individual certificate. + Please note that the CA will send notification emails to the address you provide if you do not renew certificates that are about to expire. -Let's Encrypt is working hard on automating the renewal process. Until -the tool is ready, we are sorry for the inconvenience! +Let's Encrypt is working hard on improving the renewal process, and we +apologize for any inconveniences you encounter in integrating these +commands into your individual environment. .. _where-certs: @@ -389,6 +429,52 @@ If you don't want to use the Apache plugin, you can omit the Packages for Debian Jessie are coming in the next few weeks. +**Gentoo** + +The official Let's Encrypt client is available in Gentoo Portage. If you +want to use the Apache plugin, it has to be installed separately: + +.. code-block:: shell + + emerge -av app-crypt/letsencrypt + emerge -av app-crypt/letsencrypt-apache + +Currently, only the Apache plugin is included in Portage. However, if you +want the nginx plugin, you can use Layman to add the mrueg overlay which +does include the nginx plugin package: + +.. code-block:: shell + + emerge -av app-portage/layman + layman -S + layman -a mrueg + emerge -av app-crypt/letsencrypt-nginx + +When using the Apache plugin, you will run into a "cannot find a cert or key +directive" error if you're sporting the default Gentoo ``httpd.conf``. +You can fix this by commenting out two lines in ``/etc/apache2/httpd.conf`` +as follows: + +Change + +.. code-block:: shell + + + LoadModule ssl_module modules/mod_ssl.so + + +to + +.. code-block:: shell + + # + LoadModule ssl_module modules/mod_ssl.so + # + +For the time being, this is the only way for the Apache plugin to recognise +the appropriate directives when installing the certificate. +Note: this change is not required for the other plugins. + **Other Operating Systems** OS packaging is an ongoing effort. If you'd like to package diff --git a/letsencrypt-apache/MANIFEST.in b/letsencrypt-apache/MANIFEST.in index 933cc10ac..bdb67199f 100644 --- a/letsencrypt-apache/MANIFEST.in +++ b/letsencrypt-apache/MANIFEST.in @@ -2,5 +2,6 @@ include LICENSE.txt include README.rst recursive-include docs * recursive-include letsencrypt_apache/tests/testdata * +include letsencrypt_apache/centos-options-ssl-apache.conf include letsencrypt_apache/options-ssl-apache.conf recursive-include letsencrypt_apache/augeas_lens *.aug diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py index 9e0948f12..9b51c32a9 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/augeas_configurator.py @@ -120,7 +120,8 @@ class AugeasConfigurator(common.Plugin): self.reverter.add_to_temp_checkpoint( save_files, self.save_notes) else: - self.reverter.add_to_checkpoint(save_files, self.save_notes) + self.reverter.add_to_checkpoint(save_files, + self.save_notes) except errors.ReverterError as err: raise errors.PluginError(str(err)) diff --git a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug index 0f2cb7b45..edaca3fef 100644 --- a/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug +++ b/letsencrypt-apache/letsencrypt_apache/augeas_lens/httpd.aug @@ -106,11 +106,17 @@ let section (body:lens) = let inner = (sep_spc . argv arg_sec)? . sep_osp . dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? . indent . dels "[ \t\n\r]*/ ">\n" ] +let perl_section = [ indent . label "Perl" . del //i "" + . store /[^<]*/ + . del /<\/perl>/i "" . eol ] + + let rec content = section (content|directive) + | perl_section let lns = (content|directive|comment|empty)* @@ -121,6 +127,7 @@ let filter = (incl "/etc/apache2/apache2.conf") . (incl "/etc/apache2/conf-available/*.conf") . (incl "/etc/apache2/mods-available/*") . (incl "/etc/apache2/sites-available/*") . + (incl "/etc/apache2/vhosts.d/*.conf") . (incl "/etc/httpd/conf.d/*.conf") . (incl "/etc/httpd/httpd.conf") . (incl "/etc/httpd/conf/httpd.conf") . diff --git a/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf b/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf new file mode 100644 index 000000000..fbe8da0f2 --- /dev/null +++ b/letsencrypt-apache/letsencrypt_apache/centos-options-ssl-apache.conf @@ -0,0 +1,21 @@ +# Baseline setting to Include for SSL sites + +SSLEngine on + +# Intermediate configuration, tweak to your needs +SSLProtocol all -SSLv2 -SSLv3 +SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA +SSLHonorCipherOrder on + +SSLOptions +StrictRequire + +# Add vhost name to log entries: +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined +LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common + +#CustomLog /var/log/apache2/access.log vhost_combined +#LogLevel warn +#ErrorLog /var/log/apache2/error.log + +# Always ensure Cookies have "Secure" set (JAH 2012/1) +#Header edit Set-Cookie (?i)^(.*)(;\s*secure)??((\s*;)?(.*)) "$1; Secure$3$4" diff --git a/letsencrypt-apache/letsencrypt_apache/configurator.py b/letsencrypt-apache/letsencrypt_apache/configurator.py index 2a9fb0250..f4a407974 100644 --- a/letsencrypt-apache/letsencrypt_apache/configurator.py +++ b/letsencrypt-apache/letsencrypt_apache/configurator.py @@ -8,6 +8,7 @@ import shutil import socket import time +import zope.component import zope.interface from acme import challenges @@ -105,7 +106,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): add("handle-sites", default=constants.os_constant("handle_sites"), help="Let installer handle enabling sites for you." + "(Only Ubuntu/Debian currently)") - le_util.add_deprecated_argument(add, "init-script", 1) + le_util.add_deprecated_argument(add, argument_name="ctl", nargs=1) + le_util.add_deprecated_argument( + add, argument_name="init-script", nargs=1) def __init__(self, *args, **kwargs): """Initialize an Apache Configurator. @@ -132,7 +135,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): @property def mod_ssl_conf(self): """Full absolute path to SSL configuration file.""" - return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST) + return os.path.join(self.config.config_dir, + constants.MOD_SSL_CONF_DEST) def prepare(self): """Prepare the authenticator/installer. @@ -157,6 +161,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) + if not self._check_aug_version(): + raise errors.NotSupportedError( + "Apache plugin support requires libaugeas0 and augeas-lenses " + "version 1.2.0 or higher, please make sure you have you have " + "those installed.") + self.parser = parser.ApacheParser( self.aug, self.conf("server-root"), self.conf("vhost-root"), self.version) @@ -168,16 +178,31 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): install_ssl_options_conf(self.mod_ssl_conf) + def _check_aug_version(self): + """ Checks that we have recent enough version of libaugeas. + If augeas version is recent enough, it will support case insensitive + regexp matching""" + + self.aug.set("/test/path/testing/arg", "aRgUMeNT") + try: + matches = self.aug.match( + "/test//*[self::arg=~regexp('argument', 'i')]") + except RuntimeError: + self.aug.remove("/test/path") + return False + self.aug.remove("/test/path") + return matches + def deploy_cert(self, domain, cert_path, key_path, - chain_path=None, fullchain_path=None): # pylint: disable=unused-argument + chain_path=None, fullchain_path=None): """Deploys certificate to specified virtual host. Currently tries to find the last directives to deploy the cert in the VHost associated with the given domain. If it can't find the - directives, it searches the "included" confs. The function verifies that - it has located the three directives and finally modifies them to point - to the correct destination. After the certificate is installed, the - VirtualHost is enabled if it isn't already. + directives, it searches the "included" confs. The function verifies + that it has located the three directives and finally modifies them + to point to the correct destination. After the certificate is + installed, the VirtualHost is enabled if it isn't already. .. todo:: Might be nice to remove chain directive if none exists This shouldn't happen within letsencrypt though @@ -193,8 +218,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # cert_key... can all be parsed appropriately self.prepare_server_https("443") - path = {"cert_path": self.parser.find_dir("SSLCertificateFile", None, vhost.path), - "cert_key": self.parser.find_dir("SSLCertificateKeyFile", None, vhost.path)} + path = {"cert_path": self.parser.find_dir("SSLCertificateFile", + None, vhost.path), + "cert_key": self.parser.find_dir("SSLCertificateKeyFile", + None, vhost.path)} # Only include if a certificate chain is specified if chain_path is not None: @@ -224,7 +251,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir(vhost.path, "SSLCertificateChainFile", chain_path) else: - raise errors.PluginError("--chain-path is required for your version of Apache") + raise errors.PluginError("--chain-path is required for your " + "version of Apache") else: if not fullchain_path: raise errors.PluginError("Please provide the --fullchain-path\ @@ -279,6 +307,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if not vhost.ssl: vhost = self.make_vhost_ssl(vhost) + self._add_servername_alias(target_name, vhost) self.assoc[target_name] = vhost return vhost @@ -298,7 +327,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): elif not vhost.ssl: addrs = self._get_proposed_addrs(vhost, "443") # TODO: Conflicts is too conservative - if not any(vhost.enabled and vhost.conflicts(addrs) for vhost in self.vhosts): + if not any(vhost.enabled and vhost.conflicts(addrs) for + vhost in self.vhosts): vhost = self.make_vhost_ssl(vhost) else: logger.error( @@ -308,6 +338,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "VirtualHost not able to be selected.") + self._add_servername_alias(target_name, vhost) self.assoc[target_name] = vhost return vhost @@ -326,7 +357,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Points 1 - Address name with no SSL best_candidate = None best_points = 0 - for vhost in self.vhosts: if vhost.modmacro is True: continue @@ -488,15 +518,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: list """ - # Search vhost-root, httpd.conf for possible virtual hosts - paths = self.aug.match( - ("/files%s//*[label()=~regexp('%s')]" % - (self.conf("vhost-root"), parser.case_i("VirtualHost")))) - + # Search base config, and all included paths for VirtualHosts vhs = [] + vhost_paths = {} + for vhost_path in self.parser.parser_paths.keys(): + paths = self.aug.match( + ("/files%s//*[label()=~regexp('%s')]" % + (vhost_path, parser.case_i("VirtualHost")))) + for path in paths: + new_vhost = self._create_vhost(path) + realpath = os.path.realpath(new_vhost.filep) + if realpath not in vhost_paths.keys(): + vhs.append(new_vhost) + vhost_paths[realpath] = new_vhost.filep + elif realpath == new_vhost.filep: + # Prefer "real" vhost paths instead of symlinked ones + # ex: sites-enabled/vh.conf -> sites-available/vh.conf - for path in paths: - vhs.append(self._create_vhost(path)) + # remove old (most likely) symlinked one + vhs = [v for v in vhs if v.filep != vhost_paths[realpath]] + vhs.append(new_vhost) + vhost_paths[realpath] = realpath return vhs @@ -554,15 +596,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo - listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] + listens = [self.parser.get_arg(x).split()[0] for + x in self.parser.find_dir("Listen")] # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] if port in listens: return for listen in listens: - # For any listen statement, check if the machine also listens on Port 443. - # If not, add such a listen statement. + # For any listen statement, check if the machine also listens on + # Port 443. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces if port not in listens: @@ -590,8 +633,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir_to_ifmodssl( parser.get_aug_path( self.parser.loc["listen"]), "Listen", args) - self.save_notes += "Added Listen %s:%s directive to %s\n" % ( - ip, port, self.parser.loc["listen"]) + self.save_notes += ("Added Listen %s:%s directive to " + "%s\n") % (ip, port, + self.parser.loc["listen"]) listens.append("%s:%s" % (ip, port)) def prepare_https_modules(self, temp): @@ -602,11 +646,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ if self.conf("handle-modules"): - if "ssl_module" not in self.parser.modules: - self.enable_mod("ssl", temp=temp) if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) + if "ssl_module" not in self.parser.modules: + self.enable_mod("ssl", temp=temp) def make_addrs_sni_ready(self, addrs): """Checks to see if the server is ready for SNI challenges. @@ -651,7 +695,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Reload augeas to take into account the new vhost self.aug.load() - # Get Vhost augeas path for new vhost vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % (ssl_fp, parser.case_i("VirtualHost"))) @@ -668,6 +711,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add directives self._add_dummy_ssl_directives(vh_p) + self.save() # Log actions and create save notes logger.info("Created an SSL vhost at %s", ssl_fp) @@ -697,6 +741,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): else: return non_ssl_vh_fp + self.conf("le_vhost_ext") + def _sift_line(self, line): + """Decides whether a line should be copied to a SSL vhost. + + A canonical example of when sifting a line is required: + When the http vhost contains a RewriteRule that unconditionally + redirects any request to the https version of the same site. + e.g: + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,QSA,R=permanent] + Copying the above line to the ssl vhost would cause a + redirection loop. + + :param str line: a line extracted from the http vhost. + + :returns: True - don't copy line from http vhost to SSL vhost. + :rtype: bool + + """ + if not line.lstrip().startswith("RewriteRule"): + return False + + # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html + # The syntax of a RewriteRule is: + # RewriteRule pattern target [Flag1,Flag2,Flag3] + # i.e. target is required, so it must exist. + target = line.split()[2].strip() + + # target may be surrounded with quotes + if target[0] in ("'", '"') and target[0] == target[-1]: + target = target[1:-1] + + # Sift line if it redirects the request to a HTTPS site + return target.startswith("https://") + def _copy_create_ssl_vhost_skeleton(self, avail_fp, ssl_fp): """Copies over existing Vhost with IfModule mod_ssl.c> skeleton. @@ -709,18 +786,38 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # First register the creation so that it is properly removed if # configuration is rolled back self.reverter.register_file_creation(False, ssl_fp) + sift = False try: with open(avail_fp, "r") as orig_file: with open(ssl_fp, "w") as new_file: new_file.write("\n") for line in orig_file: - new_file.write(line) + if self._sift_line(line): + if not sift: + new_file.write( + "# Some rewrite rules in this file were " + "were disabled on your HTTPS site,\n" + "# because they have the potential to " + "create redirection loops.\n") + sift = True + new_file.write("# " + line) + else: + new_file.write(line) new_file.write("\n") except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") raise errors.PluginError("Unable to write/read in make_vhost_ssl") + if sift: + reporter = zope.component.getUtility(interfaces.IReporter) + reporter.add_message( + "Some rewrite rules copied from {0} were disabled in the " + "vhost for your HTTPS site located at {1} because they have " + "the potential to create redirection loops.".format(avail_fp, + ssl_fp), + reporter.MEDIUM_PRIORITY) + def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() ssl_addr_p = self.aug.match(vh_path + "/arg") @@ -737,20 +834,25 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _clean_vhost(self, vhost): # remove duplicated or conflicting ssl directives self._deduplicate_directives(vhost.path, - ["SSLCertificateFile", "SSLCertificateKeyFile"]) + ["SSLCertificateFile", + "SSLCertificateKeyFile"]) # remove all problematic directives self._remove_directives(vhost.path, ["SSLCertificateChainFile"]) def _deduplicate_directives(self, vh_path, directives): for directive in directives: - while len(self.parser.find_dir(directive, None, vh_path, False)) > 1: - directive_path = self.parser.find_dir(directive, None, vh_path, False) + while len(self.parser.find_dir(directive, None, + vh_path, False)) > 1: + directive_path = self.parser.find_dir(directive, None, + vh_path, False) self.aug.remove(re.sub(r"/\w*$", "", directive_path[0])) def _remove_directives(self, vh_path, directives): for directive in directives: - while len(self.parser.find_dir(directive, None, vh_path, False)) > 0: - directive_path = self.parser.find_dir(directive, None, vh_path, False) + while len(self.parser.find_dir(directive, None, + vh_path, False)) > 0: + directive_path = self.parser.find_dir(directive, None, + vh_path, False) self.aug.remove(re.sub(r"/\w*$", "", directive_path[0])) def _add_dummy_ssl_directives(self, vh_path): @@ -760,6 +862,22 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "insert_key_file_path") self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) + def _add_servername_alias(self, target_name, vhost): + fp = vhost.filep + vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (fp, parser.case_i("VirtualHost"))) + if not vh_p: + return + vh_path = vh_p[0] + if (self.parser.find_dir("ServerName", target_name, start=vh_path, exclude=False) + or self.parser.find_dir("ServerAlias", target_name, start=vh_path, exclude=False)): + return + if not self.parser.find_dir("ServerName", None, start=vh_path, exclude=False): + self.parser.add_dir(vh_path, "ServerName", target_name) + else: + self.parser.add_dir(vh_path, "ServerAlias", target_name) + self._add_servernames(vhost) + def _add_name_vhost_if_necessary(self, vhost): """Add NameVirtualHost Directives if necessary for new vhost. @@ -775,20 +893,29 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # See if the exact address appears in any other vhost # Remember 1.1.1.1:* == 1.1.1.1 -> hence any() for addr in vhost.addrs: + # In Apache 2.2, when a NameVirtualHost directive is not + # set, "*" and "_default_" will conflict when sharing a port + addrs = set((addr,)) + if addr.get_addr() in ("*", "_default_"): + addrs.update(obj.Addr((a, addr.get_port(),)) + for a in ("*", "_default_")) + for test_vh in self.vhosts: if (vhost.filep != test_vh.filep and - any(test_addr == addr for test_addr in test_vh.addrs) and + any(test_addr in addrs for + test_addr in test_vh.addrs) and not self.is_name_vhost(addr)): self.add_name_vhost(addr) logger.info("Enabling NameVirtualHosts on %s", addr) need_to_save = True + break if need_to_save: self.save() - ############################################################################ + ###################################################################### # Enhancements - ############################################################################ + ###################################################################### def supported_enhancements(self): # pylint: disable=no-self-use """Returns currently supported enhancements.""" return ["redirect", "ensure-http-header"] @@ -849,14 +976,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Add directives to server self.parser.add_dir(ssl_vhost.path, "Header", - constants.HEADER_ARGS[header_substring]) + constants.HEADER_ARGS[header_substring]) self.save_notes += ("Adding %s header to ssl vhost in %s\n" % - (header_substring, ssl_vhost.filep)) + (header_substring, ssl_vhost.filep)) self.save() logger.info("Adding %s header to ssl vhost in %s", header_substring, - ssl_vhost.filep) + ssl_vhost.filep) def _verify_no_matching_http_header(self, ssl_vhost, header_substring): """Checks to see if an there is an existing Header directive that @@ -876,14 +1003,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): header_substring exists """ - header_path = self.parser.find_dir("Header", None, start=ssl_vhost.path) + header_path = self.parser.find_dir("Header", None, + start=ssl_vhost.path) if header_path: # "Existing Header directive for virtualhost" pat = '(?:[ "]|^)(%s)(?:[ "]|$)' % (header_substring.lower()) for match in header_path: if re.search(pat, self.aug.get(match).lower()): raise errors.PluginEnhancementAlreadyPresent( - "Existing %s header" % (header_substring)) + "Existing %s header" % (header_substring)) def _enable_redirect(self, ssl_vhost, unused_options): """Redirect all equivalent HTTP traffic to ssl_vhost. @@ -932,7 +1060,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Check if LetsEncrypt redirection already exists self._verify_no_letsencrypt_redirect(general_vh) - # Note: if code flow gets here it means we didn't find the exact # letsencrypt RewriteRule config for redirection. Finding # another RewriteRule is likely to be fine in most or all cases, @@ -951,10 +1078,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if self.get_version() >= (2, 3, 9): self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS_WITH_END) + constants.REWRITE_HTTPS_ARGS_WITH_END) else: self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS) + constants.REWRITE_HTTPS_ARGS) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % (general_vh.filep, ssl_vhost.filep)) @@ -976,7 +1103,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): letsencrypt redirection WriteRule exists in virtual host. """ rewrite_path = self.parser.find_dir( - "RewriteRule", None, start=vhost.path) + "RewriteRule", None, start=vhost.path) # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value @@ -991,12 +1118,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if rewrite_args_dict: redirect_args = [constants.REWRITE_HTTPS_ARGS, - constants.REWRITE_HTTPS_ARGS_WITH_END] + constants.REWRITE_HTTPS_ARGS_WITH_END] for matches in rewrite_args_dict.values(): if [self.aug.get(x) for x in matches] in redirect_args: raise errors.PluginEnhancementAlreadyPresent( - "Let's Encrypt has already enabled redirection") + "Let's Encrypt has already enabled redirection") def _is_rewrite_exists(self, vhost): """Checks if there exists a RewriteRule directive in vhost @@ -1009,7 +1136,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ rewrite_path = self.parser.find_dir( - "RewriteRule", None, start=vhost.path) + "RewriteRule", None, start=vhost.path) return bool(rewrite_path) def _is_rewrite_engine_on(self, vhost): @@ -1020,7 +1147,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ rewrite_engine_path = self.parser.find_dir("RewriteEngine", "on", - start=vhost.path) + start=vhost.path) if rewrite_engine_path: return self.parser.get_arg(rewrite_engine_path[0]) return False @@ -1066,7 +1193,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): else: rewrite_rule_args = constants.REWRITE_HTTPS_ARGS - return ("\n" "%s \n" "%s \n" @@ -1078,7 +1204,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "ErrorLog /var/log/apache2/redirect.error.log\n" "LogLevel warn\n" "\n" - % (" ".join(str(addr) for addr in self._get_proposed_addrs(ssl_vhost)), + % (" ".join(str(addr) for + addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args))) @@ -1092,7 +1219,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.conf("vhost-root"), redirect_filename) + redirect_filepath = os.path.join(self.conf("vhost-root"), + redirect_filename) # Register the new file that will be created # Note: always register the creation before writing to ensure file will @@ -1120,7 +1248,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return None - def _get_proposed_addrs(self, vhost, port="80"): # pylint: disable=no-self-use + def _get_proposed_addrs(self, vhost, port="80"): """Return all addrs of vhost with the port replaced with the specified. :param obj.VirtualHost ssl_vhost: Original Vhost @@ -1200,7 +1328,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): .. note:: Does not make sure that the site correctly works or that all modules are enabled appropriately. - .. todo:: This function should number subdomains before the domain vhost + .. todo:: This function should number subdomains before the domain + vhost .. todo:: Make sure link is not broken... @@ -1485,4 +1614,4 @@ def install_ssl_options_conf(options_ssl): # Check to make sure options-ssl.conf is installed if not os.path.isfile(options_ssl): - shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) + shutil.copyfile(constants.os_constant("MOD_SSL_CONF_SRC"), options_ssl) diff --git a/letsencrypt-apache/letsencrypt_apache/constants.py b/letsencrypt-apache/letsencrypt_apache/constants.py index 8ac88b197..77f71a461 100644 --- a/letsencrypt-apache/letsencrypt_apache/constants.py +++ b/letsencrypt-apache/letsencrypt_apache/constants.py @@ -16,7 +16,9 @@ CLI_DEFAULTS_DEBIAN = dict( le_vhost_ext="-le-ssl.conf", handle_mods=True, handle_sites=True, - challenge_location="/etc/apache2" + challenge_location="/etc/apache2", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "options-ssl-apache.conf") ) CLI_DEFAULTS_CENTOS = dict( server_root="/etc/httpd", @@ -31,7 +33,9 @@ CLI_DEFAULTS_CENTOS = dict( le_vhost_ext="-le-ssl.conf", handle_mods=False, handle_sites=False, - challenge_location="/etc/httpd/conf.d" + challenge_location="/etc/httpd/conf.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "centos-options-ssl-apache.conf") ) CLI_DEFAULTS_GENTOO = dict( server_root="/etc/apache2", @@ -46,7 +50,26 @@ CLI_DEFAULTS_GENTOO = dict( le_vhost_ext="-le-ssl.conf", handle_mods=False, handle_sites=False, - challenge_location="/etc/apache2/vhosts.d" + challenge_location="/etc/apache2/vhosts.d", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "options-ssl-apache.conf") +) +CLI_DEFAULTS_DARWIN = dict( + server_root="/etc/apache2", + vhost_root="/etc/apache2/other", + vhost_files="*.conf", + version_cmd=['/usr/sbin/httpd', '-v'], + define_cmd=['/usr/sbin/httpd', '-t', '-D', 'DUMP_RUN_CFG'], + restart_cmd=['apachectl', 'graceful'], + conftest_cmd=['apachectl', 'configtest'], + enmod=None, + dismod=None, + le_vhost_ext="-le-ssl.conf", + handle_mods=False, + handle_sites=False, + challenge_location="/etc/apache2/other", + MOD_SSL_CONF_SRC=pkg_resources.resource_filename( + "letsencrypt_apache", "options-ssl-apache.conf") ) CLI_DEFAULTS = { "debian": CLI_DEFAULTS_DEBIAN, @@ -55,25 +78,22 @@ CLI_DEFAULTS = { "centos linux": CLI_DEFAULTS_CENTOS, "fedora": CLI_DEFAULTS_CENTOS, "red hat enterprise linux server": CLI_DEFAULTS_CENTOS, - "gentoo base system": CLI_DEFAULTS_GENTOO + "gentoo base system": CLI_DEFAULTS_GENTOO, + "darwin": CLI_DEFAULTS_DARWIN, } """CLI defaults.""" MOD_SSL_CONF_DEST = "options-ssl-apache.conf" """Name of the mod_ssl config file as saved in `IConfig.config_dir`.""" -MOD_SSL_CONF_SRC = pkg_resources.resource_filename( - "letsencrypt_apache", "options-ssl-apache.conf") -"""Path to the Apache mod_ssl config file found in the Let's Encrypt -distribution.""" - AUGEAS_LENS_DIR = pkg_resources.resource_filename( "letsencrypt_apache", "augeas_lens") """Path to the Augeas lens directory""" REWRITE_HTTPS_ARGS = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] -"""Apache version<2.3.9 rewrite rule arguments used for redirections to https vhost""" +"""Apache version<2.3.9 rewrite rule arguments used for redirections to +https vhost""" REWRITE_HTTPS_ARGS_WITH_END = [ "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"] @@ -81,14 +101,14 @@ REWRITE_HTTPS_ARGS_WITH_END = [ https vhost""" HSTS_ARGS = ["always", "set", "Strict-Transport-Security", - "\"max-age=31536000\""] + "\"max-age=31536000\""] """Apache header arguments for HSTS""" UIR_ARGS = ["always", "set", "Content-Security-Policy", - "upgrade-insecure-requests"] + "upgrade-insecure-requests"] HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS, - "Upgrade-Insecure-Requests": UIR_ARGS} + "Upgrade-Insecure-Requests": UIR_ARGS} def os_constant(key): diff --git a/letsencrypt-apache/letsencrypt_apache/display_ops.py b/letsencrypt-apache/letsencrypt_apache/display_ops.py index 45c55f49a..6a2308d73 100644 --- a/letsencrypt-apache/letsencrypt_apache/display_ops.py +++ b/letsencrypt-apache/letsencrypt_apache/display_ops.py @@ -4,6 +4,7 @@ import os import zope.component +from letsencrypt import errors from letsencrypt import interfaces import letsencrypt.display.util as display_util @@ -78,11 +79,18 @@ def _vhost_menu(domain, vhosts): name_size=disp_name_size) ) - code, tag = zope.component.getUtility(interfaces.IDisplay).menu( - "We were unable to find a vhost with a ServerName or Address of {0}.{1}" - "Which virtual host would you like to choose?".format( - domain, os.linesep), - choices, help_label="More Info", ok_label="Select") + try: + code, tag = zope.component.getUtility(interfaces.IDisplay).menu( + "We were unable to find a vhost with a ServerName " + "or Address of {0}.{1}Which virtual host would you " + "like to choose?".format(domain, os.linesep), + choices, help_label="More Info", ok_label="Select") + except errors.MissingCommandlineFlag as e: + msg = ("Failed to run Apache plugin non-interactively{1}{0}{1}" + "(The best solution is to add ServerName or ServerAlias " + "entries to the VirtualHost directives of your apache " + "configuration files.)".format(e, os.linesep)) + raise errors.MissingCommandlineFlag(msg) return code, tag diff --git a/letsencrypt-apache/letsencrypt_apache/parser.py b/letsencrypt-apache/letsencrypt_apache/parser.py index 82effad2b..3c13aae5f 100644 --- a/letsencrypt-apache/letsencrypt_apache/parser.py +++ b/letsencrypt-apache/letsencrypt_apache/parser.py @@ -96,11 +96,12 @@ class ApacheParser(object): def update_runtime_variables(self): """" - .. note:: Compile time variables (apache2ctl -V) are not used within the - dynamic configuration files. These should not be parsed or + .. note:: Compile time variables (apache2ctl -V) are not used within + the dynamic configuration files. These should not be parsed or interpreted. - .. todo:: Create separate compile time variables... simply for arg_get() + .. todo:: Create separate compile time variables... + simply for arg_get() """ stdout = self._get_runtime_cfg() @@ -177,7 +178,8 @@ class ApacheParser(object): # Make sure we don't cause an IndexError (end of list) # Check to make sure arg + 1 doesn't exist if (i == (len(matches) - 1) or - not matches[i + 1].endswith("/arg[%d]" % (args + 1))): + not matches[i + 1].endswith("/arg[%d]" % + (args + 1))): filtered.append(matches[i][:-len("/arg[%d]" % args)]) return filtered @@ -311,8 +313,6 @@ class ApacheParser(object): for match in matches: dir_ = self.aug.get(match).lower() if dir_ == "include" or dir_ == "includeoptional": - # start[6:] to strip off /files - #print self._get_include_path(self.get_arg(match +"/arg")), directive, arg ordered_matches.extend(self.find_dir( directive, arg, self._get_include_path(self.get_arg(match + "/arg")), @@ -331,8 +331,8 @@ class ApacheParser(object): """ value = self.aug.get(match) - # No need to strip quotes for variables, as apache2ctl already does this - # but we do need to strip quotes for all normal arguments. + # No need to strip quotes for variables, as apache2ctl already does + # this, but we do need to strip quotes for all normal arguments. # Note: normal argument may be a quoted variable # e.g. strip now, not later @@ -454,7 +454,7 @@ class ApacheParser(object): https://apr.apache.org/docs/apr/2.0/apr__fnmatch_8h_source.html http://apache2.sourcearchive.com/documentation/2.2.16-6/apr__fnmatch_8h_source.html - :param str clean_fn_match: Apache style filename match, similar to globs + :param str clean_fn_match: Apache style filename match, like globs :returns: regex suitable for augeas :rtype: str @@ -597,7 +597,7 @@ class ApacheParser(object): .. todo:: Make sure that files are included """ - default = self._set_user_config_file() + default = self.loc["root"] temp = os.path.join(self.root, "ports.conf") if os.path.isfile(temp): @@ -618,23 +618,6 @@ class ApacheParser(object): raise errors.NoInstallationError("Could not find configuration root") - def _set_user_config_file(self): - """Set the appropriate user configuration file - - .. todo:: This will have to be updated for other distros versions - - :param str root: pathname which contains the user config - - """ - # Basic check to see if httpd.conf exists and - # in hierarchy via direct include - # httpd.conf was very common as a user file in Apache 2.2 - if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and - self.find_dir("Include", "httpd.conf", self.loc["root"])): - return os.path.join(self.root, "httpd.conf") - else: - return os.path.join(self.root, "apache2.conf") - def case_i(string): """Returns case insensitive regex. diff --git a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py index 7099c388f..1fc5281c1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py @@ -96,7 +96,8 @@ class ComplexParserTest(util.ParserTest): else: self.assertFalse(self.parser.find_dir("FNMATCH_DIRECTIVE")) - # NOTE: Only run one test per function otherwise you will have inf recursion + # NOTE: Only run one test per function otherwise you will have + # inf recursion def test_include(self): self.verify_fnmatch("test_fnmatch.?onf") @@ -104,7 +105,8 @@ class ComplexParserTest(util.ParserTest): self.verify_fnmatch("../complex_parsing/[te][te]st_*.?onf") def test_include_fullpath(self): - self.verify_fnmatch(os.path.join(self.config_path, "test_fnmatch.conf")) + self.verify_fnmatch(os.path.join(self.config_path, + "test_fnmatch.conf")) def test_include_fullpath_trailing_slash(self): self.verify_fnmatch(self.config_path + "//") diff --git a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py index 9838b4f52..5b15a20d1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py @@ -35,10 +35,10 @@ class TwoVhost80Test(util.ApacheTest): def mock_deploy_cert(self, config): """A test for a mock deploy cert""" self.config.real_deploy_cert = self.config.deploy_cert + def mocked_deploy_cert(*args, **kwargs): """a helper to mock a deployed cert""" - with mock.patch( - "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): config.real_deploy_cert(*args, **kwargs) self.config.deploy_cert = mocked_deploy_cert return self.config @@ -65,6 +65,16 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises( errors.NotSupportedError, self.config.prepare) + @mock.patch("letsencrypt_apache.parser.ApacheParser") + @mock.patch("letsencrypt_apache.configurator.le_util.exe_exists") + def test_prepare_old_aug(self, mock_exe_exists, _): + mock_exe_exists.return_value = True + self.config.config_test = mock.Mock() + # pylint: disable=protected-access + self.config._check_aug_version = mock.Mock(return_value=False) + self.assertRaises( + errors.NotSupportedError, self.config.prepare) + def test_add_parser_arguments(self): # pylint: disable=no-self-use from letsencrypt_apache.configurator import ApacheConfigurator # Weak test.. @@ -101,8 +111,8 @@ class TwoVhost80Test(util.ApacheTest): def test_add_servernames_alias(self): self.config.parser.add_dir( self.vh_truth[2].path, "ServerAlias", ["*.le.co"]) - self.config._add_servernames(self.vh_truth[2]) # pylint: disable=protected-access - + # pylint: disable=protected-access + self.config._add_servernames(self.vh_truth[2]) self.assertEqual( self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) @@ -128,20 +138,10 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(found, 6) # Handle case of non-debian layout get_virtual_hosts - orig_conf = self.config.conf with mock.patch( "letsencrypt_apache.configurator.ApacheConfigurator.conf" - ) as mock_conf: - def conf_sideeffect(key): - """Handle calls to configurator.conf() - :param key: configuration key - :return: configuration value - """ - if key == "handle-sites": - return False - else: - return orig_conf(key) - mock_conf.side_effect = conf_sideeffect + ) as mock_conf: + mock_conf.return_value = False vhs = self.config.get_virtual_hosts() self.assertEqual(len(vhs), 6) @@ -161,6 +161,7 @@ class TwoVhost80Test(util.ApacheTest): def test_choose_vhost_select_vhost_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[0] chosen_vhost = self.config.choose_vhost("none.com") + self.vh_truth[0].aliases.add("none.com") self.assertEqual( self.vh_truth[0].get_names(), chosen_vhost.get_names()) @@ -178,7 +179,8 @@ class TwoVhost80Test(util.ApacheTest): def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[3] conflicting_vhost = obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:443")]), True, True) + "path", "aug_path", set([obj.Addr.fromstring("*:443")]), + True, True) self.config.vhosts.append(conflicting_vhost) self.assertRaises( @@ -191,13 +193,14 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual( self.vh_truth[0], self.config._find_best_vhost("encryption-example.demo")) - self.assertTrue( - self.config._find_best_vhost("does-not-exist.com") is None) + self.assertEqual( + self.config._find_best_vhost("does-not-exist.com"), None) def test_find_best_vhost_variety(self): # pylint: disable=protected-access ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))]), True, False) self.config.vhosts.append(ssl_vh) self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) @@ -278,7 +281,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) self.config.parser.modules.add("ssl_module") self.config.parser.modules.add("mod_ssl.c") @@ -296,7 +300,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("ssl_module" in self.config.parser.modules) loc_cert = self.config.parser.find_dir( - "sslcertificatefile", "example/fullchain.pem", self.vh_truth[1].path) + "sslcertificatefile", "example/fullchain.pem", + self.vh_truth[1].path) loc_key = self.config.parser.find_dir( "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path) @@ -311,7 +316,8 @@ class TwoVhost80Test(util.ApacheTest): def test_deploy_cert_newssl_no_fullchain(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") @@ -321,11 +327,13 @@ class TwoVhost80Test(util.ApacheTest): self.config.assoc["random.demo"] = self.vh_truth[1] self.assertRaises(errors.PluginError, lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", "example/key.pem")) + "random.demo", "example/cert.pem", + "example/key.pem")) def test_deploy_cert_old_apache_no_chain(self): self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) + self.config_path, self.vhost_path, self.config_dir, + self.work_dir, version=(2, 4, 7)) self.config = self.mock_deploy_cert(self.config) self.config.parser.modules.add("ssl_module") @@ -335,7 +343,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.assoc["random.demo"] = self.vh_truth[1] self.assertRaises(errors.PluginError, lambda: self.config.deploy_cert( - "random.demo", "example/cert.pem", "example/key.pem")) + "random.demo", "example/cert.pem", + "example/key.pem")) def test_deploy_cert(self): self.config.parser.modules.add("ssl_module") @@ -420,9 +429,15 @@ class TwoVhost80Test(util.ApacheTest): self.config.parser.add_dir_to_ifmodssl = mock_add_dir self.config.prepare_server_https("443") + # Changing the order these modules are enabled breaks the reverter + self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb") + self.assertEqual(mock_enable.call_args[0][0], "ssl") self.assertEqual(mock_enable.call_args[1], {"temp": False}) self.config.prepare_server_https("8080", temp=True) + # Changing the order these modules are enabled breaks the reverter + self.assertEqual(mock_enable.call_args_list[2][0][0], "socache_shmcb") + self.assertEqual(mock_enable.call_args[0][0], "ssl") # Enable mod is temporary self.assertEqual(mock_enable.call_args[1], {"temp": True}) @@ -443,7 +458,8 @@ class TwoVhost80Test(util.ApacheTest): # Test Listen statements with specific ip listeed self.config.prepare_server_https("443") - # Should only be 2 here, as the third interface already listens to the correct port + # Should only be 2 here, as the third interface + # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 2) # Check argument to new Listen statements @@ -457,9 +473,12 @@ class TwoVhost80Test(util.ApacheTest): # Test self.config.prepare_server_https("8080", temp=True) self.assertEqual(mock_add_dir.call_count, 3) - self.assertEqual(mock_add_dir.call_args_list[0][0][2], ["1.2.3.4:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[0][0][2], + ["1.2.3.4:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[1][0][2], + ["[::1]:8080", "https"]) + self.assertEqual(mock_add_dir.call_args_list[2][0][2], + ["1.1.1.1:8080", "https"]) def test_prepare_server_https_mixed_listen(self): @@ -477,7 +496,8 @@ class TwoVhost80Test(util.ApacheTest): # Test Listen statements with specific ip listeed self.config.prepare_server_https("443") - # Should only be 2 here, as the third interface already listens to the correct port + # Should only be 2 here, as the third interface + # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 0) def test_make_vhost_ssl(self): @@ -511,7 +531,8 @@ class TwoVhost80Test(util.ApacheTest): for directive in ["SSLCertificateFile", "SSLCertificateKeyFile", "SSLCertificateChainFile", "SSLCACertificatePath"]: for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, directive, ["bogus"]) + self.config.parser.add_dir(self.vh_truth[1].path, + directive, ["bogus"]) self.config.save() self.config._clean_vhost(self.vh_truth[1]) @@ -537,23 +558,24 @@ class TwoVhost80Test(util.ApacheTest): # pylint: disable=protected-access DIRECTIVE = "Foo" for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, DIRECTIVE, ["bar"]) + self.config.parser.add_dir(self.vh_truth[1].path, + DIRECTIVE, ["bar"]) self.config.save() self.config._deduplicate_directives(self.vh_truth[1].path, [DIRECTIVE]) self.config.save() self.assertEqual( - len(self.config.parser.find_dir( - DIRECTIVE, None, self.vh_truth[1].path, False)), - 1) + len(self.config.parser.find_dir( + DIRECTIVE, None, self.vh_truth[1].path, False)), 1) def test_remove_directives(self): # pylint: disable=protected-access DIRECTIVES = ["Foo", "Bar"] for directive in DIRECTIVES: for _ in range(10): - self.config.parser.add_dir(self.vh_truth[1].path, directive, ["baz"]) + self.config.parser.add_dir(self.vh_truth[1].path, + directive, ["baz"]) self.config.save() self.config._remove_directives(self.vh_truth[1].path, DIRECTIVES) @@ -561,9 +583,8 @@ class TwoVhost80Test(util.ApacheTest): for directive in DIRECTIVES: self.assertEqual( - len(self.config.parser.find_dir( - directive, None, self.vh_truth[1].path, False)), - 0) + len(self.config.parser.find_dir( + directive, None, self.vh_truth[1].path, False)), 0) def test_make_vhost_ssl_extra_vhs(self): self.config.aug.match = mock.Mock(return_value=["p1", "p2"]) @@ -592,6 +613,14 @@ class TwoVhost80Test(util.ApacheTest): self.config._add_name_vhost_if_necessary(self.vh_truth[0]) self.assertTrue(self.config.save.called) + new_addrs = set() + for addr in self.vh_truth[0].addrs: + new_addrs.add(obj.Addr(("_default_", addr.get_port(),))) + + self.vh_truth[0].addrs = new_addrs + self.config._add_name_vhost_if_necessary(self.vh_truth[0]) + self.assertEqual(self.config.save.call_count, 2) + @mock.patch("letsencrypt_apache.configurator.tls_sni_01.ApacheTlsSni01.perform") @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart") def test_perform(self, mock_restart, mock_perform): @@ -652,7 +681,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertRaises(errors.PluginError, self.config.get_version) mock_script.return_value = ( - "Server Version: Apache/2.3{0} Apache/2.4.7".format(os.linesep), "") + "Server Version: Apache/2.3{0} Apache/2.4.7".format( + os.linesep), "") self.assertRaises(errors.PluginError, self.config.get_version) mock_script.side_effect = errors.SubprocessError("Can't find program") @@ -676,7 +706,8 @@ class TwoVhost80Test(util.ApacheTest): def test_config_test_bad_process(self, mock_run_script): mock_run_script.side_effect = errors.SubprocessError - self.assertRaises(errors.MisconfigurationError, self.config.config_test) + self.assertRaises(errors.MisconfigurationError, + self.config.config_test) def test_get_all_certs_keys(self): c_k = self.config.get_all_certs_keys() @@ -688,7 +719,8 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue("default-ssl" in path) def test_get_all_certs_keys_malformed_conf(self): - self.config.parser.find_dir = mock.Mock(side_effect=[["path"], [], ["path"], []]) + self.config.parser.find_dir = mock.Mock( + side_effect=[["path"], [], ["path"], []]) c_k = self.config.get_all_certs_keys() self.assertFalse(c_k) @@ -709,13 +741,13 @@ class TwoVhost80Test(util.ApacheTest): def test_supported_enhancements(self): self.assertTrue(isinstance(self.config.supported_enhancements(), list)) - @mock.patch("letsencrypt.le_util.exe_exists") def test_enhance_unknown_vhost(self, mock_exe): self.config.parser.modules.add("rewrite_module") mock_exe.return_value = True ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("satoshi.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("satoshi.com",))]), True, False) self.config.vhosts.append(ssl_vh) self.assertRaises( @@ -736,7 +768,7 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "ensure-http-header", - "Strict-Transport-Security") + "Strict-Transport-Security") self.assertTrue("headers_module" in self.config.parser.modules) @@ -746,7 +778,7 @@ class TwoVhost80Test(util.ApacheTest): # These are not immediately available in find_dir even with save() and # load(). They must be found in sites-available hsts_header = self.config.parser.find_dir( - "Header", None, ssl_vhost.path) + "Header", None, ssl_vhost.path) # four args to HSTS header self.assertEqual(len(hsts_header), 4) @@ -758,12 +790,12 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("encryption-example.demo", "ensure-http-header", - "Strict-Transport-Security") + "Strict-Transport-Security") self.assertRaises( errors.PluginEnhancementAlreadyPresent, - self.config.enhance, "encryption-example.demo", "ensure-http-header", - "Strict-Transport-Security") + self.config.enhance, "encryption-example.demo", + "ensure-http-header", "Strict-Transport-Security") @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -774,7 +806,7 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("letsencrypt.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") + "Upgrade-Insecure-Requests") self.assertTrue("headers_module" in self.config.parser.modules) @@ -784,7 +816,7 @@ class TwoVhost80Test(util.ApacheTest): # These are not immediately available in find_dir even with save() and # load(). They must be found in sites-available uir_header = self.config.parser.find_dir( - "Header", None, ssl_vhost.path) + "Header", None, ssl_vhost.path) # four args to HSTS header self.assertEqual(len(uir_header), 4) @@ -796,14 +828,12 @@ class TwoVhost80Test(util.ApacheTest): # This will create an ssl vhost for letsencrypt.demo self.config.enhance("encryption-example.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") + "Upgrade-Insecure-Requests") self.assertRaises( errors.PluginEnhancementAlreadyPresent, - self.config.enhance, "encryption-example.demo", "ensure-http-header", - "Upgrade-Insecure-Requests") - - + self.config.enhance, "encryption-example.demo", + "ensure-http-header", "Upgrade-Insecure-Requests") @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -837,7 +867,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["Unknown"]) - self.assertTrue(self.config._is_rewrite_exists(self.vh_truth[3])) # pylint: disable=protected-access + # pylint: disable=protected-access + self.assertTrue(self.config._is_rewrite_exists(self.vh_truth[3])) def test_rewrite_engine_exists(self): # Skip the enable mod @@ -845,8 +876,8 @@ class TwoVhost80Test(util.ApacheTest): self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteEngine", "on") - self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3])) # pylint: disable=protected-access - + # pylint: disable=protected-access + self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3])) @mock.patch("letsencrypt.le_util.run_script") @mock.patch("letsencrypt.le_util.exe_exists") @@ -857,7 +888,8 @@ class TwoVhost80Test(util.ApacheTest): # Create a preexisting rewrite rule self.config.parser.add_dir( - self.vh_truth[3].path, "RewriteRule", ["Unknown"]) + self.vh_truth[3].path, "RewriteRule", ["UnknownPattern", + "UnknownTarget"]) self.config.save() # This will create an ssl vhost for letsencrypt.demo @@ -872,18 +904,18 @@ class TwoVhost80Test(util.ApacheTest): self.assertEqual(len(rw_engine), 1) # three args to rw_rule + 1 arg for the pre existing rewrite - self.assertEqual(len(rw_rule), 4) + self.assertEqual(len(rw_rule), 5) self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path)) self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path)) self.assertTrue("rewrite_module" in self.config.parser.modules) - def test_redirect_with_conflict(self): self.config.parser.modules.add("rewrite_module") ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]), + "fp", "ap", set([obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))]), True, False) # No names ^ this guy should conflict. @@ -908,7 +940,8 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[1].name = "default.com" self.vh_truth[1].aliases = set(["yes.default.com"]) - self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access + # pylint: disable=protected-access + self.config._enable_redirect(self.vh_truth[1], "") self.assertEqual(len(self.config.vhosts), 7) def test_create_own_redirect_for_old_apache_version(self): @@ -918,9 +951,49 @@ class TwoVhost80Test(util.ApacheTest): self.vh_truth[1].name = "default.com" self.vh_truth[1].aliases = set(["yes.default.com"]) - self.config._enable_redirect(self.vh_truth[1], "") # pylint: disable=protected-access + # pylint: disable=protected-access + self.config._enable_redirect(self.vh_truth[1], "") self.assertEqual(len(self.config.vhosts), 7) + def test_sift_line(self): + # pylint: disable=protected-access + small_quoted_target = "RewriteRule ^ \"http://\"" + self.assertFalse(self.config._sift_line(small_quoted_target)) + + https_target = "RewriteRule ^ https://satoshi" + self.assertTrue(self.config._sift_line(https_target)) + + normal_target = "RewriteRule ^/(.*) http://www.a.com:1234/$1 [L,R]" + self.assertFalse(self.config._sift_line(normal_target)) + + @mock.patch("letsencrypt_apache.configurator.zope.component.getUtility") + def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): + self.config.parser.modules.add("rewrite_module") + + http_vhost = self.vh_truth[0] + + self.config.parser.add_dir( + http_vhost.path, "RewriteEngine", "on") + + self.config.parser.add_dir( + http_vhost.path, "RewriteRule", + ["^", + "https://%{SERVER_NAME}%{REQUEST_URI}", + "[L,QSA,R=permanent]"]) + self.config.save() + + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) + + self.assertTrue(self.config.parser.find_dir( + "RewriteEngine", "on", ssl_vhost.path, False)) + + conf_text = open(ssl_vhost.filep).read() + commented_rewrite_rule = ("# RewriteRule ^ " + "https://%{SERVER_NAME}%{REQUEST_URI} " + "[L,QSA,R=permanent]") + self.assertTrue(commented_rewrite_rule in conf_text) + mock_get_utility().add_message.assert_called_once_with(mock.ANY, + mock.ANY) def get_achalls(self): """Return testing achallenges.""" @@ -949,6 +1022,15 @@ class TwoVhost80Test(util.ApacheTest): self.assertTrue(self.config.parser.find_dir( "NameVirtualHost", "*:443", exclude=False)) + def test_aug_version(self): + mock_match = mock.Mock(return_value=["something"]) + self.config.aug.match = mock_match + # pylint: disable=protected-access + self.assertEquals(self.config._check_aug_version(), + ["something"]) + self.config.aug.match.side_effect = RuntimeError + self.assertFalse(self.config._check_aug_version()) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py index 6db319d87..f7fbac947 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py @@ -6,6 +6,7 @@ import mock import zope.component from letsencrypt.display import util as display_util +from letsencrypt import errors from letsencrypt_apache import obj @@ -31,6 +32,14 @@ class SelectVhostTest(unittest.TestCase): mock_util().menu.return_value = (display_util.OK, 3) self.assertEqual(self.vhosts[3], self._call(self.vhosts)) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") + def test_noninteractive(self, mock_util): + mock_util().menu.side_effect = errors.MissingCommandlineFlag("no vhost default") + try: + self._call(self.vhosts) + except errors.MissingCommandlineFlag as e: + self.assertTrue("VirtualHost directives" in e.message) + @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility") def test_more_info_cancel(self, mock_util): mock_util().menu.side_effect = [ diff --git a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py index 13eddaddf..a469702f1 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/obj_test.py @@ -47,7 +47,8 @@ class VirtualHostTest(unittest.TestCase): self.assertTrue(self.vhost1.conflicts([self.addr2])) self.assertFalse(self.vhost1.conflicts([self.addr_default])) - self.assertFalse(self.vhost2.conflicts([self.addr1, self.addr_default])) + self.assertFalse(self.vhost2.conflicts([self.addr1, + self.addr_default])) def test_same_server(self): from letsencrypt_apache.obj import VirtualHost diff --git a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py index 9b78bf6d6..2e6481aba 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/parser_test.py @@ -106,7 +106,7 @@ class BasicParserTest(util.ParserTest): def test_set_locations(self): with mock.patch("letsencrypt_apache.parser.os.path") as mock_path: - mock_path.isfile.side_effect = [True, False, False] + mock_path.isfile.side_effect = [False, False] # pylint: disable=protected-access results = self.parser._set_locations() @@ -114,15 +114,6 @@ class BasicParserTest(util.ParserTest): self.assertEqual(results["default"], results["listen"]) self.assertEqual(results["default"], results["name"]) - def test_set_user_config_file(self): - # pylint: disable=protected-access - path = os.path.join(self.parser.root, "httpd.conf") - open(path, 'w').close() - self.parser.add_dir(self.parser.loc["default"], "Include", "httpd.conf") - - self.assertEqual( - path, self.parser._set_user_config_file()) - @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg") def test_update_runtime_variables(self, mock_cfg): mock_cfg.return_value = ( diff --git a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py index b69406932..9681bf9fc 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py @@ -9,6 +9,8 @@ from letsencrypt.plugins import common_test from letsencrypt_apache import obj from letsencrypt_apache.tests import util +from six.moves import xrange # pylint: disable=redefined-builtin, import-error + class TlsSniPerformTest(util.ApacheTest): """Test the ApacheTlsSni01 challenge.""" @@ -58,7 +60,7 @@ class TlsSniPerformTest(util.ApacheTest): mock_setup_cert.assert_called_once_with(achall) - # Check to make sure challenge config path is included in apache config. + # Check to make sure challenge config path is included in apache config self.assertEqual( len(self.sni.configurator.parser.find_dir( "Include", self.sni.challenge_conf)), 1) @@ -78,8 +80,7 @@ class TlsSniPerformTest(util.ApacheTest): # pylint: disable=protected-access self.sni._setup_challenge_cert = mock_setup_cert - with mock.patch( - "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): + with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"): sni_responses = self.sni.perform() self.assertEqual(mock_setup_cert.call_count, 2) @@ -126,13 +127,15 @@ class TlsSniPerformTest(util.ApacheTest): def test_get_addrs_default(self): self.sni.configurator.choose_vhost = mock.Mock( return_value=obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("_default_:443")]), + "path", "aug_path", + set([obj.Addr.fromstring("_default_:443")]), False, False) ) + # pylint: disable=protected-access self.assertEqual( set([obj.Addr.fromstring("*:443")]), - self.sni._get_addrs(self.achalls[0])) # pylint: disable=protected-access + self.sni._get_addrs(self.achalls[0])) if __name__ == "__main__": diff --git a/letsencrypt-apache/letsencrypt_apache/tests/util.py b/letsencrypt-apache/letsencrypt_apache/tests/util.py index 798d4814b..97a11e851 100644 --- a/letsencrypt-apache/letsencrypt_apache/tests/util.py +++ b/letsencrypt-apache/letsencrypt_apache/tests/util.py @@ -33,7 +33,7 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods pkg="letsencrypt_apache.tests") self.ssl_options = common.setup_ssl_options( - self.config_dir, constants.MOD_SSL_CONF_SRC, + self.config_dir, constants.os_constant("MOD_SSL_CONF_SRC"), constants.MOD_SSL_CONF_DEST) self.config_path = os.path.join(self.temp_dir, config_root) @@ -42,6 +42,20 @@ class ApacheTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + # Make sure all vhosts in sites-enabled are symlinks (Python packaging + # does not preserve symlinks) + sites_enabled = os.path.join(self.config_path, "sites-enabled") + if not os.path.exists(sites_enabled): + return + + for vhost_basename in os.listdir(sites_enabled): + vhost = os.path.join(sites_enabled, vhost_basename) + if not os.path.islink(vhost): # pragma: no cover + os.remove(vhost) + target = os.path.join( + os.path.pardir, "sites-available", vhost_basename) + os.symlink(target, vhost) + class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods @@ -62,7 +76,8 @@ class ParserTest(ApacheTest): # pytlint: disable=too-few-public-methods def get_apache_configurator( - config_path, vhost_path, config_dir, work_dir, version=(2, 4, 7), conf=None): + config_path, vhost_path, + config_dir, work_dir, version=(2, 4, 7), conf=None): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -129,11 +144,13 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, "mod_macro-example.conf/Macro/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, modmacro=True), + set([obj.Addr.fromstring("*:80")]), False, True, + modmacro=True), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), - os.path.join(aug_pre, "default-ssl-port-only.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), True, False), + os.path.join(aug_pre, ("default-ssl-port-only.conf/" + "IfModule/VirtualHost")), + set([obj.Addr.fromstring("_default_:443")]), True, False) ] return vh_truth diff --git a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py index ca7985f35..cc1d749a0 100644 --- a/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py +++ b/letsencrypt-apache/letsencrypt_apache/tls_sni_01.py @@ -10,6 +10,7 @@ from letsencrypt_apache import parser logger = logging.getLogger(__name__) + class ApacheTlsSni01(common.TLSSNI01): """Class that performs TLS-SNI-01 challenges within the Apache configurator @@ -75,6 +76,7 @@ class ApacheTlsSni01(common.TLSSNI01): # Setup the configuration addrs = self._mod_config() + self.configurator.save("Don't lose mod_config changes", True) self.configurator.make_addrs_sni_ready(addrs) # Save reversible changes @@ -125,7 +127,8 @@ class ApacheTlsSni01(common.TLSSNI01): addrs.add(default_addr) else: addrs.add( - addr.get_sni_addr(self.configurator.config.tls_sni_01_port)) + addr.get_sni_addr( + self.configurator.config.tls_sni_01_port)) return addrs diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index a5c5e8a7a..a8e010f0e 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -63,4 +63,5 @@ setup( 'apache = letsencrypt_apache.configurator:ApacheConfigurator', ], }, + test_suite='letsencrypt_apache', ) diff --git a/letsencrypt-auto b/letsencrypt-auto deleted file mode 100755 index 20465dbb1..000000000 --- a/letsencrypt-auto +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/sh -e -# -# A script to run the latest release version of the Let's Encrypt in a -# virtual environment -# -# Installs and updates the letencrypt virtualenv, and runs letsencrypt -# using that virtual environment. This allows the client to function decently -# without requiring specific versions of its dependencies from the operating -# system. - -# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, -# if you want to change where the virtual environment will be installed -XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} -VENV_NAME="letsencrypt" -VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} -VENV_BIN=${VENV_PATH}/bin -# The path to the letsencrypt-auto script. Everything that uses these might -# at some point be inlined... -LEA_PATH=`dirname "$0"` -BOOTSTRAP=${LEA_PATH}/bootstrap - -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) -for arg in "$@" ; do - # This first clause is redundant with the third, but hedging on portability - if [ "$arg" = "-v" ] || [ "$arg" = "--verbose" ] || echo "$arg" | grep -E -- "-v+$" ; then - VERBOSE=1 - elif [ "$arg" = "--debug" ] ; then - DEBUG=1 - fi -done - -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation -# The "normal" case is that sudo is used for the steps that need root, but -# this script *can* be run as root (not recommended), or fall back to using -# `su` -if test "`id -u`" -ne "0" ; then - if command -v sudo 1>/dev/null 2>&1; then - SUDO=sudo - else - echo \"sudo\" is not available, will use \"su\" for installation steps... - # Because the parameters in `su -c` has to be a string, - # we need properly escape it - su_sudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" - } - SUDO=su_sudo - fi -else - SUDO= -fi - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$DEBUG" = 1 ] ; then - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi - else - echo "WARNING: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - exit 1 - fi -} - -DeterminePythonVersion() { - if command -v python2.7 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2.7} - elif command -v python27 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python27} - elif command -v python2 > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python2} - elif command -v python > /dev/null ; then - export LE_PYTHON=${LE_PYTHON:-python} - else - echo "Cannot find any Pythons... please install one!" - exit 1 - fi - - PYVER=`$LE_PYTHON --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` - if [ $PYVER -eq 26 ] ; then - ExperimentalBootstrap "Python 2.6" - elif [ $PYVER -lt 26 ] ; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." - exit 1 - fi -} - - -# virtualenv call is not idempotent: it overwrites pip upgraded in -# later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -d $VENV_PATH ] -then - if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 - fi - - if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh - elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh - elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh - elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi - elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" - elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" - elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" - elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root - elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" - else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" - fi - - DeterminePythonVersion - echo "Creating virtual environment..." - if [ "$VERBOSE" = 1 ] ; then - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH - else - virtualenv --no-site-packages --python $LE_PYTHON $VENV_PATH > /dev/null - fi -else - DeterminePythonVersion -fi - - -printf "Updating letsencrypt and virtual environment dependencies..." -if [ "$VERBOSE" = 1 ] ; then - echo - $VENV_BIN/pip install -U setuptools - $VENV_BIN/pip install -U pip - $VENV_BIN/pip install -U letsencrypt letsencrypt-apache - # nginx is buggy / disabled for now, but upgrade it if the user has - # installed it manually - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - $VENV_BIN/pip install -U letsencrypt letsencrypt-nginx - fi -else - $VENV_BIN/pip install -U setuptools > /dev/null - printf . - $VENV_BIN/pip install -U pip > /dev/null - printf . - # nginx is buggy / disabled for now... - $VENV_BIN/pip install -U letsencrypt > /dev/null - printf . - $VENV_BIN/pip install -U letsencrypt-apache > /dev/null - if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then - printf . - $VENV_BIN/pip install -U letsencrypt-nginx > /dev/null - fi - echo -fi - -# Explain what's about to happen, for the benefit of those getting sudo -# password prompts... -echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@" -$SUDO $VENV_BIN/letsencrypt "$@" diff --git a/letsencrypt-auto b/letsencrypt-auto new file mode 120000 index 000000000..af7e83a70 --- /dev/null +++ b/letsencrypt-auto @@ -0,0 +1 @@ +letsencrypt-auto-source/letsencrypt-auto \ No newline at end of file diff --git a/letsencrypt-auto-source/Dockerfile b/letsencrypt-auto-source/Dockerfile new file mode 100644 index 000000000..fd7fe4851 --- /dev/null +++ b/letsencrypt-auto-source/Dockerfile @@ -0,0 +1,32 @@ +# For running tests, build a docker image with a passwordless sudo and a trust +# store we can manipulate. + +FROM ubuntu:trusty + +# Add an unprivileged user: +RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea + +# Let that user sudo: +RUN sed -i.bkp -e \ + 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \ + /etc/sudoers + +# Install pip and nose: +RUN apt-get update && \ + apt-get -q -y install python-pip && \ + apt-get clean +RUN pip install nose + +RUN mkdir -p /home/lea/letsencrypt/letsencrypt + +# Install fake testing CA: +COPY ./tests/certs/ca/my-root-ca.crt.pem /usr/local/share/ca-certificates/ +RUN update-ca-certificates + +# Copy code: +COPY . /home/lea/letsencrypt/letsencrypt-auto-source + +USER lea +WORKDIR /home/lea + +CMD ["nosetests", "-v", "-s", "letsencrypt/letsencrypt-auto-source/tests"] diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py new file mode 100755 index 000000000..9a5fc46a7 --- /dev/null +++ b/letsencrypt-auto-source/build.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +"""Stitch together the letsencrypt-auto script. + +Implement a simple templating language in which {{ some/file }} turns into the +contents of the file at ./pieces/some/file except for certain tokens which have +other, special definitions. + +""" +from os.path import abspath, dirname, join +import re +from sys import argv + + +DIR = dirname(abspath(__file__)) + + +def le_version(build_script_dir): + """Return the version number stamped in letsencrypt/__init__.py.""" + return re.search('''^__version__ = ['"](.+)['"].*''', + file_contents(join(dirname(build_script_dir), + 'letsencrypt', + '__init__.py')), + re.M).group(1) + + +def file_contents(path): + with open(path) as file: + return file.read() + + +def build(version=None, requirements=None): + """Return the built contents of the letsencrypt-auto script. + + :arg version: The version to attach to the script. Default: the version of + the letsencrypt package + :arg requirements: The contents of the requirements file to embed. Default: + contents of letsencrypt-auto-requirements.txt + + """ + special_replacements = { + 'LE_AUTO_VERSION': version or le_version(DIR) + } + if requirements: + special_replacements['letsencrypt-auto-requirements.txt'] = requirements + + def replacer(match): + token = match.group(1) + if token in special_replacements: + return special_replacements[token] + else: + return file_contents(join(DIR, 'pieces', token)) + + return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', + replacer, + file_contents(join(DIR, 'letsencrypt-auto.template'))) + + +def main(): + with open(join(DIR, 'letsencrypt-auto'), 'w') as out: + out.write(build()) + + +if __name__ == '__main__': + main() diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto new file mode 100755 index 000000000..b542c8d3e --- /dev/null +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -0,0 +1,1821 @@ +#!/bin/sh +# +# Download and run the latest release version of the Let's Encrypt client. +# +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* + +set -e # Work even if somebody does "sh thisscript.sh". + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN="$VENV_PATH/bin" +LE_AUTO_VERSION="0.5.0.dev0" + +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + case "$arg" in + --debug) + DEBUG=1;; + --os-packages-only) + OS_PACKAGES_ONLY=1;; + --no-self-upgrade) + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1;; + --verbose) + VERBOSE=1;; + [!-]*|-*[!v]*|-) + # Anything that isn't -v, -vv, etc.: that is, anything that does not + # start with a -, contains anything that's not a v, or is just "-" + ;; + *) # -v+ remains. + VERBOSE=1;; + esac +done + +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + command -v "$LE_PYTHON" > /dev/null && break + done + if [ "$?" != "0" ]; then + echo "Cannot find any Pythons; please install one!" + exit 1 + fi + export LE_PYTHON + + PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ "$PYVER" -lt 26 ]; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null 2>&1; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null 2>&1; then + virtualenv="$virtualenv python-virtualenv" + fi + + augeas_pkg="libaugeas0 augeas-lenses" + AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + + } + + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs + fi + + $SUDO apt-get install -y --no-install-recommends \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + $augeas_pkg \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if ! $SUDO $tool install -y \ + python \ + python-devel \ + python-virtualenv \ + python-tools \ + python-pip + then + if ! $SUDO $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv \ + python27-tools \ + python27-pip + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + fi + + if ! $SUDO $tool install -y \ + gcc \ + dialog \ + augeas-libs \ + openssl \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi + + + if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then + if ! $SUDO $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi + fi +} + +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + $SUDO zypper -nq in -l \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates +} + +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./tools/_venv_common.sh + + deps=" + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + missing=$("$SUDO" pacman -T $deps) + + if [ "$missing" ]; then + "$SUDO" pacman -S --needed $missing + fi +} + +BootstrapGentooCommon() { + PACKAGES=" + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + "$SUDO" cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + "$SUDO" pmerge --noreplace --oneshot $PACKAGES + ;; + (portage|*) + "$SUDO" emerge --noreplace --oneshot $PACKAGES + ;; + esac +} + +BootstrapFreeBsd() { + "$SUDO" pkg install -Ay \ + python \ + py27-virtualenv \ + augeas \ + libffi +} + +BootstrapMac() { + if ! hash brew 2>/dev/null; then + echo "Homebrew not installed.\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + if [ -z "$(brew list --versions augeas)" ]; then + echo "augeas not installed.\nInstalling augeas from Homebrew..." + brew install augeas + fi + + if [ -z "$(brew list --versions dialog)" ]; then + echo "dialog not installed.\nInstalling dialog from Homebrew..." + brew install dialog + fi + + if ! hash pip 2>/dev/null; then + echo "pip not installed.\nInstalling python from Homebrew..." + brew install python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv not installed.\nInstalling with pip..." + pip install virtualenv + fi +} + + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ]; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/redhat-release ]; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchCommon + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ]; then + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon + elif [ -f /etc/gentoo-release ]; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a peep install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + fi +} + +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + + + +if [ "$NO_SELF_UPGRADE" = 1 ]; then + # Phase 2: Create venv, install LE, and run. + + if [ -f "$VENV_BIN/letsencrypt" ]; then + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) + else + INSTALLED_VERSION="none" + fi + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then + echo "Creating virtual environment..." + DeterminePythonVersion + rm -rf "$VENV_PATH" + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi + + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + # There is no $ interpolation due to quotes on starting heredoc delimiter. + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" +# This is the flattened list of packages letsencrypt-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# then use `hashin` or a more secure method to gather the hashes. + +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: U8HJ3bMEMVE-t_PN7wo-BrDxJSGIqqd0SvD1pM1F268 +# sha256: pWj0nfyhKo2fNwGHJX78WKOBCeHu5xTZKFYdegGKZPg +# sha256: gJxsqM-8ruv71DK0V2ABtA04_yRjdzy1dXfXXhoCC8M +# sha256: hs3KLNnLpBQiIwOQ3xff6qnzRKkR45dci-naV7NVSOk +# sha256: JLE9uErsOFyiPHuN7YPvi7QXe8GB0UdY-fl1vl0CDYY +# sha256: lprv_XwOCX9r4e_WgsFWriJlkaB5OpS2wtXkKT9MjU4 +# sha256: AA81jUsPokn-qrnBzn1bL-fgLnvfaAbCZBhQX8aF4mg +# sha256: qdhvRgu9g1ii1ROtd54_P8h447k6ALUAL66_YW_-a5w +# sha256: MSezqzPrI8ysBx-aCAJ0jlz3xcvNAkgrsGPjW0HbsLA +# sha256: 4rLUIjZGmkAiTTnntsYFdfOIsvQj81TH7pClt_WMgGU +# sha256: jC3Mr-6JsbQksL7GrS3ZYiyUnSAk6Sn12h7YAerHXx0 +# sha256: pN56TRGu1Ii6tPsU9JiFh6gpvs5aIEM_eA1uM7CAg8s +# sha256: XKj-MEJSZaSSdOSwITobyY9LE0Sa5elvmEdx5dg-WME +# sha256: pP04gC9Z5xTrqBoCT2LbcQsn2-J6fqEukRU3MnqoTTA +# sha256: hs1pErvIPpQF1Kc81_S07oNTZS0tvHyCAQbtW00bqzo +# sha256: jx0XfTZOo1kAQVriTKPkcb49UzTtBBkpQGjEn0WROZg +cffi==1.4.2 + +# sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc +ConfigArgParse==0.10.0 + +# sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI +configobj==5.0.6 + +# sha256: 1U_hszrB4J8cEj4vl0948z6V1h1PSALdISIKXD6MEX0 +# sha256: B1X2aE4RhSAFs2MTdh7ctbqEOmTNAizhrC3L1JqTYG0 +# sha256: zjhNo4lZlluh90VKJfVp737yqxRd8ueiml4pS3TgRnc +# sha256: GvQDkV3LmWHDB2iuZRr6tpKC0dpaut-mN1IhrBGHdQM +# sha256: ag08d91PH-W8ZfJ--3fsjQSjiNpesl66DiBAwJgZ30o +# sha256: KdelgcO6_wTh--IAaltHjZ7cfPmib8ijWUkkf09lA3k +# sha256: IPAWEKpAh_bVadjMIMR4uB8DhIYnWqqx3Dx12VAsZ-A +# sha256: l9hGUIulDVomml82OK4cFmWbNTFaH0B_oVF2cH2j0Jc +# sha256: djfqRMLL1NsvLKccsmtmPRczORqnafi8g2xZVilbd5g +# sha256: gR-eqJVbPquzLgQGU0XDB4Ui5rPuPZLz0n08fNcWpjM +# sha256: DXCMjYz97Qm4fCoLqHY856ZjWG4EPmrEL9eDHpKQHLY +# sha256: Efnq11YqPgATWGytM5o_em9Yg8zhw7S5jhrGnft3p_Y +# sha256: dNhnm55-0ePs-wq1NNyTUruxz3PTYsmQkJTAlyivqJY +# sha256: z1Hd-123eBaiB1OKZgEUuC4w4IAD_uhJmwILi4SA2sU +# sha256: 47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU +# sha256: dITvgYGUFB3_eUdf-74vd6-FHiw7v-Lk1ZEjEi-KTjM +# sha256: 7gLB6J7l7pUBV6VK1YTXN8Ec83putMCFPozz8n6WLcA +# sha256: pfGPaxhQpVVKV9v2YsrSUSpGBW5paHJqmFjngN1bnQo +# sha256: 26GA8xrb5xi6qdbPirY0hJSwlLK4GAL_8zvVDSfRPnM +# sha256: 5RinlLjzjoOC9_B3kUGBPOtIE6z9MRVBwNsOGJ69eN4 +# sha256: f1FFn4TWcERCdeYVg59FQsk1R6Euk4oKSQba_l994VM +cryptography==1.1.2 + +# sha256: JHXX_N31lR6S_1RpcnWIAt5SYL9Akxmp8ZNOa7yLHcc +# sha256: NZB977D5krdat3iPZf7cHPIP-iJojg5vbxKvwGs-pQE +enum34==1.1.2 + +# sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 +# sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM +funcsigs==0.4 + +# sha256: my_FC9PEujBrllG2lBHvIgJtTYM1uTr8IhTO8SRs5wc +# sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs +idna==2.0 + +# sha256: k1cSgAzkdgcB2JrWd2Zs1SaR_S9vCzQMi0I5o8F5iKU +# sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA +ipaddress==1.0.16 + +# sha256: 54vpwKDfy6xxL-BPv5K5bN2ugLG4QvJCSCFMhJbwBu8 +# sha256: Syb_TnEQ23butvWntkqCYjg51ZXCA47tpmLyott46Xw +linecache2==1.0.0 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + +# sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs +# sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs +parsedatetime==1.5 + +# sha256: Rsjbda51oFa9HMB_ohc0_i5gPRGgeDPswe63TDXHLgw +# sha256: 4hJ2JqkebIhduJZol22zECDwry2nKJJLVkgPx8zwlkk +pbr==1.8.1 + +# sha256: WE8LKfzF1SO0M8uJGLL8dNZ-MO4LRKlbrwMVKPQkYZ8 +# sha256: KMoLbp2Zqo3Chuh0ekRxNitpgSolKR3im2qNcKFUWg0 +# sha256: FnrV__UqZyxN3BwaCyUUbWgT67CKmqsKOsRfiltmnDs +# sha256: 5t6mFzqYhye7Ij00lzSa1c3vXAsoLv8tg-X5BlxT-F8 +# sha256: KvXgpKrWYEmVXQc0qk49yMqhep6vi0waJ6Xx7m5A9vw +# sha256: 2YhNwNwuVeJEjklXeNyYmcHIvzeusvQ0wb6nSvk8JoM +# sha256: 4nwv5t_Mhzi-PSxaAi94XrcpcQV-Gp4eNPunO86KcaY +# sha256: Za_W_syPOu0J7kvmNYO8jrRy8GzqpP4kxNHVoaPA4T8 +# sha256: uhxVj7_N-UUVwjlLEVXB3FbivCqcF9MDSYJ8ntimfkY +# sha256: upXqACLctk028MEzXAYF-uNb3z4P6o2S9dD2RWo15Vs +# sha256: QhtlkdFrUJqqjYwVgh1mu5TLSo3EOFytXFG4XUoJbYU +# sha256: MmswXL22-U2vv-LCaxHaiLCrB7igf4GIq511_wxuhBo +# sha256: mu3lsrb-RrN0jqjlIURDiQ0WNAJ77z0zt9rRZVaDAng +# sha256: c77R24lNGqnDx-YR0wLN6reuig3A7q92cnh42xrFzYc +# sha256: k1td1tVYr1EvQlAafAj0HXr_E5rxuzlZ2qOruFkjTWw +# sha256: TKARHPFX3MDy9poyPFtUeHGNaNRfyUNdhL4OwPGGIVs +# sha256: tvE8lTmKP88CJsTc-kSFYLpYZSWc2W7CgQZYZR6TIYk +# sha256: 7mvjDRY1u96kxDJdUH3IoNu95-HBmL1i3bn0MZi54hQ +# sha256: 36eGhYwmjX-74bYXXgAewCc418-uCnzne_m2Ua9nZyk +# sha256: qnf53nKvnBbMKIzUokz1iCQ4j1fXqB5ADEYWRXYphw4 +# sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus +psutil==3.3.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + +# sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M +pycparser==2.14 + +# sha256: iORea7Jd_tJyoe8ucoRh1EtjTCzWiemJtuVqNJxaOuU +# sha256: 8KJgcNbbCIHei8x4RpNLfDyTDY-cedRYg-5ImEvA1nI +pyOpenSSL==0.15.1 + +# sha256: 7qMYNcVuIJavQ2OldFp4SHimHQQ-JH06bWoKMql0H1Y +# sha256: jfvGxFi42rocDzYgqMeACLMjomiye3NZ6SpK5BMl9TU +pyRFC3339==1.0 + +# sha256: Z9WdZs26jWJOA4m4eyqDoXbyHxaodVO1D1cDsj8pusI +python-augeas==0.5.0 + +# sha256: BOk_JJlcQ92Q8zjV2GXKcs4_taU1jU2qSWVXHbNfw-w +# sha256: Pm9ZP-rZj4pSa8PjBpM1MyNuM3KfVS9SiW6lBPVTE_o +python2-pythondialog==3.3.0 + +# sha256: Or5qbT_C-75MYBRCEfRdou2-MYKm9lEa9ru6BZix-ZI +# sha256: k575weEiTZgEBWial__PeCjFbRUXsx1zRkNWwfK3dp4 +# sha256: 6tSu-nAHJJ4F5RsBCVcZ1ajdlXYAifVzCqxWmLGTKRg +# sha256: PMoN8IvQ7ZhDI5BJTOPe0AP15mGqRgvnpzS__jWYNgU +# sha256: Pt5HDT0XujwHY436DRBFK8G25a0yYSemW6d-aq6xG-w +# sha256: aMR5ZPcYbuwwaxNilidyK5B5zURH7Z5eyuzU6shMpzQ +# sha256: 3V05kZUKrkCmyB3hV4lC5z1imAjO_FHRLNFXmA5s_Bg +# sha256: p3xSBiwH63x7MFRdvHPjKZW34Rfup1Axe1y1x6RhjxQ +# sha256: ga-a7EvJYKmgEnxIjxh3La5GNGiSM_BvZUQ-exHr61E +# sha256: 4Hmx2txcBiRswbtv4bI6ULHRFz8u3VEE79QLtzoo9AY +# sha256: -9JnRncsJMuTyLl8va1cueRshrvbG52KdD7gDi-x_F0 +# sha256: mSZu8wo35Dky3uwrfKc-g8jbw7n_cD7HPsprHa5r7-o +# sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM +pytz==2015.7 + +# sha256: ET-7pVManjSUW302szoIToul0GZLcDyBp8Vy2RkZpbg +# sha256: xXeBXdAPE5QgP8ROuXlySwmPiCZKnviY7kW45enPWH8 +requests==2.9.1 + +# sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE +# sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo +six==1.10.0 + +# sha256: glPOvsSxkJTWfMXtWvmb8duhKFKSIm6Yoxkp-HpdayM +# sha256: BazGegmYDC7P7dNCP3rgEEg57MtV_GRXc-HKoJUcMDA +traceback2==1.4.0 + +# sha256: E_d9CHXbbZtDXh1PQedK1MwutuHVyCSZYJKzQw8Ii7g +# sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk +unittest2==1.1.0 + +# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms +# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI +Werkzeug==0.11.3 + +# sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo +zope.component==4.2.2 + +# sha256: 3HpZov2Rcw03kxMaXSYbKek-xOKpfxvEh86N7-4v54Y +zope.event==4.1.0 + +# sha256: 8HtjH3pgHNjL0zMtVPQxQscIioMpn4WTVvCNHU1CWbM +# sha256: 3lzKCDuUOdgAL7drvmtJmMWlpyH6sluEKYln8ALfTJQ +# sha256: Z4hBb36n9bipe-lIJTd6ol6L3HNGPge6r5hYsp5zcHc +# sha256: bzIw9yVFGCAeWjcIy7LemMhIME8G497Yv7OeWCXLouE +# sha256: X6V1pSQPBCAMMIhCfQ1Le3N_bpAYgYpR2ND5J6aiUXo +# sha256: UiGUrWpUVzXt11yKg_SNZdGvBk5DKn0yDWT1a6_BLpk +# sha256: 6Mey1AlD9xyZFIyX9myqf1E0FH9XQj-NtbSCUJnOmgk +# sha256: J5Ak8CCGAcPKqQfFOHbjetiGJffq8cs4QtvjYLIocBc +# sha256: LiIanux8zFiImieOoT3P7V75OdgLB4Gamos8scaBSE8 +# sha256: aRGJZUEOyG1E3GuQF-4929WC4MCr7vYrOhnb9sitEys +# sha256: 0E34aG7IZNDK3ozxmff4OuzUFhCaIINNVo-DEN7RLeo +# sha256: 51qUfhXul-fnHgLqMC_rL8YtOiu0Zov5377UOlBqx-c +# sha256: TkXSL7iDIipaufKCoRb-xe4ujRpWjM_2otdbvQ62vPw +# sha256: vOkzm7PHpV4IA7Y9IcWDno5Hm8hcSt9CrkFbcvlPrLI +# sha256: koE4NlJFoOiGmlmZ-8wqRUdaCm7VKklNYNvcVAM1_t0 +# sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro +# sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I +zope.interface==4.1.3 + +# sha256: ilvjjTWOS86xchl0WBZ0YOAw_0rmqdnjNsxb1hq2RD8 +# sha256: T37KMj0TnsuvHIzCCmoww2fpfpOBTj7cd4NAqucXcpw +acme==0.4.0 + +# sha256: 33BQiANlNLGqGpirTfdCEElTF9YbpaKiYpTbK4zeGD8 +# sha256: lwsV1OdEzzlMeb08C_PRxaCXZ2vOk_1AI2755rZHmPM +letsencrypt==0.4.0 + +# sha256: D3YDaVFjLsMSEfjI5B5D5tn5FeWUtNHYXCObw3ih2tg +# sha256: VTgvsePYGRmI4IOSAnxoYFHd8KciD73bxIuIHtbVFd8 +letsencrypt-apache==0.4.0 + +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +mock==1.0.1 + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" +#!/usr/bin/env python +"""peep ("prudently examine every package") verifies that packages conform to a +trusted, locally stored hash and only then installs them:: + + peep install -r requirements.txt + +This makes your deployments verifiably repeatable without having to maintain a +local PyPI mirror or use a vendor lib. Just update the version numbers and +hashes in requirements.txt, and you're all set. + +""" +# This is here so embedded copies of peep.py are MIT-compliant: +# Copyright (c) 2013 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +try: + xrange = xrange +except NameError: + xrange = range +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify +import cgi +from collections import defaultdict +from functools import wraps +from hashlib import sha256 +from itertools import chain, islice +import mimetypes +from optparse import OptionParser +from os.path import join, basename, splitext, isdir +from pickle import dumps, loads +import re +import sys +from shutil import rmtree, copy +from sys import argv, exit +from tempfile import mkdtemp +import traceback +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 +# TODO: Probably use six to make urllib stuff work across 2/3. + +from pkg_resources import require, VersionConflict, DistributionNotFound + +# We don't admit our dependency on pip in setup.py, lest a naive user simply +# say `pip install peep.tar.gz` and thus pull down an untrusted copy of pip +# from PyPI. Instead, we make sure it's installed and new enough here and spit +# out an error message if not: + + +def activate(specifier): + """Make a compatible version of pip importable. Raise a RuntimeError if we + couldn't.""" + try: + for distro in require(specifier): + distro.activate() + except (VersionConflict, DistributionNotFound): + raise RuntimeError('The installed version of pip is too old; peep ' + 'requires ' + specifier) + +# Before 0.6.2, the log module wasn't there, so some +# of our monkeypatching fails. It probably wouldn't be +# much work to support even earlier, though. +activate('pip>=0.6.2') + +import pip +from pip.commands.install import InstallCommand +try: + from pip.download import url_to_path # 1.5.6 +except ImportError: + try: + from pip.util import url_to_path # 0.7.0 + except ImportError: + from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.index import PackageFinder, Link +try: + from pip.log import logger +except ImportError: + from pip import logger # 6.0 +from pip.req import parse_requirements +try: + from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner +except ImportError: + class NullProgressBar(object): + def __init__(self, *args, **kwargs): + pass + + def iter(self, ret, *args, **kwargs): + return ret + + DownloadProgressBar = DownloadProgressSpinner = NullProgressBar + +__version__ = 3, 0, 0 + +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True + + +ITS_FINE_ITS_FINE = 0 +SOMETHING_WENT_WRONG = 1 +# "Traditional" for command-line errors according to optparse docs: +COMMAND_LINE_ERROR = 2 + +ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') + +MARKER = object() + + +class PipException(Exception): + """When I delegated to pip, it exited with an error.""" + + def __init__(self, error_code): + self.error_code = error_code + + +class UnsupportedRequirementError(Exception): + """An unsupported line was encountered in a requirements file.""" + + +class DownloadError(Exception): + def __init__(self, link, exc): + self.link = link + self.reason = str(exc) + + def __str__(self): + return 'Downloading %s failed: %s' % (self.link, self.reason) + + +def encoded_hash(sha): + """Return a short, 7-bit-safe representation of a hash. + + If you pass a sha256, this results in the hash algorithm that the Wheel + format (PEP 427) uses, except here it's intended to be run across the + downloaded archive before unpacking. + + """ + return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') + + +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + +def run_pip(initial_args): + """Delegate to pip the given args (starting with the subcommand), and raise + ``PipException`` if something goes wrong.""" + status_code = pip.main(initial_args) + + # Clear out the registrations in the pip "logger" singleton. Otherwise, + # loggers keep getting appended to it with every run. Pip assumes only one + # command invocation will happen per interpreter lifetime. + logger.consumers = [] + + if status_code: + raise PipException(status_code) + + +def hash_of_file(path): + """Return the hash of a downloaded file.""" + with open(path, 'rb') as archive: + sha = sha256() + while True: + data = archive.read(2 ** 20) + if not data: + break + sha.update(data) + return encoded_hash(sha) + + +def is_git_sha(text): + """Return whether this is probably a git sha""" + # Handle both the full sha as well as the 7-character abbreviation + if len(text) in (40, 7): + try: + int(text, 16) + return True + except ValueError: + pass + return False + + +def filename_from_url(url): + parsed = urlparse(url) + path = parsed.path + return path.split('/')[-1] + + +def requirement_args(argv, want_paths=False, want_other=False): + """Return an iterable of filtered arguments. + + :arg argv: Arguments, starting after the subcommand + :arg want_paths: If True, the returned iterable includes the paths to any + requirements files following a ``-r`` or ``--requirement`` option. + :arg want_other: If True, the returned iterable includes the args that are + not a requirement-file path or a ``-r`` or ``--requirement`` flag. + + """ + was_r = False + for arg in argv: + # Allow for requirements files named "-r", don't freak out if there's a + # trailing "-r", etc. + if was_r: + if want_paths: + yield arg + was_r = False + elif arg in ['-r', '--requirement']: + was_r = True + else: + if want_other: + yield arg + +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') + +HASH_COMMENT_RE = re.compile( + r""" + \s*\#\s+ # Lines that start with a '#' + (?Psha256):\s+ # Hash type is hardcoded to be sha256 for now. + (?P[^\s]+) # Hashes can be anything except '#' or spaces. + \s* # Suck up whitespace before the comment or + # just trailing whitespace if there is no + # comment. Also strip trailing newlines. + (?:\#(?P.*))? # Comments can be anything after a whitespace+# + # and are optional. + $""", re.X) + + +def peep_hash(argv): + """Return the peep hash of one or more files, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + parser = OptionParser( + usage='usage: %prog hash file [file ...]', + description='Print a peep hash line for one or more files: for ' + 'example, "# sha256: ' + 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY".') + _, paths = parser.parse_args(args=argv) + if paths: + for path in paths: + print('# sha256:', hash_of_file(path)) + return ITS_FINE_ITS_FINE + else: + parser.print_usage() + return COMMAND_LINE_ERROR + + +class EmptyOptions(object): + """Fake optparse options for compatibility with pip<1.2 + + pip<1.2 had a bug in parse_requirements() in which the ``options`` kwarg + was required. We work around that by passing it a mock object. + + """ + default_vcs = None + skip_requirements_regex = None + isolated_mode = False + + +def memoize(func): + """Memoize a method that should return the same result every time on a + given instance. + + """ + @wraps(func) + def memoizer(self): + if not hasattr(self, '_cache'): + self._cache = {} + if func.__name__ not in self._cache: + self._cache[func.__name__] = func(self) + return self._cache[func.__name__] + return memoizer + + +def package_finder(argv): + """Return a PackageFinder respecting command-line options. + + :arg argv: Everything after the subcommand + + """ + # We instantiate an InstallCommand and then use some of its private + # machinery--its arg parser--for our own purposes, like a virus. This + # approach is portable across many pip versions, where more fine-grained + # ones are not. Ignoring options that don't exist on the parser (for + # instance, --use-wheel) gives us a straightforward method of backward + # compatibility. + try: + command = InstallCommand() + except TypeError: + # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 + # given)" error. In that version, InstallCommand takes a top=level + # parser passed in from outside. + from pip.baseparser import create_main_parser + command = InstallCommand(create_main_parser()) + # The downside is that it essentially ruins the InstallCommand class for + # further use. Calling out to pip.main() within the same interpreter, for + # example, would result in arguments parsed this time turning up there. + # Thus, we deepcopy the arg parser so we don't trash its singletons. Of + # course, deepcopy doesn't work on these objects, because they contain + # uncopyable regex patterns, so we pickle and unpickle instead. Fun! + options, _ = loads(dumps(command.parser)).parse_args(argv) + + # Carry over PackageFinder kwargs that have [about] the same names as + # options attr names: + possible_options = [ + 'find_links', + FORMAT_CONTROL_ARG, + ('allow_all_prereleases', 'pre'), + 'process_dependency_links' + ] + kwargs = {} + for option in possible_options: + kw, attr = option if isinstance(option, tuple) else (option, option) + value = getattr(options, attr, MARKER) + if value is not MARKER: + kwargs[kw] = value + + # Figure out index_urls: + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + index_urls = [] + index_urls += getattr(options, 'mirrors', []) + + # If pip is new enough to have a PipSession, initialize one, since + # PackageFinder requires it: + if hasattr(command, '_build_session'): + kwargs['session'] = command._build_session(options) + + return PackageFinder(index_urls=index_urls, **kwargs) + + +class DownloadedReq(object): + """A wrapper around InstallRequirement which offers additional information + based on downloading and examining a corresponding package archive + + These are conceptually immutable, so we can get away with memoizing + expensive things. + + """ + def __init__(self, req, argv, finder): + """Download a requirement, compare its hashes, and return a subclass + of DownloadedReq depending on its state. + + :arg req: The InstallRequirement I am based on + :arg argv: The args, starting after the subcommand + + """ + self._req = req + self._argv = argv + self._finder = finder + + # We use a separate temp dir for each requirement so requirements + # (from different indices) that happen to have the same archive names + # don't overwrite each other, leading to a security hole in which the + # latter is a hash mismatch, the former has already passed the + # comparison, and the latter gets installed. + self._temp_path = mkdtemp(prefix='peep-') + # Think of DownloadedReq as a one-shot state machine. It's an abstract + # class that ratchets forward to being one of its own subclasses, + # depending on its package status. Then it doesn't move again. + self.__class__ = self._class() + + def dispose(self): + """Delete temp files and dirs I've made. Render myself useless. + + Do not call further methods on me after calling dispose(). + + """ + rmtree(self._temp_path) + + def _version(self): + """Deduce the version number of the downloaded package from its filename.""" + # TODO: Can we delete this method and just print the line from the + # reqs file verbatim instead? + def version_of_archive(filename, package_name): + # Since we know the project_name, we can strip that off the left, strip + # any archive extensions off the right, and take the rest as the + # version. + for ext in ARCHIVE_EXTENSIONS: + if filename.endswith(ext): + filename = filename[:-len(ext)] + break + # Handle github sha tarball downloads. + if is_git_sha(filename): + filename = package_name + '-' + filename + if not filename.lower().replace('_', '-').startswith(package_name.lower()): + # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -? + give_up(filename, package_name) + return filename[len(package_name) + 1:] # Strip off '-' before version. + + def version_of_wheel(filename, package_name): + # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file- + # name-convention) we know the format bits are '-' separated. + whl_package_name, version, _rest = filename.split('-', 2) + # Do the alteration to package_name from PEP 427: + our_package_name = re.sub(r'[^\w\d.]+', '_', package_name, re.UNICODE) + if whl_package_name != our_package_name: + give_up(filename, whl_package_name) + return version + + def give_up(filename, package_name): + raise RuntimeError("The archive '%s' didn't start with the package name " + "'%s', so I couldn't figure out the version number. " + "My bad; improve me." % + (filename, package_name)) + + get_version = (version_of_wheel + if self._downloaded_filename().endswith('.whl') + else version_of_archive) + return get_version(self._downloaded_filename(), self._project_name()) + + def _is_always_unsatisfied(self): + """Returns whether this requirement is always unsatisfied + + This would happen in cases where we can't determine the version + from the filename. + + """ + # If this is a github sha tarball, then it is always unsatisfied + # because the url has a commit sha in it and not the version + # number. + url = self._url() + if url: + filename = filename_from_url(url) + if filename.endswith(ARCHIVE_EXTENSIONS): + filename, ext = splitext(filename) + if is_git_sha(filename): + return True + return False + + @memoize # Avoid hitting the file[cache] over and over. + def _expected_hashes(self): + """Return a list of known-good hashes for this package.""" + return hashes_above(*path_and_line(self._req)) + + def _download(self, link): + """Download a file, and return its name within my temp dir. + + This does no verification of HTTPS certs, but our checking hashes + makes that largely unimportant. It would be nice to be able to use the + requests lib, which can verify certs, but it is guaranteed to be + available only in pip >= 1.5. + + This also drops support for proxies and basic auth, though those could + be added back in. + + """ + # Based on pip 1.4.1's URLOpener but with cert verification removed + def opener(is_https): + if is_https: + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + else: + opener = build_opener() + return opener + + # Descended from unpack_http_url() in pip 1.4.1 + def best_filename(link, response): + """Return the most informative possible filename for a download, + ideally with a proper extension. + + """ + content_type = response.info().get('content-type', '') + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess: + content_disposition = response.info().get('content-disposition') + if content_disposition: + type, params = cgi.parse_header(content_disposition) + # We use ``or`` here because we don't want to use an "empty" value + # from the filename param: + filename = params.get('filename') or filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != response.geturl(): + ext = splitext(response.geturl())[1] + if ext: + filename += ext + return filename + + # Descended from _download_url() in pip 1.4.1 + def pipe_to_file(response, path, size=0): + """Pull the data off an HTTP response, shove it in a new file, and + show progress. + + :arg response: A file-like object to read from + :arg path: The path of the new file + :arg size: The expected size, in bytes, of the download. 0 for + unknown or to suppress progress indication (as for cached + downloads) + + """ + def response_chunks(chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + print('Downloading %s%s...' % ( + self._req.req, + (' (%sK)' % (size / 1000)) if size > 1000 else '')) + progress_indicator = (DownloadProgressBar(max=size).iter if size + else DownloadProgressSpinner().iter) + with open(path, 'wb') as file: + for chunk in progress_indicator(response_chunks(4096), 4096): + file.write(chunk) + + url = link.url.split('#', 1)[0] + try: + response = opener(urlparse(url).scheme != 'http').open(url) + except (HTTPError, IOError) as exc: + raise DownloadError(link, exc) + filename = best_filename(link, response) + try: + size = int(response.headers['content-length']) + except (ValueError, KeyError, TypeError): + size = 0 + pipe_to_file(response, join(self._temp_path, filename), size=size) + return filename + + # Based on req_set.prepare_files() in pip bb2a8428d4aebc8d313d05d590f386fa3f0bbd0f + @memoize # Avoid re-downloading. + def _downloaded_filename(self): + """Download the package's archive if necessary, and return its + filename. + + --no-deps is implied, as we have reimplemented the bits that would + ordinarily do dependency resolution. + + """ + # Peep doesn't support requirements that don't come down as a single + # file, because it can't hash them. Thus, it doesn't support editable + # requirements, because pip itself doesn't support editable + # requirements except for "local projects or a VCS url". Nor does it + # support VCS requirements yet, because we haven't yet come up with a + # portable, deterministic way to hash them. In summary, all we support + # is == requirements and tarballs/zips/etc. + + # TODO: Stop on reqs that are editable or aren't ==. + + # If the requirement isn't already specified as a URL, get a URL + # from an index: + link = self._link() or self._finder.find_requirement(self._req, upgrade=False) + + if link: + lower_scheme = link.scheme.lower() # pip lower()s it for some reason. + if lower_scheme == 'http' or lower_scheme == 'https': + file_path = self._download(link) + return basename(file_path) + elif lower_scheme == 'file': + # The following is inspired by pip's unpack_file_url(): + link_path = url_to_path(link.url_without_fragment) + if isdir(link_path): + raise UnsupportedRequirementError( + "%s: %s is a directory. So that it can compute " + "a hash, peep supports only filesystem paths which " + "point to files" % + (self._req, link.url_without_fragment)) + else: + copy(link_path, self._temp_path) + return basename(link_path) + else: + raise UnsupportedRequirementError( + "%s: The download link, %s, would not result in a file " + "that can be hashed. Peep supports only == requirements, " + "file:// URLs pointing to files (not folders), and " + "http:// and https:// URLs pointing to tarballs, zips, " + "etc." % (self._req, link.url)) + else: + raise UnsupportedRequirementError( + "%s: couldn't determine where to download this requirement from." + % (self._req,)) + + def install(self): + """Install the package I represent, without dependencies. + + Obey typical pip-install options passed in on the command line. + + """ + other_args = list(requirement_args(self._argv, want_other=True)) + archive_path = join(self._temp_path, self._downloaded_filename()) + # -U so it installs whether pip deems the requirement "satisfied" or + # not. This is necessary for GitHub-sourced zips, which change without + # their version numbers changing. + run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path]) + + @memoize + def _actual_hash(self): + """Download the package's archive if necessary, and return its hash.""" + return hash_of_file(join(self._temp_path, self._downloaded_filename())) + + def _project_name(self): + """Return the inner Requirement's "unsafe name". + + Raise ValueError if there is no name. + + """ + name = getattr(self._req.req, 'project_name', '') + if name: + return name + raise ValueError('Requirement has no project_name.') + + def _name(self): + return self._req.name + + def _link(self): + try: + return self._req.link + except AttributeError: + # The link attribute isn't available prior to pip 6.1.0, so fall + # back to the now deprecated 'url' attribute. + return Link(self._req.url) if self._req.url else None + + def _url(self): + link = self._link() + return link.url if link else None + + @memoize # Avoid re-running expensive check_if_exists(). + def _is_satisfied(self): + self._req.check_if_exists() + return (self._req.satisfied_by and + not self._is_always_unsatisfied()) + + def _class(self): + """Return the class I should be, spanning a continuum of goodness.""" + try: + self._project_name() + except ValueError: + return MalformedReq + if self._is_satisfied(): + return SatisfiedReq + if not self._expected_hashes(): + return MissingReq + if self._actual_hash() not in self._expected_hashes(): + return MismatchedReq + return InstallableReq + + @classmethod + def foot(cls): + """Return the text to be printed once, after all of the errors from + classes of my type are printed. + + """ + return '' + + +class MalformedReq(DownloadedReq): + """A requirement whose package name could not be determined""" + + @classmethod + def head(cls): + return 'The following requirements could not be processed:\n' + + def error(self): + return '* Unable to determine package name from URL %s; add #egg=' % self._url() + + +class MissingReq(DownloadedReq): + """A requirement for which no hashes were specified in the requirements file""" + + @classmethod + def head(cls): + return ('The following packages had no hashes specified in the requirements file, which\n' + 'leaves them open to tampering. Vet these packages to your satisfaction, then\n' + 'add these "sha256" lines like so:\n\n') + + def error(self): + if self._url(): + # _url() always contains an #egg= part, or this would be a + # MalformedRequest. + line = self._url() + else: + line = '%s==%s' % (self._name(), self._version()) + return '# sha256: %s\n%s\n' % (self._actual_hash(), line) + + +class MismatchedReq(DownloadedReq): + """A requirement for which the downloaded file didn't match any of my hashes.""" + @classmethod + def head(cls): + return ("THE FOLLOWING PACKAGES DIDN'T MATCH THE HASHES SPECIFIED IN THE REQUIREMENTS\n" + "FILE. If you have updated the package versions, update the hashes. If not,\n" + "freak out, because someone has tampered with the packages.\n\n") + + def error(self): + preamble = ' %s: expected' % self._project_name() + if len(self._expected_hashes()) > 1: + preamble += ' one of' + padding = '\n' + ' ' * (len(preamble) + 1) + return '%s %s\n%s got %s' % (preamble, + padding.join(self._expected_hashes()), + ' ' * (len(preamble) - 4), + self._actual_hash()) + + @classmethod + def foot(cls): + return '\n' + + +class SatisfiedReq(DownloadedReq): + """A requirement which turned out to be already installed""" + + @classmethod + def head(cls): + return ("These packages were already installed, so we didn't need to download or build\n" + "them again. If you installed them with peep in the first place, you should be\n" + "safe. If not, uninstall them, then re-attempt your install with peep.\n") + + def error(self): + return ' %s' % (self._req,) + + +class InstallableReq(DownloadedReq): + """A requirement whose hash matched and can be safely installed""" + + +# DownloadedReq subclasses that indicate an error that should keep us from +# going forward with installation, in the order in which their errors should +# be reported: +ERROR_CLASSES = [MismatchedReq, MissingReq, MalformedReq] + + +def bucket(things, key): + """Return a map of key -> list of things.""" + ret = defaultdict(list) + for thing in things: + ret[key(thing)].append(thing) + return ret + + +def first_every_last(iterable, first, every, last): + """Execute something before the first item of iter, something else for each + item, and a third thing after the last. + + If there are no items in the iterable, don't execute anything. + + """ + did_first = False + for item in iterable: + if not did_first: + did_first = True + first(item) + every(item) + if did_first: + last(item) + + +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + +def downloaded_reqs_from_path(path, argv): + """Return a list of DownloadedReqs representing the requirements parsed + out of a given requirements file. + + :arg path: The path to the requirements file + :arg argv: The commandline args, starting after the subcommand + + """ + finder = package_finder(argv) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] + + +def peep_install(argv): + """Perform the ``peep install`` subcommand, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + output = [] + out = output.append + reqs = [] + try: + req_paths = list(requirement_args(argv, want_paths=True)) + if not req_paths: + out("You have to specify one or more requirements files with the -r option, because\n" + "otherwise there's nowhere for peep to look up the hashes.\n") + return COMMAND_LINE_ERROR + + # We're a "peep install" command, and we have some requirement paths. + reqs = list(chain.from_iterable( + downloaded_reqs_from_path(path, argv) + for path in req_paths)) + buckets = bucket(reqs, lambda r: r.__class__) + + # Skip a line after pip's "Cleaning up..." so the important stuff + # stands out: + if any(buckets[b] for b in ERROR_CLASSES): + out('\n') + + printers = (lambda r: out(r.head()), + lambda r: out(r.error() + '\n'), + lambda r: out(r.foot())) + for c in ERROR_CLASSES: + first_every_last(buckets[c], *printers) + + if any(buckets[b] for b in ERROR_CLASSES): + out('-------------------------------\n' + 'Not proceeding to installation.\n') + return SOMETHING_WENT_WRONG + else: + for req in buckets[InstallableReq]: + req.install() + + first_every_last(buckets[SatisfiedReq], *printers) + + return ITS_FINE_ITS_FINE + except (UnsupportedRequirementError, DownloadError) as exc: + out(str(exc)) + return SOMETHING_WENT_WRONG + finally: + for req in reqs: + req.dispose() + print(''.join(output)) + + +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + +def main(): + """Be the top-level entrypoint. Return a shell status code.""" + commands = {'hash': peep_hash, + 'install': peep_install, + 'port': peep_port} + try: + if len(argv) >= 2 and argv[1] in commands: + return commands[argv[1]](argv[2:]) + else: + # Fall through to top-level pip main() for everything else: + return pip.main() + except PipException as exc: + return exc.error_code + + +def exception_handler(exc_type, exc_value, exc_tb): + print('Oh no! Peep had a problem while trying to do stuff. Please write up a bug report') + print('with the specifics so we can fix it:') + print() + print('https://github.com/erikrose/peep/issues/new') + print() + print('Here are some particulars you can copy and paste into the bug report:') + print() + print('---') + print('peep:', repr(__version__)) + print('python:', repr(sys.version)) + print('pip:', repr(getattr(pip, '__version__', 'no __version__ attr'))) + print('Command line: ', repr(sys.argv)) + print( + ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))) + print('---') + + +if __name__ == '__main__': + try: + exit(main()) + except Exception: + exception_handler(*sys.exc_info()) + exit(SOMETHING_WENT_WRONG) + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + set +e + PEEP_OUT=`"$VENV_BIN/python" "$TEMP_DIR/peep.py" install -r "$TEMP_DIR/letsencrypt-auto-requirements.txt"` + PEEP_STATUS=$? + set -e + rm -rf "$TEMP_DIR" + if [ "$PEEP_STATUS" != 0 ]; then + # Report error. (Otherwise, be quiet.) + echo "Had a problem while downloading and verifying Python packages:" + echo "$PEEP_OUT" + rm -rf "$VENV_PATH" + exit 1 + fi + echo "Installation succeeded." + fi + echo "Requesting root privileges to run letsencrypt..." + echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" + $SUDO "$VENV_BIN/letsencrypt" "$@" +else + # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # letsencrypt-auto (which is always the same as that of the letsencrypt + # package). Phase 2 checks the version of the locally installed letsencrypt. + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$OS_PACKAGES_ONLY" = 1 ]; then + echo "OS packages installed." + exit 0 + fi + + echo "Checking for new version..." + TEMP_DIR=$(TempDir) + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 + +On failure, return non-zero. + +""" +from distutils.version import LooseVersion +from json import loads +from os import devnull, environ +from os.path import dirname, join +import re +from subprocess import check_call, CalledProcessError +from sys import argv, exit +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB +-----END PUBLIC KEY----- +""") + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/letsencrypt/json'))) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp_dir): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. + + """ + le_auto_dir = environ.get( + 'LE_AUTO_DIR_TEMPLATE', + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'letsencrypt-auto-source/') % tag + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp_dir, 'public_key.pem'), + '-signature', + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + + +def main(): + get = HttpsGetter().get + flag = argv[1] + try: + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) + +UNLIKELY_EOF + # --------------------------------------------------------------------------- + DeterminePythonVersion + REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` + if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Replacing letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + rm -rf "$TEMP_DIR" + fi # should upgrade + "$0" --no-self-upgrade "$@" +fi diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig new file mode 100644 index 000000000..532a48207 Binary files /dev/null and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template new file mode 100755 index 000000000..bccd9e2c9 --- /dev/null +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -0,0 +1,265 @@ +#!/bin/sh +# +# Download and run the latest release version of the Let's Encrypt client. +# +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* + +set -e # Work even if somebody does "sh thisscript.sh". + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN="$VENV_PATH/bin" +LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" + +# This script takes the same arguments as the main letsencrypt program, but it +# additionally responds to --verbose (more output) and --debug (allow support +# for experimental platforms) +for arg in "$@" ; do + case "$arg" in + --debug) + DEBUG=1;; + --os-packages-only) + OS_PACKAGES_ONLY=1;; + --no-self-upgrade) + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1;; + --verbose) + VERBOSE=1;; + [!-]*|-*[!v]*|-) + # Anything that isn't -v, -vv, etc.: that is, anything that does not + # start with a -, contains anything that's not a v, or is just "-" + ;; + *) # -v+ remains. + VERBOSE=1;; + esac +done + +# letsencrypt-auto needs root access to bootstrap OS dependencies, and +# letsencrypt itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + command -v "$LE_PYTHON" > /dev/null && break + done + if [ "$?" != "0" ]; then + echo "Cannot find any Pythons; please install one!" + exit 1 + fi + export LE_PYTHON + + PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ "$PYVER" -lt 26 ]; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + +{{ bootstrappers/deb_common.sh }} +{{ bootstrappers/rpm_common.sh }} +{{ bootstrappers/suse_common.sh }} +{{ bootstrappers/arch_common.sh }} +{{ bootstrappers/gentoo_common.sh }} +{{ bootstrappers/free_bsd.sh }} +{{ bootstrappers/mac.sh }} + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ]; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/redhat-release ]; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchCommon + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S letsencrypt letsencrypt-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ]; then + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon + elif [ -f /etc/gentoo-release ]; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + else + echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run a peep install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + fi +} + +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + + + +if [ "$NO_SELF_UPGRADE" = 1 ]; then + # Phase 2: Create venv, install LE, and run. + + if [ -f "$VENV_BIN/letsencrypt" ]; then + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | cut -d " " -f 2) + else + INSTALLED_VERSION="none" + fi + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then + echo "Creating virtual environment..." + DeterminePythonVersion + rm -rf "$VENV_PATH" + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi + + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + # There is no $ interpolation due to quotes on starting heredoc delimiter. + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" +{{ letsencrypt-auto-requirements.txt }} +UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/peep.py" +{{ peep.py }} +UNLIKELY_EOF + # ------------------------------------------------------------------------- + set +e + PEEP_OUT=`"$VENV_BIN/python" "$TEMP_DIR/peep.py" install -r "$TEMP_DIR/letsencrypt-auto-requirements.txt"` + PEEP_STATUS=$? + set -e + rm -rf "$TEMP_DIR" + if [ "$PEEP_STATUS" != 0 ]; then + # Report error. (Otherwise, be quiet.) + echo "Had a problem while downloading and verifying Python packages:" + echo "$PEEP_OUT" + rm -rf "$VENV_PATH" + exit 1 + fi + echo "Installation succeeded." + fi + echo "Requesting root privileges to run letsencrypt..." + echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" + $SUDO "$VENV_BIN/letsencrypt" "$@" +else + # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # letsencrypt-auto (which is always the same as that of the letsencrypt + # package). Phase 2 checks the version of the locally installed letsencrypt. + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$OS_PACKAGES_ONLY" = 1 ]; then + echo "OS packages installed." + exit 0 + fi + + echo "Checking for new version..." + TEMP_DIR=$(TempDir) + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" +{{ fetch.py }} +UNLIKELY_EOF + # --------------------------------------------------------------------------- + DeterminePythonVersion + REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` + if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of letsencrypt-auto. This preserves permissions and + # ownership from the old copy. + # TODO: Deal with quotes in pathnames. + echo "Replacing letsencrypt-auto..." + echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$0" + # TODO: Clean up temp dir safely, even if it has quotes in its path. + rm -rf "$TEMP_DIR" + fi # should upgrade + "$0" --no-self-upgrade "$@" +fi diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh new file mode 100755 index 000000000..90a9d43c4 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -0,0 +1,26 @@ +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./tools/_venv_common.sh + + deps=" + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + missing=$("$SUDO" pacman -T $deps) + + if [ "$missing" ]; then + "$SUDO" pacman -S --needed $missing + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh new file mode 100644 index 000000000..b45e43ba9 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -0,0 +1,94 @@ +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null 2>&1; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null 2>&1; then + virtualenv="$virtualenv python-virtualenv" + fi + + augeas_pkg="libaugeas0 augeas-lenses" + AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + if echo $BACKPORT_NAME | grep -q wheezy ; then + /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + fi + + sudo sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + + } + + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Let's Encrypt apache plugin..." + fi + # XXX add a case for ubuntu PPAs + fi + + $SUDO apt-get install -y --no-install-recommends \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + $augeas_pkg \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh new file mode 100755 index 000000000..c9abd22eb --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh @@ -0,0 +1,7 @@ +BootstrapFreeBsd() { + "$SUDO" pkg install -Ay \ + python \ + py27-virtualenv \ + augeas \ + libffi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh new file mode 100755 index 000000000..1d8211df4 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh @@ -0,0 +1,23 @@ +BootstrapGentooCommon() { + PACKAGES=" + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + "$SUDO" cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + "$SUDO" pmerge --noreplace --oneshot $PACKAGES + ;; + (portage|*) + "$SUDO" emerge --noreplace --oneshot $PACKAGES + ;; + esac +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh new file mode 100755 index 000000000..23c12eec3 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh @@ -0,0 +1,26 @@ +BootstrapMac() { + if ! hash brew 2>/dev/null; then + echo "Homebrew not installed.\nDownloading..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + fi + + if [ -z "$(brew list --versions augeas)" ]; then + echo "augeas not installed.\nInstalling augeas from Homebrew..." + brew install augeas + fi + + if [ -z "$(brew list --versions dialog)" ]; then + echo "dialog not installed.\nInstalling dialog from Homebrew..." + brew install dialog + fi + + if ! hash pip 2>/dev/null; then + echo "pip not installed.\nInstalling python from Homebrew..." + brew install python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv not installed.\nInstalling with pip..." + pip install virtualenv + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh new file mode 100755 index 000000000..68a11a531 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -0,0 +1,61 @@ +BootstrapRpmCommon() { + # Tested with: + # - Fedora 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if ! $SUDO $tool install -y \ + python \ + python-devel \ + python-virtualenv \ + python-tools \ + python-pip + then + if ! $SUDO $tool install -y \ + python27 \ + python27-devel \ + python27-virtualenv \ + python27-tools \ + python27-pip + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + fi + + if ! $SUDO $tool install -y \ + gcc \ + dialog \ + augeas-libs \ + openssl \ + openssl-devel \ + libffi-devel \ + redhat-rpm-config \ + ca-certificates + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi + + + if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then + if ! $SUDO $tool install -y mod_ssl + then + echo "Apache found, but mod_ssl could not be installed." + fi + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh new file mode 100755 index 000000000..46c60f992 --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh @@ -0,0 +1,14 @@ +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + $SUDO zypper -nq in -l \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates +} diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py new file mode 100644 index 000000000..39ff7777c --- /dev/null +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -0,0 +1,126 @@ +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 + +On failure, return non-zero. + +""" +from distutils.version import LooseVersion +from json import loads +from os import devnull, environ +from os.path import dirname, join +import re +from subprocess import check_call, CalledProcessError +from sys import argv, exit +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB +-----END PUBLIC KEY----- +""") + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/letsencrypt/json'))) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp_dir): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. + + """ + le_auto_dir = environ.get( + 'LE_AUTO_DIR_TEMPLATE', + 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'letsencrypt-auto-source/') % tag + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp_dir, 'public_key.pem'), + '-signature', + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "letsencrypt-auto.", exc) + + +def main(): + get = HttpsGetter().get + flag = argv[1] + try: + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt new file mode 100644 index 000000000..574e567c3 --- /dev/null +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -0,0 +1,218 @@ +# This is the flattened list of packages letsencrypt-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# then use `hashin` or a more secure method to gather the hashes. + +# sha256: wxZH7baf09RlqEfqMVfTe-0flfGXYLEaR6qRwEtmYxQ +# sha256: YrCJpVvh2JSc0rx-DfC9254Cj678jDIDjMhIYq791uQ +argparse==1.4.0 + +# sha256: U8HJ3bMEMVE-t_PN7wo-BrDxJSGIqqd0SvD1pM1F268 +# sha256: pWj0nfyhKo2fNwGHJX78WKOBCeHu5xTZKFYdegGKZPg +# sha256: gJxsqM-8ruv71DK0V2ABtA04_yRjdzy1dXfXXhoCC8M +# sha256: hs3KLNnLpBQiIwOQ3xff6qnzRKkR45dci-naV7NVSOk +# sha256: JLE9uErsOFyiPHuN7YPvi7QXe8GB0UdY-fl1vl0CDYY +# sha256: lprv_XwOCX9r4e_WgsFWriJlkaB5OpS2wtXkKT9MjU4 +# sha256: AA81jUsPokn-qrnBzn1bL-fgLnvfaAbCZBhQX8aF4mg +# sha256: qdhvRgu9g1ii1ROtd54_P8h447k6ALUAL66_YW_-a5w +# sha256: MSezqzPrI8ysBx-aCAJ0jlz3xcvNAkgrsGPjW0HbsLA +# sha256: 4rLUIjZGmkAiTTnntsYFdfOIsvQj81TH7pClt_WMgGU +# sha256: jC3Mr-6JsbQksL7GrS3ZYiyUnSAk6Sn12h7YAerHXx0 +# sha256: pN56TRGu1Ii6tPsU9JiFh6gpvs5aIEM_eA1uM7CAg8s +# sha256: XKj-MEJSZaSSdOSwITobyY9LE0Sa5elvmEdx5dg-WME +# sha256: pP04gC9Z5xTrqBoCT2LbcQsn2-J6fqEukRU3MnqoTTA +# sha256: hs1pErvIPpQF1Kc81_S07oNTZS0tvHyCAQbtW00bqzo +# sha256: jx0XfTZOo1kAQVriTKPkcb49UzTtBBkpQGjEn0WROZg +cffi==1.4.2 + +# sha256: O1CoPdWBSd_O6Yy2VlJl0QtT6cCivKfu73-19VJIkKc +ConfigArgParse==0.10.0 + +# sha256: ovVlB3DhyH-zNa8Zqbfrc_wFzPIhROto230AzSvLCQI +configobj==5.0.6 + +# sha256: 1U_hszrB4J8cEj4vl0948z6V1h1PSALdISIKXD6MEX0 +# sha256: B1X2aE4RhSAFs2MTdh7ctbqEOmTNAizhrC3L1JqTYG0 +# sha256: zjhNo4lZlluh90VKJfVp737yqxRd8ueiml4pS3TgRnc +# sha256: GvQDkV3LmWHDB2iuZRr6tpKC0dpaut-mN1IhrBGHdQM +# sha256: ag08d91PH-W8ZfJ--3fsjQSjiNpesl66DiBAwJgZ30o +# sha256: KdelgcO6_wTh--IAaltHjZ7cfPmib8ijWUkkf09lA3k +# sha256: IPAWEKpAh_bVadjMIMR4uB8DhIYnWqqx3Dx12VAsZ-A +# sha256: l9hGUIulDVomml82OK4cFmWbNTFaH0B_oVF2cH2j0Jc +# sha256: djfqRMLL1NsvLKccsmtmPRczORqnafi8g2xZVilbd5g +# sha256: gR-eqJVbPquzLgQGU0XDB4Ui5rPuPZLz0n08fNcWpjM +# sha256: DXCMjYz97Qm4fCoLqHY856ZjWG4EPmrEL9eDHpKQHLY +# sha256: Efnq11YqPgATWGytM5o_em9Yg8zhw7S5jhrGnft3p_Y +# sha256: dNhnm55-0ePs-wq1NNyTUruxz3PTYsmQkJTAlyivqJY +# sha256: z1Hd-123eBaiB1OKZgEUuC4w4IAD_uhJmwILi4SA2sU +# sha256: 47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU +# sha256: dITvgYGUFB3_eUdf-74vd6-FHiw7v-Lk1ZEjEi-KTjM +# sha256: 7gLB6J7l7pUBV6VK1YTXN8Ec83putMCFPozz8n6WLcA +# sha256: pfGPaxhQpVVKV9v2YsrSUSpGBW5paHJqmFjngN1bnQo +# sha256: 26GA8xrb5xi6qdbPirY0hJSwlLK4GAL_8zvVDSfRPnM +# sha256: 5RinlLjzjoOC9_B3kUGBPOtIE6z9MRVBwNsOGJ69eN4 +# sha256: f1FFn4TWcERCdeYVg59FQsk1R6Euk4oKSQba_l994VM +cryptography==1.1.2 + +# sha256: JHXX_N31lR6S_1RpcnWIAt5SYL9Akxmp8ZNOa7yLHcc +# sha256: NZB977D5krdat3iPZf7cHPIP-iJojg5vbxKvwGs-pQE +enum34==1.1.2 + +# sha256: _1rZ4vjZ5dHou_vPR3IqtSfPDVHK7u2dptD0B5k4P94 +# sha256: 2Dzm3wsOpmGHAP4ds1NSY5Goo62ht6ulL-16Ydp3IDM +funcsigs==0.4 + +# sha256: my_FC9PEujBrllG2lBHvIgJtTYM1uTr8IhTO8SRs5wc +# sha256: FhmarZOLKQ9b4QV8Dh78ZUYik5HCPOphypQMEV99PTs +idna==2.0 + +# sha256: k1cSgAzkdgcB2JrWd2Zs1SaR_S9vCzQMi0I5o8F5iKU +# sha256: WjGCsyKnBlJcRigspvBk0noCz_vUSfn0dBbx3JaqcbA +ipaddress==1.0.16 + +# sha256: 54vpwKDfy6xxL-BPv5K5bN2ugLG4QvJCSCFMhJbwBu8 +# sha256: Syb_TnEQ23butvWntkqCYjg51ZXCA47tpmLyott46Xw +linecache2==1.0.0 + +# sha256: 6MFV_evZxLywgQtO0BrhmHVUse4DTddTLXuP2uOKYnQ +ndg-httpsclient==0.4.0 + +# sha256: HDW0rCBs7y0kgWyJ-Jzyid09OM98RJuz-re_bUPwGx8 +ordereddict==1.1 + +# sha256: OnTxAPkNZZGDFf5kkHca0gi8PxOv0y01_P5OjQs7gSs +# sha256: Paa-K-UG9ZzOMuGeMOIBBT4btNB-JWaJGOAPikmtQKs +parsedatetime==1.5 + +# sha256: Rsjbda51oFa9HMB_ohc0_i5gPRGgeDPswe63TDXHLgw +# sha256: 4hJ2JqkebIhduJZol22zECDwry2nKJJLVkgPx8zwlkk +pbr==1.8.1 + +# sha256: WE8LKfzF1SO0M8uJGLL8dNZ-MO4LRKlbrwMVKPQkYZ8 +# sha256: KMoLbp2Zqo3Chuh0ekRxNitpgSolKR3im2qNcKFUWg0 +# sha256: FnrV__UqZyxN3BwaCyUUbWgT67CKmqsKOsRfiltmnDs +# sha256: 5t6mFzqYhye7Ij00lzSa1c3vXAsoLv8tg-X5BlxT-F8 +# sha256: KvXgpKrWYEmVXQc0qk49yMqhep6vi0waJ6Xx7m5A9vw +# sha256: 2YhNwNwuVeJEjklXeNyYmcHIvzeusvQ0wb6nSvk8JoM +# sha256: 4nwv5t_Mhzi-PSxaAi94XrcpcQV-Gp4eNPunO86KcaY +# sha256: Za_W_syPOu0J7kvmNYO8jrRy8GzqpP4kxNHVoaPA4T8 +# sha256: uhxVj7_N-UUVwjlLEVXB3FbivCqcF9MDSYJ8ntimfkY +# sha256: upXqACLctk028MEzXAYF-uNb3z4P6o2S9dD2RWo15Vs +# sha256: QhtlkdFrUJqqjYwVgh1mu5TLSo3EOFytXFG4XUoJbYU +# sha256: MmswXL22-U2vv-LCaxHaiLCrB7igf4GIq511_wxuhBo +# sha256: mu3lsrb-RrN0jqjlIURDiQ0WNAJ77z0zt9rRZVaDAng +# sha256: c77R24lNGqnDx-YR0wLN6reuig3A7q92cnh42xrFzYc +# sha256: k1td1tVYr1EvQlAafAj0HXr_E5rxuzlZ2qOruFkjTWw +# sha256: TKARHPFX3MDy9poyPFtUeHGNaNRfyUNdhL4OwPGGIVs +# sha256: tvE8lTmKP88CJsTc-kSFYLpYZSWc2W7CgQZYZR6TIYk +# sha256: 7mvjDRY1u96kxDJdUH3IoNu95-HBmL1i3bn0MZi54hQ +# sha256: 36eGhYwmjX-74bYXXgAewCc418-uCnzne_m2Ua9nZyk +# sha256: qnf53nKvnBbMKIzUokz1iCQ4j1fXqB5ADEYWRXYphw4 +# sha256: 9QAJM1fQTagUDYeTLKwuVO9ZKlTKinQ6uyhQ9gwsIus +psutil==3.3.0 + +# sha256: YfnZnjzvZf6xv-Oi7vepPrk4GdNFv1S81C9OY9UgTa4 +# sha256: GAKm3TIEXkcqQZ2xRBrsq0adM-DSdJ4ZKr3sUhAXJK8 +# sha256: NQJc2UIsllBJEvBOLxX-eTkKhZe0MMLKXQU0z5MJ_6A +# sha256: L5btWgwynKFiMLMmyhK3Rh7I9l4L4-T5l1FvNr-Co0U +# sha256: KP7kQheZHPrZ5qC59-PyYEHiHryWYp6U5YXM0F1J-mU +# sha256: Mm56hUoX-rB2kSBHR2lfj2ktZ0WIo1XEQfsU9mC_Tmg +# sha256: zaWpBIVwnKZ5XIYFbD5f5yZgKLBeU_HVJ_35OmNlprg +# sha256: DLKhR0K1Q_3Wj5MaFM44KRhu0rGyJnoGeHOIyWst2b4 +# sha256: UZH_a5Em0sA53Yf4_wJb7SdLrwf6eK-kb1VrGtcmXW4 +# sha256: gyPgNjey0HLMcEEwC6xuxEjDwolQq0A3YDZ4jpoa9ik +# sha256: hTys2W0fcB3dZ6oD7MBfUYkBNbcmLpInEBEvEqLtKn8 +pyasn1==0.1.9 + +# sha256: eVm0p0q9wnsxL-0cIebK-TCc4LKeqGtZH9Lpns3yf3M +pycparser==2.14 + +# sha256: iORea7Jd_tJyoe8ucoRh1EtjTCzWiemJtuVqNJxaOuU +# sha256: 8KJgcNbbCIHei8x4RpNLfDyTDY-cedRYg-5ImEvA1nI +pyOpenSSL==0.15.1 + +# sha256: 7qMYNcVuIJavQ2OldFp4SHimHQQ-JH06bWoKMql0H1Y +# sha256: jfvGxFi42rocDzYgqMeACLMjomiye3NZ6SpK5BMl9TU +pyRFC3339==1.0 + +# sha256: Z9WdZs26jWJOA4m4eyqDoXbyHxaodVO1D1cDsj8pusI +python-augeas==0.5.0 + +# sha256: BOk_JJlcQ92Q8zjV2GXKcs4_taU1jU2qSWVXHbNfw-w +# sha256: Pm9ZP-rZj4pSa8PjBpM1MyNuM3KfVS9SiW6lBPVTE_o +python2-pythondialog==3.3.0 + +# sha256: Or5qbT_C-75MYBRCEfRdou2-MYKm9lEa9ru6BZix-ZI +# sha256: k575weEiTZgEBWial__PeCjFbRUXsx1zRkNWwfK3dp4 +# sha256: 6tSu-nAHJJ4F5RsBCVcZ1ajdlXYAifVzCqxWmLGTKRg +# sha256: PMoN8IvQ7ZhDI5BJTOPe0AP15mGqRgvnpzS__jWYNgU +# sha256: Pt5HDT0XujwHY436DRBFK8G25a0yYSemW6d-aq6xG-w +# sha256: aMR5ZPcYbuwwaxNilidyK5B5zURH7Z5eyuzU6shMpzQ +# sha256: 3V05kZUKrkCmyB3hV4lC5z1imAjO_FHRLNFXmA5s_Bg +# sha256: p3xSBiwH63x7MFRdvHPjKZW34Rfup1Axe1y1x6RhjxQ +# sha256: ga-a7EvJYKmgEnxIjxh3La5GNGiSM_BvZUQ-exHr61E +# sha256: 4Hmx2txcBiRswbtv4bI6ULHRFz8u3VEE79QLtzoo9AY +# sha256: -9JnRncsJMuTyLl8va1cueRshrvbG52KdD7gDi-x_F0 +# sha256: mSZu8wo35Dky3uwrfKc-g8jbw7n_cD7HPsprHa5r7-o +# sha256: i2zhyZOQl4O8luC0806iI7_3pN8skL25xODxrJKGieM +pytz==2015.7 + +# sha256: ET-7pVManjSUW302szoIToul0GZLcDyBp8Vy2RkZpbg +# sha256: xXeBXdAPE5QgP8ROuXlySwmPiCZKnviY7kW45enPWH8 +requests==2.9.1 + +# sha256: D_eMQD2bzPWkJabTGhKqa0fxwhyk3CVzp-LzKpczXrE +# sha256: EF-NaGFvgkjiS_DpNy7wTTzBAQTxmA9U1Xss5zpa1Wo +six==1.10.0 + +# sha256: glPOvsSxkJTWfMXtWvmb8duhKFKSIm6Yoxkp-HpdayM +# sha256: BazGegmYDC7P7dNCP3rgEEg57MtV_GRXc-HKoJUcMDA +traceback2==1.4.0 + +# sha256: E_d9CHXbbZtDXh1PQedK1MwutuHVyCSZYJKzQw8Ii7g +# sha256: IogqDkGMKE4fcYqCKzsCKUTVPS2QjhaQsxmp0-ssBXk +unittest2==1.1.0 + +# sha256: aUkbUwUVfDxuDwSnAZhNaud_1yn8HJrNJQd_HfOFMms +# sha256: 619wCpv8lkILBVY1r5AC02YuQ9gMP_0x8iTCW8DV9GI +Werkzeug==0.11.3 + +# sha256: KCwRK1XdjjyGmjVx-GdnwVCrEoSprOK97CJsWSrK-Bo +zope.component==4.2.2 + +# sha256: 3HpZov2Rcw03kxMaXSYbKek-xOKpfxvEh86N7-4v54Y +zope.event==4.1.0 + +# sha256: 8HtjH3pgHNjL0zMtVPQxQscIioMpn4WTVvCNHU1CWbM +# sha256: 3lzKCDuUOdgAL7drvmtJmMWlpyH6sluEKYln8ALfTJQ +# sha256: Z4hBb36n9bipe-lIJTd6ol6L3HNGPge6r5hYsp5zcHc +# sha256: bzIw9yVFGCAeWjcIy7LemMhIME8G497Yv7OeWCXLouE +# sha256: X6V1pSQPBCAMMIhCfQ1Le3N_bpAYgYpR2ND5J6aiUXo +# sha256: UiGUrWpUVzXt11yKg_SNZdGvBk5DKn0yDWT1a6_BLpk +# sha256: 6Mey1AlD9xyZFIyX9myqf1E0FH9XQj-NtbSCUJnOmgk +# sha256: J5Ak8CCGAcPKqQfFOHbjetiGJffq8cs4QtvjYLIocBc +# sha256: LiIanux8zFiImieOoT3P7V75OdgLB4Gamos8scaBSE8 +# sha256: aRGJZUEOyG1E3GuQF-4929WC4MCr7vYrOhnb9sitEys +# sha256: 0E34aG7IZNDK3ozxmff4OuzUFhCaIINNVo-DEN7RLeo +# sha256: 51qUfhXul-fnHgLqMC_rL8YtOiu0Zov5377UOlBqx-c +# sha256: TkXSL7iDIipaufKCoRb-xe4ujRpWjM_2otdbvQ62vPw +# sha256: vOkzm7PHpV4IA7Y9IcWDno5Hm8hcSt9CrkFbcvlPrLI +# sha256: koE4NlJFoOiGmlmZ-8wqRUdaCm7VKklNYNvcVAM1_t0 +# sha256: DYQbobuEDuoOZIncXsr6YSVVSXH1O1rLh3ZEQeYbzro +# sha256: sJyMHUezUxxADgGVaX8UFKYyId5u9HhZik8UYPfZo5I +zope.interface==4.1.3 + +# sha256: ilvjjTWOS86xchl0WBZ0YOAw_0rmqdnjNsxb1hq2RD8 +# sha256: T37KMj0TnsuvHIzCCmoww2fpfpOBTj7cd4NAqucXcpw +acme==0.4.0 + +# sha256: 33BQiANlNLGqGpirTfdCEElTF9YbpaKiYpTbK4zeGD8 +# sha256: lwsV1OdEzzlMeb08C_PRxaCXZ2vOk_1AI2755rZHmPM +letsencrypt==0.4.0 + +# sha256: D3YDaVFjLsMSEfjI5B5D5tn5FeWUtNHYXCObw3ih2tg +# sha256: VTgvsePYGRmI4IOSAnxoYFHd8KciD73bxIuIHtbVFd8 +letsencrypt-apache==0.4.0 + +# sha256: uDndLZwRfHAUMMFJlWkYpCOphjtIsJyQ4wpgE-fS9E8 +# sha256: j4MIDaoknQNsvM-4rlzG_wB7iNbZN1ITca-r57Gbrbw +mock==1.0.1 diff --git a/letsencrypt-auto-source/pieces/peep.py b/letsencrypt-auto-source/pieces/peep.py new file mode 100755 index 000000000..c4e51f483 --- /dev/null +++ b/letsencrypt-auto-source/pieces/peep.py @@ -0,0 +1,961 @@ +#!/usr/bin/env python +"""peep ("prudently examine every package") verifies that packages conform to a +trusted, locally stored hash and only then installs them:: + + peep install -r requirements.txt + +This makes your deployments verifiably repeatable without having to maintain a +local PyPI mirror or use a vendor lib. Just update the version numbers and +hashes in requirements.txt, and you're all set. + +""" +# This is here so embedded copies of peep.py are MIT-compliant: +# Copyright (c) 2013 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +try: + xrange = xrange +except NameError: + xrange = range +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify +import cgi +from collections import defaultdict +from functools import wraps +from hashlib import sha256 +from itertools import chain, islice +import mimetypes +from optparse import OptionParser +from os.path import join, basename, splitext, isdir +from pickle import dumps, loads +import re +import sys +from shutil import rmtree, copy +from sys import argv, exit +from tempfile import mkdtemp +import traceback +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler + from urllib.error import HTTPError +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 +# TODO: Probably use six to make urllib stuff work across 2/3. + +from pkg_resources import require, VersionConflict, DistributionNotFound + +# We don't admit our dependency on pip in setup.py, lest a naive user simply +# say `pip install peep.tar.gz` and thus pull down an untrusted copy of pip +# from PyPI. Instead, we make sure it's installed and new enough here and spit +# out an error message if not: + + +def activate(specifier): + """Make a compatible version of pip importable. Raise a RuntimeError if we + couldn't.""" + try: + for distro in require(specifier): + distro.activate() + except (VersionConflict, DistributionNotFound): + raise RuntimeError('The installed version of pip is too old; peep ' + 'requires ' + specifier) + +# Before 0.6.2, the log module wasn't there, so some +# of our monkeypatching fails. It probably wouldn't be +# much work to support even earlier, though. +activate('pip>=0.6.2') + +import pip +from pip.commands.install import InstallCommand +try: + from pip.download import url_to_path # 1.5.6 +except ImportError: + try: + from pip.util import url_to_path # 0.7.0 + except ImportError: + from pip.util import url_to_filename as url_to_path # 0.6.2 +from pip.index import PackageFinder, Link +try: + from pip.log import logger +except ImportError: + from pip import logger # 6.0 +from pip.req import parse_requirements +try: + from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner +except ImportError: + class NullProgressBar(object): + def __init__(self, *args, **kwargs): + pass + + def iter(self, ret, *args, **kwargs): + return ret + + DownloadProgressBar = DownloadProgressSpinner = NullProgressBar + +__version__ = 3, 0, 0 + +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True + + +ITS_FINE_ITS_FINE = 0 +SOMETHING_WENT_WRONG = 1 +# "Traditional" for command-line errors according to optparse docs: +COMMAND_LINE_ERROR = 2 + +ARCHIVE_EXTENSIONS = ('.tar.bz2', '.tar.gz', '.tgz', '.tar', '.zip') + +MARKER = object() + + +class PipException(Exception): + """When I delegated to pip, it exited with an error.""" + + def __init__(self, error_code): + self.error_code = error_code + + +class UnsupportedRequirementError(Exception): + """An unsupported line was encountered in a requirements file.""" + + +class DownloadError(Exception): + def __init__(self, link, exc): + self.link = link + self.reason = str(exc) + + def __str__(self): + return 'Downloading %s failed: %s' % (self.link, self.reason) + + +def encoded_hash(sha): + """Return a short, 7-bit-safe representation of a hash. + + If you pass a sha256, this results in the hash algorithm that the Wheel + format (PEP 427) uses, except here it's intended to be run across the + downloaded archive before unpacking. + + """ + return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') + + +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + +def run_pip(initial_args): + """Delegate to pip the given args (starting with the subcommand), and raise + ``PipException`` if something goes wrong.""" + status_code = pip.main(initial_args) + + # Clear out the registrations in the pip "logger" singleton. Otherwise, + # loggers keep getting appended to it with every run. Pip assumes only one + # command invocation will happen per interpreter lifetime. + logger.consumers = [] + + if status_code: + raise PipException(status_code) + + +def hash_of_file(path): + """Return the hash of a downloaded file.""" + with open(path, 'rb') as archive: + sha = sha256() + while True: + data = archive.read(2 ** 20) + if not data: + break + sha.update(data) + return encoded_hash(sha) + + +def is_git_sha(text): + """Return whether this is probably a git sha""" + # Handle both the full sha as well as the 7-character abbreviation + if len(text) in (40, 7): + try: + int(text, 16) + return True + except ValueError: + pass + return False + + +def filename_from_url(url): + parsed = urlparse(url) + path = parsed.path + return path.split('/')[-1] + + +def requirement_args(argv, want_paths=False, want_other=False): + """Return an iterable of filtered arguments. + + :arg argv: Arguments, starting after the subcommand + :arg want_paths: If True, the returned iterable includes the paths to any + requirements files following a ``-r`` or ``--requirement`` option. + :arg want_other: If True, the returned iterable includes the args that are + not a requirement-file path or a ``-r`` or ``--requirement`` flag. + + """ + was_r = False + for arg in argv: + # Allow for requirements files named "-r", don't freak out if there's a + # trailing "-r", etc. + if was_r: + if want_paths: + yield arg + was_r = False + elif arg in ['-r', '--requirement']: + was_r = True + else: + if want_other: + yield arg + +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') + +HASH_COMMENT_RE = re.compile( + r""" + \s*\#\s+ # Lines that start with a '#' + (?Psha256):\s+ # Hash type is hardcoded to be sha256 for now. + (?P[^\s]+) # Hashes can be anything except '#' or spaces. + \s* # Suck up whitespace before the comment or + # just trailing whitespace if there is no + # comment. Also strip trailing newlines. + (?:\#(?P.*))? # Comments can be anything after a whitespace+# + # and are optional. + $""", re.X) + + +def peep_hash(argv): + """Return the peep hash of one or more files, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + parser = OptionParser( + usage='usage: %prog hash file [file ...]', + description='Print a peep hash line for one or more files: for ' + 'example, "# sha256: ' + 'oz42dZy6Gowxw8AelDtO4gRgTW_xPdooH484k7I5EOY".') + _, paths = parser.parse_args(args=argv) + if paths: + for path in paths: + print('# sha256:', hash_of_file(path)) + return ITS_FINE_ITS_FINE + else: + parser.print_usage() + return COMMAND_LINE_ERROR + + +class EmptyOptions(object): + """Fake optparse options for compatibility with pip<1.2 + + pip<1.2 had a bug in parse_requirements() in which the ``options`` kwarg + was required. We work around that by passing it a mock object. + + """ + default_vcs = None + skip_requirements_regex = None + isolated_mode = False + + +def memoize(func): + """Memoize a method that should return the same result every time on a + given instance. + + """ + @wraps(func) + def memoizer(self): + if not hasattr(self, '_cache'): + self._cache = {} + if func.__name__ not in self._cache: + self._cache[func.__name__] = func(self) + return self._cache[func.__name__] + return memoizer + + +def package_finder(argv): + """Return a PackageFinder respecting command-line options. + + :arg argv: Everything after the subcommand + + """ + # We instantiate an InstallCommand and then use some of its private + # machinery--its arg parser--for our own purposes, like a virus. This + # approach is portable across many pip versions, where more fine-grained + # ones are not. Ignoring options that don't exist on the parser (for + # instance, --use-wheel) gives us a straightforward method of backward + # compatibility. + try: + command = InstallCommand() + except TypeError: + # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 + # given)" error. In that version, InstallCommand takes a top=level + # parser passed in from outside. + from pip.baseparser import create_main_parser + command = InstallCommand(create_main_parser()) + # The downside is that it essentially ruins the InstallCommand class for + # further use. Calling out to pip.main() within the same interpreter, for + # example, would result in arguments parsed this time turning up there. + # Thus, we deepcopy the arg parser so we don't trash its singletons. Of + # course, deepcopy doesn't work on these objects, because they contain + # uncopyable regex patterns, so we pickle and unpickle instead. Fun! + options, _ = loads(dumps(command.parser)).parse_args(argv) + + # Carry over PackageFinder kwargs that have [about] the same names as + # options attr names: + possible_options = [ + 'find_links', + FORMAT_CONTROL_ARG, + ('allow_all_prereleases', 'pre'), + 'process_dependency_links' + ] + kwargs = {} + for option in possible_options: + kw, attr = option if isinstance(option, tuple) else (option, option) + value = getattr(options, attr, MARKER) + if value is not MARKER: + kwargs[kw] = value + + # Figure out index_urls: + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index: + index_urls = [] + index_urls += getattr(options, 'mirrors', []) + + # If pip is new enough to have a PipSession, initialize one, since + # PackageFinder requires it: + if hasattr(command, '_build_session'): + kwargs['session'] = command._build_session(options) + + return PackageFinder(index_urls=index_urls, **kwargs) + + +class DownloadedReq(object): + """A wrapper around InstallRequirement which offers additional information + based on downloading and examining a corresponding package archive + + These are conceptually immutable, so we can get away with memoizing + expensive things. + + """ + def __init__(self, req, argv, finder): + """Download a requirement, compare its hashes, and return a subclass + of DownloadedReq depending on its state. + + :arg req: The InstallRequirement I am based on + :arg argv: The args, starting after the subcommand + + """ + self._req = req + self._argv = argv + self._finder = finder + + # We use a separate temp dir for each requirement so requirements + # (from different indices) that happen to have the same archive names + # don't overwrite each other, leading to a security hole in which the + # latter is a hash mismatch, the former has already passed the + # comparison, and the latter gets installed. + self._temp_path = mkdtemp(prefix='peep-') + # Think of DownloadedReq as a one-shot state machine. It's an abstract + # class that ratchets forward to being one of its own subclasses, + # depending on its package status. Then it doesn't move again. + self.__class__ = self._class() + + def dispose(self): + """Delete temp files and dirs I've made. Render myself useless. + + Do not call further methods on me after calling dispose(). + + """ + rmtree(self._temp_path) + + def _version(self): + """Deduce the version number of the downloaded package from its filename.""" + # TODO: Can we delete this method and just print the line from the + # reqs file verbatim instead? + def version_of_archive(filename, package_name): + # Since we know the project_name, we can strip that off the left, strip + # any archive extensions off the right, and take the rest as the + # version. + for ext in ARCHIVE_EXTENSIONS: + if filename.endswith(ext): + filename = filename[:-len(ext)] + break + # Handle github sha tarball downloads. + if is_git_sha(filename): + filename = package_name + '-' + filename + if not filename.lower().replace('_', '-').startswith(package_name.lower()): + # TODO: Should we replace runs of [^a-zA-Z0-9.], not just _, with -? + give_up(filename, package_name) + return filename[len(package_name) + 1:] # Strip off '-' before version. + + def version_of_wheel(filename, package_name): + # For Wheel files (http://legacy.python.org/dev/peps/pep-0427/#file- + # name-convention) we know the format bits are '-' separated. + whl_package_name, version, _rest = filename.split('-', 2) + # Do the alteration to package_name from PEP 427: + our_package_name = re.sub(r'[^\w\d.]+', '_', package_name, re.UNICODE) + if whl_package_name != our_package_name: + give_up(filename, whl_package_name) + return version + + def give_up(filename, package_name): + raise RuntimeError("The archive '%s' didn't start with the package name " + "'%s', so I couldn't figure out the version number. " + "My bad; improve me." % + (filename, package_name)) + + get_version = (version_of_wheel + if self._downloaded_filename().endswith('.whl') + else version_of_archive) + return get_version(self._downloaded_filename(), self._project_name()) + + def _is_always_unsatisfied(self): + """Returns whether this requirement is always unsatisfied + + This would happen in cases where we can't determine the version + from the filename. + + """ + # If this is a github sha tarball, then it is always unsatisfied + # because the url has a commit sha in it and not the version + # number. + url = self._url() + if url: + filename = filename_from_url(url) + if filename.endswith(ARCHIVE_EXTENSIONS): + filename, ext = splitext(filename) + if is_git_sha(filename): + return True + return False + + @memoize # Avoid hitting the file[cache] over and over. + def _expected_hashes(self): + """Return a list of known-good hashes for this package.""" + return hashes_above(*path_and_line(self._req)) + + def _download(self, link): + """Download a file, and return its name within my temp dir. + + This does no verification of HTTPS certs, but our checking hashes + makes that largely unimportant. It would be nice to be able to use the + requests lib, which can verify certs, but it is guaranteed to be + available only in pip >= 1.5. + + This also drops support for proxies and basic auth, though those could + be added back in. + + """ + # Based on pip 1.4.1's URLOpener but with cert verification removed + def opener(is_https): + if is_https: + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + else: + opener = build_opener() + return opener + + # Descended from unpack_http_url() in pip 1.4.1 + def best_filename(link, response): + """Return the most informative possible filename for a download, + ideally with a proper extension. + + """ + content_type = response.info().get('content-type', '') + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess: + content_disposition = response.info().get('content-disposition') + if content_disposition: + type, params = cgi.parse_header(content_disposition) + # We use ``or`` here because we don't want to use an "empty" value + # from the filename param: + filename = params.get('filename') or filename + ext = splitext(filename)[1] + if not ext: + ext = mimetypes.guess_extension(content_type) + if ext: + filename += ext + if not ext and link.url != response.geturl(): + ext = splitext(response.geturl())[1] + if ext: + filename += ext + return filename + + # Descended from _download_url() in pip 1.4.1 + def pipe_to_file(response, path, size=0): + """Pull the data off an HTTP response, shove it in a new file, and + show progress. + + :arg response: A file-like object to read from + :arg path: The path of the new file + :arg size: The expected size, in bytes, of the download. 0 for + unknown or to suppress progress indication (as for cached + downloads) + + """ + def response_chunks(chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + print('Downloading %s%s...' % ( + self._req.req, + (' (%sK)' % (size / 1000)) if size > 1000 else '')) + progress_indicator = (DownloadProgressBar(max=size).iter if size + else DownloadProgressSpinner().iter) + with open(path, 'wb') as file: + for chunk in progress_indicator(response_chunks(4096), 4096): + file.write(chunk) + + url = link.url.split('#', 1)[0] + try: + response = opener(urlparse(url).scheme != 'http').open(url) + except (HTTPError, IOError) as exc: + raise DownloadError(link, exc) + filename = best_filename(link, response) + try: + size = int(response.headers['content-length']) + except (ValueError, KeyError, TypeError): + size = 0 + pipe_to_file(response, join(self._temp_path, filename), size=size) + return filename + + # Based on req_set.prepare_files() in pip bb2a8428d4aebc8d313d05d590f386fa3f0bbd0f + @memoize # Avoid re-downloading. + def _downloaded_filename(self): + """Download the package's archive if necessary, and return its + filename. + + --no-deps is implied, as we have reimplemented the bits that would + ordinarily do dependency resolution. + + """ + # Peep doesn't support requirements that don't come down as a single + # file, because it can't hash them. Thus, it doesn't support editable + # requirements, because pip itself doesn't support editable + # requirements except for "local projects or a VCS url". Nor does it + # support VCS requirements yet, because we haven't yet come up with a + # portable, deterministic way to hash them. In summary, all we support + # is == requirements and tarballs/zips/etc. + + # TODO: Stop on reqs that are editable or aren't ==. + + # If the requirement isn't already specified as a URL, get a URL + # from an index: + link = self._link() or self._finder.find_requirement(self._req, upgrade=False) + + if link: + lower_scheme = link.scheme.lower() # pip lower()s it for some reason. + if lower_scheme == 'http' or lower_scheme == 'https': + file_path = self._download(link) + return basename(file_path) + elif lower_scheme == 'file': + # The following is inspired by pip's unpack_file_url(): + link_path = url_to_path(link.url_without_fragment) + if isdir(link_path): + raise UnsupportedRequirementError( + "%s: %s is a directory. So that it can compute " + "a hash, peep supports only filesystem paths which " + "point to files" % + (self._req, link.url_without_fragment)) + else: + copy(link_path, self._temp_path) + return basename(link_path) + else: + raise UnsupportedRequirementError( + "%s: The download link, %s, would not result in a file " + "that can be hashed. Peep supports only == requirements, " + "file:// URLs pointing to files (not folders), and " + "http:// and https:// URLs pointing to tarballs, zips, " + "etc." % (self._req, link.url)) + else: + raise UnsupportedRequirementError( + "%s: couldn't determine where to download this requirement from." + % (self._req,)) + + def install(self): + """Install the package I represent, without dependencies. + + Obey typical pip-install options passed in on the command line. + + """ + other_args = list(requirement_args(self._argv, want_other=True)) + archive_path = join(self._temp_path, self._downloaded_filename()) + # -U so it installs whether pip deems the requirement "satisfied" or + # not. This is necessary for GitHub-sourced zips, which change without + # their version numbers changing. + run_pip(['install'] + other_args + ['--no-deps', '-U', archive_path]) + + @memoize + def _actual_hash(self): + """Download the package's archive if necessary, and return its hash.""" + return hash_of_file(join(self._temp_path, self._downloaded_filename())) + + def _project_name(self): + """Return the inner Requirement's "unsafe name". + + Raise ValueError if there is no name. + + """ + name = getattr(self._req.req, 'project_name', '') + if name: + return name + raise ValueError('Requirement has no project_name.') + + def _name(self): + return self._req.name + + def _link(self): + try: + return self._req.link + except AttributeError: + # The link attribute isn't available prior to pip 6.1.0, so fall + # back to the now deprecated 'url' attribute. + return Link(self._req.url) if self._req.url else None + + def _url(self): + link = self._link() + return link.url if link else None + + @memoize # Avoid re-running expensive check_if_exists(). + def _is_satisfied(self): + self._req.check_if_exists() + return (self._req.satisfied_by and + not self._is_always_unsatisfied()) + + def _class(self): + """Return the class I should be, spanning a continuum of goodness.""" + try: + self._project_name() + except ValueError: + return MalformedReq + if self._is_satisfied(): + return SatisfiedReq + if not self._expected_hashes(): + return MissingReq + if self._actual_hash() not in self._expected_hashes(): + return MismatchedReq + return InstallableReq + + @classmethod + def foot(cls): + """Return the text to be printed once, after all of the errors from + classes of my type are printed. + + """ + return '' + + +class MalformedReq(DownloadedReq): + """A requirement whose package name could not be determined""" + + @classmethod + def head(cls): + return 'The following requirements could not be processed:\n' + + def error(self): + return '* Unable to determine package name from URL %s; add #egg=' % self._url() + + +class MissingReq(DownloadedReq): + """A requirement for which no hashes were specified in the requirements file""" + + @classmethod + def head(cls): + return ('The following packages had no hashes specified in the requirements file, which\n' + 'leaves them open to tampering. Vet these packages to your satisfaction, then\n' + 'add these "sha256" lines like so:\n\n') + + def error(self): + if self._url(): + # _url() always contains an #egg= part, or this would be a + # MalformedRequest. + line = self._url() + else: + line = '%s==%s' % (self._name(), self._version()) + return '# sha256: %s\n%s\n' % (self._actual_hash(), line) + + +class MismatchedReq(DownloadedReq): + """A requirement for which the downloaded file didn't match any of my hashes.""" + @classmethod + def head(cls): + return ("THE FOLLOWING PACKAGES DIDN'T MATCH THE HASHES SPECIFIED IN THE REQUIREMENTS\n" + "FILE. If you have updated the package versions, update the hashes. If not,\n" + "freak out, because someone has tampered with the packages.\n\n") + + def error(self): + preamble = ' %s: expected' % self._project_name() + if len(self._expected_hashes()) > 1: + preamble += ' one of' + padding = '\n' + ' ' * (len(preamble) + 1) + return '%s %s\n%s got %s' % (preamble, + padding.join(self._expected_hashes()), + ' ' * (len(preamble) - 4), + self._actual_hash()) + + @classmethod + def foot(cls): + return '\n' + + +class SatisfiedReq(DownloadedReq): + """A requirement which turned out to be already installed""" + + @classmethod + def head(cls): + return ("These packages were already installed, so we didn't need to download or build\n" + "them again. If you installed them with peep in the first place, you should be\n" + "safe. If not, uninstall them, then re-attempt your install with peep.\n") + + def error(self): + return ' %s' % (self._req,) + + +class InstallableReq(DownloadedReq): + """A requirement whose hash matched and can be safely installed""" + + +# DownloadedReq subclasses that indicate an error that should keep us from +# going forward with installation, in the order in which their errors should +# be reported: +ERROR_CLASSES = [MismatchedReq, MissingReq, MalformedReq] + + +def bucket(things, key): + """Return a map of key -> list of things.""" + ret = defaultdict(list) + for thing in things: + ret[key(thing)].append(thing) + return ret + + +def first_every_last(iterable, first, every, last): + """Execute something before the first item of iter, something else for each + item, and a third thing after the last. + + If there are no items in the iterable, don't execute anything. + + """ + did_first = False + for item in iterable: + if not did_first: + did_first = True + first(item) + every(item) + if did_first: + last(item) + + +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + +def downloaded_reqs_from_path(path, argv): + """Return a list of DownloadedReqs representing the requirements parsed + out of a given requirements file. + + :arg path: The path to the requirements file + :arg argv: The commandline args, starting after the subcommand + + """ + finder = package_finder(argv) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] + + +def peep_install(argv): + """Perform the ``peep install`` subcommand, returning a shell status code + or raising a PipException. + + :arg argv: The commandline args, starting after the subcommand + + """ + output = [] + out = output.append + reqs = [] + try: + req_paths = list(requirement_args(argv, want_paths=True)) + if not req_paths: + out("You have to specify one or more requirements files with the -r option, because\n" + "otherwise there's nowhere for peep to look up the hashes.\n") + return COMMAND_LINE_ERROR + + # We're a "peep install" command, and we have some requirement paths. + reqs = list(chain.from_iterable( + downloaded_reqs_from_path(path, argv) + for path in req_paths)) + buckets = bucket(reqs, lambda r: r.__class__) + + # Skip a line after pip's "Cleaning up..." so the important stuff + # stands out: + if any(buckets[b] for b in ERROR_CLASSES): + out('\n') + + printers = (lambda r: out(r.head()), + lambda r: out(r.error() + '\n'), + lambda r: out(r.foot())) + for c in ERROR_CLASSES: + first_every_last(buckets[c], *printers) + + if any(buckets[b] for b in ERROR_CLASSES): + out('-------------------------------\n' + 'Not proceeding to installation.\n') + return SOMETHING_WENT_WRONG + else: + for req in buckets[InstallableReq]: + req.install() + + first_every_last(buckets[SatisfiedReq], *printers) + + return ITS_FINE_ITS_FINE + except (UnsupportedRequirementError, DownloadError) as exc: + out(str(exc)) + return SOMETHING_WENT_WRONG + finally: + for req in reqs: + req.dispose() + print(''.join(output)) + + +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + +def main(): + """Be the top-level entrypoint. Return a shell status code.""" + commands = {'hash': peep_hash, + 'install': peep_install, + 'port': peep_port} + try: + if len(argv) >= 2 and argv[1] in commands: + return commands[argv[1]](argv[2:]) + else: + # Fall through to top-level pip main() for everything else: + return pip.main() + except PipException as exc: + return exc.error_code + + +def exception_handler(exc_type, exc_value, exc_tb): + print('Oh no! Peep had a problem while trying to do stuff. Please write up a bug report') + print('with the specifics so we can fix it:') + print() + print('https://github.com/erikrose/peep/issues/new') + print() + print('Here are some particulars you can copy and paste into the bug report:') + print() + print('---') + print('peep:', repr(__version__)) + print('python:', repr(sys.version)) + print('pip:', repr(getattr(pip, '__version__', 'no __version__ attr'))) + print('Command line: ', repr(sys.argv)) + print( + ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))) + print('---') + + +if __name__ == '__main__': + try: + exit(main()) + except Exception: + exception_handler(*sys.exc_info()) + exit(SOMETHING_WENT_WRONG) diff --git a/letsencrypt-auto-source/tests/__init__.py b/letsencrypt-auto-source/tests/__init__.py new file mode 100644 index 000000000..45db90444 --- /dev/null +++ b/letsencrypt-auto-source/tests/__init__.py @@ -0,0 +1,7 @@ +"""Tests for letsencrypt-auto + +Run these locally by saying... :: + + ./build.py && docker build -t lea . && docker run --rm -t -i lea + +""" diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py new file mode 100644 index 000000000..90e09f57f --- /dev/null +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -0,0 +1,349 @@ +"""Tests for letsencrypt-auto""" + +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from contextlib import contextmanager +from functools import partial +from json import dumps +from os import chmod, environ +from os.path import abspath, dirname, exists, join +import re +from shutil import copy, rmtree +import socket +import ssl +from stat import S_IRUSR, S_IXUSR +from subprocess import CalledProcessError, check_output, Popen, PIPE +import sys +from tempfile import mkdtemp +from threading import Thread +from unittest import TestCase + +from nose.tools import eq_, nottest, ok_ + + +@nottest +def tests_dir(): + """Return a path to the "tests" directory.""" + return dirname(abspath(__file__)) + + +sys.path.insert(0, dirname(tests_dir())) +from build import build as build_le_auto + + +class RequestHandler(BaseHTTPRequestHandler): + """An HTTPS request handler which is quiet and serves a specific folder.""" + + def __init__(self, resources, *args, **kwargs): + """ + :arg resources: A dict of resource paths pointing to content bytes + + """ + self.resources = resources + BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + """Don't log each request to the terminal.""" + + def do_GET(self): + """Serve a GET request.""" + content = self.send_head() + if content is not None: + self.wfile.write(content) + + def send_head(self): + """Common code for GET and HEAD commands + + This sends the response code and MIME headers and returns either a + bytestring of content or, if none is found, None. + + """ + path = self.path[1:] # Strip leading slash. + content = self.resources.get(path) + if content is None: + self.send_error(404, 'Path "%s" not found in self.resources' % path) + else: + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.send_header('Content-Length', str(len(content))) + self.end_headers() + return content + + +def server_and_port(resources): + """Return an unstarted HTTPS server and the port it will use.""" + # Find a port, and bind to it. I can't get the OS to close the socket + # promptly after we shut down the server, so we typically need to try + # a couple ports after the first test case. Setting + # TCPServer.allow_reuse_address = True seems to have nothing to do + # with this behavior. + worked = False + for port in xrange(4443, 4543): + try: + server = HTTPServer(('localhost', port), + partial(RequestHandler, resources)) + except socket.error: + pass + else: + worked = True + server.socket = ssl.wrap_socket( + server.socket, + certfile=join(tests_dir(), 'certs', 'localhost', 'server.pem'), + server_side=True) + break + if not worked: + raise RuntimeError("Couldn't find an unused socket for the testing HTTPS server.") + return server, port + + +@contextmanager +def serving(resources): + """Spin up a local HTTPS server, and yield its base URL. + + Use a self-signed cert generated as outlined by + https://coolaj86.com/articles/create-your-own-certificate-authority-for- + testing/. + + """ + server, port = server_and_port(resources) + thread = Thread(target=server.serve_forever) + try: + thread.start() + yield 'https://localhost:{port}/'.format(port=port) + finally: + server.shutdown() + thread.join() + + +LE_AUTO_PATH = join(dirname(tests_dir()), 'letsencrypt-auto') + + +@contextmanager +def ephemeral_dir(): + dir = mkdtemp(prefix='le-test-') + try: + yield dir + finally: + rmtree(dir) + + +def out_and_err(command, input=None, shell=False, env=None): + """Run a shell command, and return stderr and stdout as string. + + If the command returns nonzero, raise CalledProcessError. + + :arg command: A list of commandline args + :arg input: Data to pipe to stdin. Omit for none. + + Remaining args have the same meaning as for Popen. + + """ + process = Popen(command, + stdout=PIPE, + stdin=PIPE, + stderr=PIPE, + shell=shell, + env=env) + out, err = process.communicate(input=input) + status = process.poll() # same as in check_output(), though wait() sounds better + if status: + raise CalledProcessError(status, command, output=out) + return out, err + + +def signed(content, private_key_name='signing.key'): + """Return the signed SHA-256 hash of ``content``, using the given key file.""" + command = ['openssl', 'dgst', '-sha256', '-sign', + join(tests_dir(), private_key_name)] + out, err = out_and_err(command, input=content) + return out + + +def install_le_auto(contents, venv_dir): + """Install some given source code as the letsencrypt-auto script at the + root level of a virtualenv. + + :arg contents: The contents of the built letsencrypt-auto script + :arg venv_dir: The path under which to install the script + + """ + venv_le_auto_path = join(venv_dir, 'letsencrypt-auto') + with open(venv_le_auto_path, 'w') as le_auto: + le_auto.write(contents) + chmod(venv_le_auto_path, S_IRUSR | S_IXUSR) + + +def run_le_auto(venv_dir, base_url, **kwargs): + """Run the prebuilt version of letsencrypt-auto, returning stdout and + stderr strings. + + If the command returns other than 0, raise CalledProcessError. + + """ + env = environ.copy() + d = dict(XDG_DATA_HOME=venv_dir, + # URL to PyPI-style JSON that tell us the latest released version + # of LE: + LE_AUTO_JSON_URL=base_url + 'letsencrypt/json', + # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: + LE_AUTO_DIR_TEMPLATE=base_url + '%s/', + # The public key corresponding to signing.key: + LE_AUTO_PUBLIC_KEY="""-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg +tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G +hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT +uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl +LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 +Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 +iQIDAQAB +-----END PUBLIC KEY-----""", + **kwargs) + env.update(d) + return out_and_err( + join(venv_dir, 'letsencrypt-auto') + ' --version', + shell=True, + env=env) + + +def set_le_script_version(venv_dir, version): + """Tell the letsencrypt script to report a certain version. + + We actually replace the script with a dummy version that knows only how to + print its version. + + """ + with open(join(venv_dir, 'letsencrypt', 'bin', 'letsencrypt'), 'w') as script: + script.write("#!/usr/bin/env python\n" + "from sys import stderr\n" + "stderr.write('letsencrypt %s\\n')" % version) + + +class AutoTests(TestCase): + """Test the major branch points of letsencrypt-auto: + + * An le-auto upgrade is needed. + * An le-auto upgrade is not needed. + * There was an out-of-date LE script installed. + * There was a current LE script installed. + * There was no LE script installed (less important). + * Peep verification passes. + * Peep has a hash mismatch. + * The OpenSSL sig matches. + * The OpenSSL sig mismatches. + + For tests which get to the end, we run merely ``letsencrypt --version``. + The functioning of the rest of the letsencrypt script is covered by other + test suites. + + """ + def test_successes(self): + """Exercise most branches of letsencrypt-auto. + + They just happen to be the branches in which everything goes well. + + I violate my usual rule of having small, decoupled tests, because... + + 1. We shouldn't need to run a Cartesian product of the branches: the + phases run in separate shell processes, containing state leakage + pretty effectively. The only shared state is FS state, and it's + limited to a temp dir, assuming (if we dare) all functions properly. + 2. One combination of branches happens to set us up nicely for testing + the next, saving code. + + """ + NEW_LE_AUTO = build_le_auto( + version='99.9.9', + requirements='# sha256: HMFNYatCTN7kRvUeUPESP4SC7HQFh_54YmyTO7ooc6A\n' + 'letsencrypt==99.9.9') + NEW_LE_AUTO_SIG = signed(NEW_LE_AUTO) + + with ephemeral_dir() as venv_dir: + # This serves a PyPI page with a higher version, a GitHub-alike + # with a corresponding le-auto script, and a matching signature. + resources = {'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG} + with serving(resources) as base_url: + run_letsencrypt_auto = partial( + run_le_auto, + venv_dir, + base_url, + PIP_FIND_LINKS=join(tests_dir(), + 'fake-letsencrypt', + 'dist')) + + # Test when a phase-1 upgrade is needed, there's no LE binary + # installed, and peep verifies: + install_le_auto(build_le_auto(version='50.0.0'), venv_dir) + out, err = run_letsencrypt_auto() + ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', + err.strip().splitlines()[-1])) + # Make a few assertions to test the validity of the next tests: + self.assertIn('Upgrading letsencrypt-auto ', out) + self.assertIn('Creating virtual environment...', out) + + # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This + # conveniently sets us up to test the next 2 cases. + + # Test when neither phase-1 upgrade nor phase-2 upgrade is + # needed (probably a common case): + out, err = run_letsencrypt_auto() + self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertNotIn('Creating virtual environment...', out) + + # Test when a phase-1 upgrade is not needed but a phase-2 + # upgrade is: + set_le_script_version(venv_dir, '0.0.1') + out, err = run_letsencrypt_auto() + self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertIn('Creating virtual environment...', out) + + def test_openssl_failure(self): + """Make sure we stop if the openssl signature check fails.""" + with ephemeral_dir() as venv_dir: + # Serve an unrelated hash signed with the good key (easier than + # making a bad key, and a mismatch is a mismatch): + resources = {'': 'letsencrypt/', + 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), + 'v99.9.9/letsencrypt-auto.sig': signed('something else')} + with serving(resources) as base_url: + copy(LE_AUTO_PATH, venv_dir) + try: + out, err = run_le_auto(venv_dir, base_url) + except CalledProcessError as exc: + eq_(exc.returncode, 1) + self.assertIn("Couldn't verify signature of downloaded " + "letsencrypt-auto.", + exc.output) + else: + self.fail('Signature check on letsencrypt-auto erroneously passed.') + + def test_peep_failure(self): + """Make sure peep stops us if there is a hash mismatch.""" + with ephemeral_dir() as venv_dir: + resources = {'': 'letsencrypt/', + 'letsencrypt/json': dumps({'releases': {'99.9.9': None}})} + with serving(resources) as base_url: + # Build a le-auto script embedding a bad requirements file: + install_le_auto( + build_le_auto( + version='99.9.9', + requirements='# sha256: badbadbadbadbadbadbadbadbadbadbadbadbadbadb\n' + 'configobj==5.0.6'), + venv_dir) + try: + out, err = run_le_auto(venv_dir, base_url) + except CalledProcessError as exc: + eq_(exc.returncode, 1) + self.assertIn("THE FOLLOWING PACKAGES DIDN'T MATCH THE " + "HASHES SPECIFIED IN THE REQUIREMENTS", + exc.output) + ok_(not exists(join(venv_dir, 'letsencrypt')), + msg="The virtualenv was left around, even though " + "installation didn't succeed. We shouldn't do " + "this, as it foils our detection of whether we " + "need to recreate the virtualenv, which hinges " + "on the presence of $VENV_BIN/letsencrypt.") + else: + self.fail("Peep didn't detect a bad hash and stop the " + "installation.") diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem new file mode 100644 index 000000000..4e4d29bd2 --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.crt.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID5jCCAs6gAwIBAgIJAI1Qkfyw88REMA0GCSqGSIb3DQEBBQUAMFUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMRswGQYDVQQKExJNeSBCb2d1cyBS +b290IENlcnQxFDASBgNVBAMTC2V4YW1wbGUuY29tMB4XDTE1MTIwNDIwNTIxNVoX +DTQwMTIwMzIwNTIxNVowVTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3Rh +dGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJvb3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBs +ZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQVQpQ2EH4gTJB +NJP6+ocT3xJwT8mSXYUnvzjj6iv+JxZiXRGzAPziNzrrSRKY0yDHF+UiJwuOerLa +n8laZkLb1Ogqzs2u64rKeb0xWv90Qp+eXG0J/1xb4dw+GExqe5QFo1JUJzO/eK7m +1S04SeFkN1qV9mD5yJUy7DGiTUzDHgCxM2tXMLusXYqkxsQQ9+2EJ7BEOK4YJGEx +Sign5FuSxb64PiNow6OA97CaLl7tV4INP4w195ueDRIaS4poeOep4s8U7IAdMjIZ +EryJgKNCij50xK92vPBBJSj0NOitltBlwoEqkOZpQCOZamFd6nvt78LQ6W8Am+l6 +y6oCON5JAgMBAAGjgbgwgbUwHQYDVR0OBBYEFAlrdStDhaayLLj89Whe3Gc+HE8y +MIGFBgNVHSMEfjB8gBQJa3UrQ4Wmsiy4/PVoXtxnPhxPMqFZpFcwVTELMAkGA1UE +BhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxGzAZBgNVBAoTEk15IEJvZ3VzIFJv +b3QgQ2VydDEUMBIGA1UEAxMLZXhhbXBsZS5jb22CCQCNUJH8sPPERDAMBgNVHRME +BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQC7KAQfDTiNM3QO8Ic3x21CAPJUavkH +zshifN+Ei0+nmseHDTCTgsGfGDOToLUpUEZ4PuiHnz08UwRfd9wotc3SgY9ZaXMe +vRs8KUAF9EoyTvESzPyv2b6cS9NNMpj5y7KyXSyP17VoGbNavtiGQ4dwgEH6VgNl +0RtBvcSBv/tqxIIx1tWzL74tVEm0Kbd9BAZsYpQNKL8e6WXP35/j0PvCCvtofGrA +E8LTqMz4kCwnX+QaJIMJhBophRCsjXdAkvFbFxX0DGPztQtzIwBPcdMjsft7AFeE +0XchhDDXxw8YsbpvPfCvrD8XiiVuBycbnB1zt0LLVwB/QsCzUW9ImpLC +-----END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem new file mode 100644 index 000000000..9caa7ddaa --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0FUKUNhB+IEyQTST+vqHE98ScE/Jkl2FJ7844+or/icWYl0R +swD84jc660kSmNMgxxflIicLjnqy2p/JWmZC29ToKs7NruuKynm9MVr/dEKfnlxt +Cf9cW+HcPhhManuUBaNSVCczv3iu5tUtOEnhZDdalfZg+ciVMuwxok1Mwx4AsTNr +VzC7rF2KpMbEEPfthCewRDiuGCRhMUooJ+RbksW+uD4jaMOjgPewmi5e7VeCDT+M +Nfebng0SGkuKaHjnqeLPFOyAHTIyGRK8iYCjQoo+dMSvdrzwQSUo9DTorZbQZcKB +KpDmaUAjmWphXep77e/C0OlvAJvpesuqAjjeSQIDAQABAoIBAH+qbVzneV3wxjwh +HUHi/p3VyHXc3xh7iNq3mwRH/1eK2nPCttLsGwwBbnC64dOXJfH7maWZKcLRPAMv +gfOM0RHn4bJB8tdrbizv91lke0DihvBDkWpb+1wvB4lh2Io0Wpwt3ojFUTfXm87G ++iQRWjbQmQlm5zyKh6uiBDSCjDTQdb9omZEBMAwlGPTZwt8TRUEtWd8QgW8FCHoB +iLER2WBwXdvn3PBtocI3VE6IYDSeZ81Xv+d7925RtVintT8Suk4toYwX+jfSz+wZ +sgHd5V6PSv9a7GUlWoUihD99D9wqDZE8IvMDZ5ofSAUd1KfICDtmsEyugY7u2yYZ +tYt49AECgYEA73f7ITMHg8JsUipqb6eG10gCRtRhkqrrO1g/TNeTBh3CTrQGb56e +y6kmUivn5gK46t3T2N4Ht4IR8fpLcJcbPYPQNulSjmWm5y6WduafXW/VCW1NA9Lc +FyGPkMxFCIVJTLFxfLFepBVvtUzLLDKGGtQxru/GNbBzjdtmVfDPIoECgYEA3rbM +cTfvj+jWrV1YsRbphyjy+k3OJEIVx6KA4s5d7Tp12UfYQp/B3HPhXXm5wqeo1Nos +UAEWZIMi1VoE8iu6jjeJ6uERtbKKQVed25Us/ff0jUPbxlXgiBOtRcllq9d9Srjm +ybHUgfjLsZ2/xpIcOl+oI5pDM9JvD8Sq4ZCFR8kCgYBK/H0tFjeiML2OtS2DLShy +PWBJIbQ0I0Vp3eZkf5TQc30m/ASP61G6YItZa9pAElYpZbEy1cQA2MAZz9DTvt2O +07ndmA57/KTY+6OuM+Vvctd5DjrxmZPFwoKcSvrLAkHDvETXUQtbwkKquRNeEawg +tpWgPAELSufEYhGXk8KpAQKBgBDCqPgMQZcO6rj5QWdyVfi5+C8mE9Fet8ziSdjH +twHXWG8VnQzGgQxaHCewtW4Ut/vsv1D2A/1kcQalU6H18IArZdGrRm3qFcV9FoAj +5dLnChxncu6mH9Odx3htA52/BcrNx3B+VYPCeXHQcVI8RKuP71NelJgdygXhwwpe +mekhAoGBAOUovnqylciYa9HRqo+xZk59eyX+ehhnlV8SeJ2K0PwaQkzQ0KYtCmE7 +kdSdhcv8h/IQKGaFfc/LyFMM/a26PfAeY5bj41UjkT0K5hQrYuL/52xaT401YLcb +Xo+bZz9K0hrdP7TdZFuTY/WxojXgjsVAuAN1NwnJumqxhzPh+hfl +-----END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl new file mode 100644 index 000000000..ad6d262b4 --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/ca/my-root-ca.srl @@ -0,0 +1 @@ +D613482D0EF95DD0 diff --git a/letsencrypt-auto-source/tests/certs/localhost/cert.pem b/letsencrypt-auto-source/tests/certs/localhost/cert.pem new file mode 100644 index 000000000..ac83535ce --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/localhost/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD +ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy +MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw +HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH +a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y +DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 +SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T +Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn +ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM +V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P +NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n +v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN +AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond +3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi +uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== +-----END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem b/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem new file mode 100644 index 000000000..8a6189f88 --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/localhost/localhost.csr.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICnjCCAYYCAQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx +ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJbG9j +YWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZYgiLzoyKzh +RAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/ZfeUJ5aEqavcIlhdWADur/bc85FACK5 +XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGseR7+IDxOQO5ltYbNUtvxMHzeKkE4 +PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E+IJCXDI5rKMeZ2WHxyp9UTytYSbn +/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAdvQJoUQb1C65QM8mXkrvhGvoicxBk +o+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrKVzYV25ruj/B/RLBKHFLgDUOoD8dY +sQxXoxIQXwIDAQABoAAwDQYJKoZIhvcNAQEFBQADggEBAFbg3WrAokoPx7iAYG6z +PqeDd4/XanXjeL4Ryxv6LoGhu69mmBAd3N5ILPyQJjnkWpIjEmJDzEcPMzhQjRh5 +GlWTyvKWO4zClYU840KZk7crVkpzNZ+HP0YeM/Agz6sab00ffRcq5m1wEF9MCvDE +8FUXk1HBHRAb/6t9QV/7axsPOkGT8SjQ1v2SCaiB0HQL3sYChYLi5zu4dfmQNPGq +ar9Xm5a0YqOQIFfmy8RSwxk0Q/ipNFTGN1uvlIRkgbT9zPnodxjWZsSI9BF+q5Af +uiE/oAk7MxfJ0LyLfhOWB+T98bKIOVtFT3wMLS1IIgMogwqCEXFf30Q9p2iTEzqT +6UE= +-----END CERTIFICATE REQUEST----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/privkey.pem b/letsencrypt-auto-source/tests/certs/localhost/privkey.pem new file mode 100644 index 000000000..18feba403 --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/localhost/privkey.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe +UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs +eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E ++IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd +vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK +VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 +16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK +46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 +K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P +EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 +Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP +0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x +h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk +JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX +lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K +Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX +nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji +5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl +UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K +fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR +tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G +Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO +mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX +qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB +okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow +-----END RSA PRIVATE KEY----- diff --git a/letsencrypt-auto-source/tests/certs/localhost/server.pem b/letsencrypt-auto-source/tests/certs/localhost/server.pem new file mode 100644 index 000000000..c5765dd89 --- /dev/null +++ b/letsencrypt-auto-source/tests/certs/localhost/server.pem @@ -0,0 +1,46 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArZYgiLzoyKzhRAdrk+sVtfY8G1Adsje3d6Q3UJ7BGyBD/Zfe +UJ5aEqavcIlhdWADur/bc85FACK5XrIMbZ1AiuN5qFYJdoKm1OLj2WN9VFNbYjGs +eR7+IDxOQO5ltYbNUtvxMHzeKkE4PjVKO6iag3gD+S2ch9s2pGqCOMezbDAkcN/E ++IJCXDI5rKMeZ2WHxyp9UTytYSbn/dMi7RfUnndJqaZHVJtSawsk2h/EVgwrWiAd +vQJoUQb1C65QM8mXkrvhGvoicxBko+ed7hbdwagAvE8cBxrvKJylIxKZ2yZ3qmrK +VzYV25ruj/B/RLBKHFLgDUOoD8dYsQxXoxIQXwIDAQABAoIBAG8bVJ+xKt6nqVg9 +16HKKw9ZGIfy888K0qgFuFImCzwtntdGycmYUdb2Uf0aMgNK/ZgfDXxGXuwDTdtK +46GVsaY0i74vs8bjQZ2pzGVsxN+gqzFi0h6Es+w2LXBqJzfVnL6YgPykMB+jtzg6 +K9Wbyaq0uvZXN4XNzl/WvJtTV4i7Cff1MOd5EhKFdqxrZvB/SRBCr/SMMafRtB9P +EvMneNKzhmlrutHAxuyxEKZR32Kkx7ydAdTjGgn+rE+NL5BweXfeWhLU4Bv14bn9 +Mkneu3w5o1ryJfE2YnVajUP//jeopUT0nTQ3MpEusBQCLBlvFXjjM9uCaFX+5+MP +0H4xVcECgYEA1Q+wR3GHbk37vIGSlbENyUsri5WlMt8IVAHsDsTOpxAjYB0yyo+x +h9RS+RJZQECJlA6H72peUl3GM7RgdWIcKOT3nZ12XqYKG57rr/N5zlUuxbdS8KBk +JhyZeJdYjq/Jrno1ZP+OSmc7VvBLcM7irY7LHlvK0o8W1W0TNJ8jrZkCgYEA0JHX +lJd+fiezcUS7g4moHtzJp0JKquQiXLX+c2urmpyhb3ZrTuQ8OUjSy6DlwHlgDx8K +Hg2sdx/ZCuDaGjR4IY/Qs5RFt9WUqlK9gi9V3nYVrzBOQkdFOf/Ad3j4pQ8/aeCX +nP6snHXz1WqPpbCXG6l6GzFGbQU473GfuKsDuLcCgYAWQaNKc0OQdDj9whNL68ji +5CVSWXl+TOoTzHeaO1jS/s6TNbmei1AiPj3EovQL0DIO802j5tqfhAg2UntZB7yl +UPXE0zQQQwv/QqSgJrDsqt1N7g6N8FNF3+rwO+8WSKqqvT1ipYd5ojsCo+tdh18K +fkYdj70qLaRW+yPsdUtG0QKBgEYc8NqbvsML94+ZKmwCh4iwcf2PFGi0PjTqXTpR +tKNKCh7dMR+ZLAGZ0HrxgKqeYsNSjOUjdZmqFB1LDyaGAuhNXzwvGOy+mLZVEC3G +Wdhp28pDs9sl+EiSCBJhkTxzjr656F23YzFJmYlhxB5P6cw7wbeIbgNSIRylFqtO +mfarAoGBAICsAEWypOctxtmtOcjxgJ7jMbOA7rrsGlXpiy1/WlwIwRGF5LMvIIFX +qFAfiPcZn05ZgdAGzaFYowdjmQB10FW0jZbDf+nIHfOF5YmfmfWjsaweEGALJmqB +okGu/lGNGf3XoYzy0/hC3WAqk3znSZtQLUq8jEWF7dLNUizUeUow +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDKjCCAhICCQDWE0gtDvld0DANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJB +VTETMBEGA1UECBMKU29tZS1TdGF0ZTEbMBkGA1UEChMSTXkgQm9ndXMgUm9vdCBD +ZXJ0MRQwEgYDVQQDEwtleGFtcGxlLmNvbTAeFw0xNTEyMDQyMDU0MzFaFw00MDEy +MDMyMDU0MzFaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEw +HwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2WIIi86Mis4UQH +a5PrFbX2PBtQHbI3t3ekN1CewRsgQ/2X3lCeWhKmr3CJYXVgA7q/23PORQAiuV6y +DG2dQIrjeahWCXaCptTi49ljfVRTW2IxrHke/iA8TkDuZbWGzVLb8TB83ipBOD41 +SjuomoN4A/ktnIfbNqRqgjjHs2wwJHDfxPiCQlwyOayjHmdlh8cqfVE8rWEm5/3T +Iu0X1J53SammR1SbUmsLJNofxFYMK1ogHb0CaFEG9QuuUDPJl5K74Rr6InMQZKPn +ne4W3cGoALxPHAca7yicpSMSmdsmd6pqylc2Fdua7o/wf0SwShxS4A1DqA/HWLEM +V6MSEF8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAz5sMAFG6W/ZEULZITkBTCU6P +NttpGiKufnqyBW5HyNylaczfnHnClvQjr8f/84xvKVcfC3xP0lz+92aIQqo+5L/n +v7gLhBFR4Vr2XwMt2qz2FpkaxmVwnhVAHaaC05WIKQ6W2gDwWT0u1K8YdTh+7mvN +AT9FW4vDgtNZWq4W/PePh9QCiOOQhGOuBYj/7zqLtz4XPifhi66ILIRDHiu0kond +3YMFcECIAf4MPT9vT0iNcWX+c8CfAixPt8nMD6bzOo3oTcfuZh/2enfgLbMqOlOi +uk72FM5VVPXTWAckJvL/vVjqsvDuJQKqbr0oUc3bdWbS36xtWZUycp4IQLguAQ== +-----END CERTIFICATE----- diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz new file mode 100644 index 000000000..5f9a48a34 Binary files /dev/null and b/letsencrypt-auto-source/tests/fake-letsencrypt/dist/letsencrypt-99.9.9.tar.gz differ diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py new file mode 100755 index 000000000..9d811fab5 --- /dev/null +++ b/letsencrypt-auto-source/tests/fake-letsencrypt/letsencrypt.py @@ -0,0 +1,8 @@ +from sys import argv, stderr + + +def main(): + """Act like letsencrypt --version insofar as printing the version number to + stderr.""" + if '--version' in argv: + stderr.write('letsencrypt 99.9.9\n') diff --git a/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py new file mode 100644 index 000000000..e5f7fde35 --- /dev/null +++ b/letsencrypt-auto-source/tests/fake-letsencrypt/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup + + +setup( + name='letsencrypt', + version='99.9.9', + description='A mock version of letsencrypt that just prints its version', + py_modules=['letsencrypt'], + entry_points={ + 'console_scripts': ['letsencrypt = letsencrypt:main'] + } +) diff --git a/letsencrypt-auto-source/tests/signing.key b/letsencrypt-auto-source/tests/signing.key new file mode 100644 index 000000000..b9964d00c --- /dev/null +++ b/letsencrypt-auto-source/tests/signing.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAsMoSzLYQ7E1sdSOkwelgtzKIh2qi3bpXuYtcfFC0XrvWig07 +1NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7GhFW0VdbxL6JdGzS2ShNWkX9hE9z+ +j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTTuUtJmmGcuk3a9Aq/sCT6DdfmTSdP +5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVglLsIVPBuy9IcgHidUQ96hJnoPsDCW +sHwX62495QKEarauyKQrJzFes0EY95orDM47Z5o/NDiQB11m91yNB0MmPYY9QSbn +OA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68iQIDAQABAoIBAQCJE3W2Mqk2f+XL +geKa1BjAkzcXQJCduYGRhUQlw/HGzoBPtGki56Tf53MeHTAkIGfIq3CAr1zRhiNv +8SQzvrLQIx/buvhxhcQJdzqsfwgNcqXT3/OliF34P3LMx8GUfPy/6xq2Qdv4fvwA +nLJH8wyDTKP6RxtdvUY7GSZ+Ln2QQv/3Nco7tax4GHNGom8iSgeH/YKTDnvitdqh +a0fr930QzU39TfOftLmasdmKUOIg8G2wr4Sy6Kn060+OUoQr1fZF5mnLvvQeILCK +uav91JkIeMLggzk+t88IJUFWdOoxv5hWTnNzHyt+/GYfovyRz2fKQMwzdh1F8iM5 ++867rEb9AoGBANn1ncemJBedDshStdCBUH0+2ExPrawveaXOZKnx8/VGFXNi0hAf +KzkntMWd5g5kB077FtKO9CYTBvK4pZBWIFLcJEqAz88JeXME6dfUbRucDr72ko+l +rcLHXj7F0IDVzj/9CphMGAhC9J/4YW9SPcSbMw6dQ6xOk73f1Vowve0DAoGBAM+k +/F+hVqCS3f22Bg9KuDtx+zCydaZxC842DgIkV1SO2iFhNHjnpQ5EIR0WrSYeV2n+ +rD7kVs5OH1HvnGScHaQKtAVqZClSwF14jzE+Aj8XDwxiHLSOhJgKlzfVX7h1ymMh +7fsslDl6xNGQ+40gubhkCLT5qABFKy1mrZ8b+3yDAoGAGLGUI6d2FVrM7vM3+Bx+ +gwIYvWSVl5l1XcypaPupmRNMoNsEU6FEY2BVQcJm6yB4F4GpD0f0709ejSdQUq7/ +UIPydKJtaNZ49QgMelBt4B/pJ8eFyVKLAjNWQSRmQAJ5MJS5m5Gbc2wqjOk2GMen +idvPiAtXPHFWmb9/S42UJwMCgYEAjymAe2qgcGtyNNfIC8kHhqzKdEPGi/ALJKzu +MZnewEURrcv4QpfrnA9rCUQ2Mz7eJA1bsqz6EJmaTIK4wEFGynA6uDUnQ7pzOL7D +cz7+i4MZc/89LVvJnY5Hvk4WBfboiDq/etq8g3jatGaSmTYD9la6DhTHORB3eYD+ +meHQHYMCgYEA18y9hnx2k4vNeBei4YXF4pAvKdwKLQD+CcP9ljb3VT+kXktjRA1C +aWj3HhMwvcxtttfkQzEnwwGRAkTEtNewJ8KFxhmc9nYElZTNZ+SuHD5Dkv8xqoj8 +NvG8rU1eiEyPwE2wQxpM5JLqbo7IWtR0dmptjKoF1gRxn6Wh4TwEiHA= +-----END RSA PRIVATE KEY----- diff --git a/letsencrypt-compatibility-test/setup.py b/letsencrypt-compatibility-test/setup.py index 1ff9e7649..67262ba72 100644 --- a/letsencrypt-compatibility-test/setup.py +++ b/letsencrypt-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.5.0.dev0' install_requires = [ 'letsencrypt=={0}'.format(version), diff --git a/letsencrypt-nginx/letsencrypt_nginx/configurator.py b/letsencrypt-nginx/letsencrypt_nginx/configurator.py index 4a5a3ddcd..d2f45a8c2 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/configurator.py +++ b/letsencrypt-nginx/letsencrypt_nginx/configurator.py @@ -5,7 +5,6 @@ import re import shutil import socket import subprocess -import sys import time import OpenSSL @@ -106,11 +105,18 @@ class NginxConfigurator(common.Plugin): # This is called in determine_authenticator and determine_installer def prepare(self): - """Prepare the authenticator/installer.""" + """Prepare the authenticator/installer. + + :raises .errors.NoInstallationError: If Nginx ctl cannot be found + :raises .errors.MisconfigurationError: If Nginx is misconfigured + """ # Verify Nginx is installed if not le_util.exe_exists(self.conf('ctl')): raise errors.NoInstallationError + # Make sure configuration is valid + self.config_test() + self.parser = parser.NginxParser( self.conf('server-root'), self.mod_ssl_conf) @@ -184,6 +190,12 @@ class NginxConfigurator(common.Plugin): ", ".join(str(addr) for addr in vhost.addrs))) self.save_notes += "\tssl_certificate %s\n" % fullchain_path self.save_notes += "\tssl_certificate_key %s\n" % key_path + if len(stapling_directives) > 0: + self.save_notes += "\tssl_trusted_certificate %s\n" % chain_path + self.save_notes += "\tssl_stapling on\n" + self.save_notes += "\tssl_stapling_verify on\n" + + ####################### # Vhost parsing methods @@ -233,6 +245,7 @@ class NginxConfigurator(common.Plugin): def _get_ranked_matches(self, target_name): """Returns a ranked list of vhosts that match target_name. + The ranking gives preference to SSL vhosts. :param str target_name: The name to match :returns: list of dicts containing the vhost, the matching name, and @@ -303,10 +316,10 @@ class NginxConfigurator(common.Plugin): key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, le_key.pem) cert = acme_crypto_util.gen_ss_cert(key, domains=[socket.gethostname()]) - cert_path = os.path.join(tmp_dir, "cert.pem") cert_pem = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, cert) - with open(cert_path, 'w') as cert_file: + cert_file, cert_path = le_util.unique_file(os.path.join(tmp_dir, "cert.pem")) + with cert_file: cert_file.write(cert_pem) return cert_path, le_key.file @@ -409,26 +422,13 @@ class NginxConfigurator(common.Plugin): def config_test(self): # pylint: disable=no-self-use """Check the configuration of Nginx for errors. - :returns: Success - :rtype: bool + :raises .errors.MisconfigurationError: If config_test fails """ try: - proc = subprocess.Popen( - [self.conf('ctl'), "-c", self.nginx_conf, "-t"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - except (OSError, ValueError): - logger.fatal("Unable to run nginx config test") - sys.exit(1) - - if proc.returncode != 0: - # Enter recovery routine... - logger.error("Config test failed\n%s\n%s", stdout, stderr) - return False - - return True + le_util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"]) + except errors.SubprocessError as err: + raise errors.MisconfigurationError(str(err)) def _verify_setup(self): """Verify the setup to ensure safe operating environment. @@ -518,21 +518,33 @@ class NginxConfigurator(common.Plugin): :param bool temporary: Indicates whether the changes made will be quickly reversed in the future (ie. challenges) + :raises .errors.PluginError: If there was an error in + an attempt to save the configuration, or an error creating a + checkpoint + """ save_files = set(self.parser.parsed.keys()) - # Create Checkpoint - if temporary: - self.reverter.add_to_temp_checkpoint( - save_files, self.save_notes) - else: - self.reverter.add_to_checkpoint(save_files, + try: + # Create Checkpoint + if temporary: + self.reverter.add_to_temp_checkpoint( + save_files, self.save_notes) + else: + self.reverter.add_to_checkpoint(save_files, self.save_notes) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) + + self.save_notes = "" # Change 'ext' to something else to not override existing conf files self.parser.filedump(ext='') if title and not temporary: - self.reverter.finalize_checkpoint(title) + try: + self.reverter.finalize_checkpoint(title) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) return True @@ -541,13 +553,25 @@ class NginxConfigurator(common.Plugin): Reverts all modified files that have not been saved as a checkpoint + :raises .errors.PluginError: If unable to recover the configuration + """ - self.reverter.recovery_routine() + try: + self.reverter.recovery_routine() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.parser.load() def revert_challenge_config(self): - """Used to cleanup challenge configurations.""" - self.reverter.revert_temporary_config() + """Used to cleanup challenge configurations. + + :raises .errors.PluginError: If unable to revert the challenge config. + + """ + try: + self.reverter.revert_temporary_config() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.parser.load() def rollback_checkpoints(self, rollback=1): @@ -555,13 +579,27 @@ class NginxConfigurator(common.Plugin): :param int rollback: Number of checkpoints to revert + :raises .errors.PluginError: If there is a problem with the input or + the function is unable to correctly revert the configuration + """ - self.reverter.rollback_checkpoints(rollback) + try: + self.reverter.rollback_checkpoints(rollback) + except errors.ReverterError as err: + raise errors.PluginError(str(err)) self.parser.load() def view_config_changes(self): - """Show all of the configuration changes that have taken place.""" - self.reverter.view_config_changes() + """Show all of the configuration changes that have taken place. + + :raises .errors.PluginError: If there is a problem while processing + the checkpoints directories. + + """ + try: + self.reverter.view_config_changes() + except errors.ReverterError as err: + raise errors.PluginError(str(err)) ########################################################################### # Challenges Section for IAuthenticator diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py index f9af5183a..4d15d6a75 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/configurator_test.py @@ -54,6 +54,7 @@ class NginxConfiguratorTest(util.NginxTest): mock_exe_exists.return_value = True self.config.version = None + self.config.config_test = mock.Mock() self.config.prepare() self.assertEquals((1, 6, 2), self.config.version) @@ -361,12 +362,39 @@ class NginxConfiguratorTest(util.NginxTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("letsencrypt_nginx.configurator.subprocess.Popen") - def test_config_test(self, mock_popen): - mocked = mock_popen() - mocked.communicate.return_value = ('', '') - mocked.returncode = 0 - self.assertTrue(self.config.config_test()) + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test(self, _): + self.config.config_test() + + @mock.patch("letsencrypt.le_util.run_script") + def test_config_test_bad_process(self, mock_run_script): + mock_run_script.side_effect = errors.SubprocessError + self.assertRaises(errors.MisconfigurationError, self.config.config_test) + + @mock.patch("letsencrypt.reverter.Reverter.recovery_routine") + def test_recovery_routine_throws_error_from_reverter(self, mock_recovery_routine): + mock_recovery_routine.side_effect = errors.ReverterError("foo") + self.assertRaises(errors.PluginError, self.config.recovery_routine) + + @mock.patch("letsencrypt.reverter.Reverter.view_config_changes") + def test_view_config_changes_throws_error_from_reverter(self, mock_view_config_changes): + mock_view_config_changes.side_effect = errors.ReverterError("foo") + self.assertRaises(errors.PluginError, self.config.view_config_changes) + + @mock.patch("letsencrypt.reverter.Reverter.rollback_checkpoints") + def test_rollback_checkpoints_throws_error_from_reverter(self, mock_rollback_checkpoints): + mock_rollback_checkpoints.side_effect = errors.ReverterError("foo") + self.assertRaises(errors.PluginError, self.config.rollback_checkpoints) + + @mock.patch("letsencrypt.reverter.Reverter.revert_temporary_config") + def test_revert_challenge_config_throws_error_from_reverter(self, mock_revert_temporary_config): + mock_revert_temporary_config.side_effect = errors.ReverterError("foo") + self.assertRaises(errors.PluginError, self.config.revert_challenge_config) + + @mock.patch("letsencrypt.reverter.Reverter.add_to_checkpoint") + def test_save_throws_error_from_reverter(self, mock_add_to_checkpoint): + mock_add_to_checkpoint.side_effect = errors.ReverterError("foo") + self.assertRaises(errors.PluginError, self.config.save) def test_get_snakeoil_paths(self): # pylint: disable=protected-access diff --git a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py index 3d70f7ac7..7a16e3738 100644 --- a/letsencrypt-nginx/letsencrypt_nginx/tests/util.py +++ b/letsencrypt-nginx/letsencrypt_nginx/tests/util.py @@ -49,25 +49,26 @@ def get_nginx_configurator( backups = os.path.join(work_dir, "backups") - with mock.patch("letsencrypt_nginx.configurator.le_util." - "exe_exists") as mock_exe_exists: - mock_exe_exists.return_value = True - - config = configurator.NginxConfigurator( - config=mock.MagicMock( - nginx_server_root=config_path, - le_vhost_ext="-le-ssl.conf", - config_dir=config_dir, - work_dir=work_dir, - backup_dir=backups, - temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), - in_progress_dir=os.path.join(backups, "IN_PROGRESS"), - server="https://acme-server.org:443/new", - tls_sni_01_port=5001, - ), - name="nginx", - version=version) - config.prepare() + with mock.patch("letsencrypt_nginx.configurator.NginxConfigurator." + "config_test"): + with mock.patch("letsencrypt_nginx.configurator.le_util." + "exe_exists") as mock_exe_exists: + mock_exe_exists.return_value = True + config = configurator.NginxConfigurator( + config=mock.MagicMock( + nginx_server_root=config_path, + le_vhost_ext="-le-ssl.conf", + config_dir=config_dir, + work_dir=work_dir, + backup_dir=backups, + temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"), + in_progress_dir=os.path.join(backups, "IN_PROGRESS"), + server="https://acme-server.org:443/new", + tls_sni_01_port=5001, + ), + name="nginx", + version=version) + config.prepare() # Provide general config utility. nsconfig = configuration.NamespaceConfig(config.config) diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index bfb3c3758..656d6e04f 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -63,4 +63,5 @@ setup( 'nginx = letsencrypt_nginx.configurator:NginxConfigurator', ], }, + test_suite='letsencrypt_nginx', ) diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index 1c7815f78..0dbeb1567 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.2.0.dev0' +__version__ = '0.5.0.dev0' diff --git a/letsencrypt/auth_handler.py b/letsencrypt/auth_handler.py index c63d8c8d4..ffbd70ced 100644 --- a/letsencrypt/auth_handler.py +++ b/letsencrypt/auth_handler.py @@ -57,7 +57,7 @@ class AuthHandler(object): def get_authorizations(self, domains, best_effort=False): """Retrieve all authorizations for challenges. - :param set domains: Domains for authorization + :param list domains: Domains for authorization :param bool best_effort: Whether or not all authorizations are required (this is useful in renewal) @@ -480,6 +480,9 @@ def is_preferred(offered_challb, satisfied, return True +_ACME_PREFIX = "urn:acme:error:" + + _ERROR_HELP_COMMON = ( "To fix these errors, please make sure that your domain name was entered " "correctly and the DNS A record(s) for that domain contain(s) the " @@ -490,7 +493,9 @@ _ERROR_HELP = { "connection": _ERROR_HELP_COMMON + " Additionally, please check that your computer " "has a publicly routable IP address and that no firewalls are preventing " - "the server from communicating with the client.", + "the server from communicating with the client. If you're using the " + "webroot plugin, you should also verify that you are serving files " + "from the webroot path you provided.", "dnssec": _ERROR_HELP_COMMON + " Additionally, if you have DNSSEC enabled for " "your domain, please ensure that the signature is valid.", @@ -540,11 +545,13 @@ def _generate_failed_chall_msg(failed_achalls): """ typ = failed_achalls[0].error.typ + if typ.startswith(_ACME_PREFIX): + typ = typ[len(_ACME_PREFIX):] msg = ["The following errors were reported by the server:"] for achall in failed_achalls: msg.append("\n\nDomain: %s\nType: %s\nDetail: %s" % ( - achall.domain, achall.error.typ, achall.error.detail)) + achall.domain, typ, achall.error.detail)) if typ in _ERROR_HELP: msg.append("\n\n") diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 89606089f..855c7a467 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -6,7 +6,9 @@ from __future__ import print_function # (TODO: split this file into main.py and cli.py) import argparse import atexit +import copy import functools +import glob import json import logging import logging.handlers @@ -44,6 +46,27 @@ from letsencrypt.plugins import disco as plugins_disco logger = logging.getLogger(__name__) +# Global, to save us from a lot of argument passing within the scope of this module +_parser = None + +# These are the items which get pulled out of a renewal configuration +# file's renewalparams and actually used in the client configuration +# during the renewal process. We have to record their types here because +# the renewal configuration process loses this information. +STR_CONFIG_ITEMS = ["config_dir", "logs_dir", "work_dir", "user_agent", + "server", "account", "authenticator", "installer", + "standalone_supported_challenges"] +INT_CONFIG_ITEMS = ["rsa_key_size", "tls_sni_01_port", "http01_port"] + +# For help strings, figure out how the user ran us. +# When invoked from letsencrypt-auto, sys.argv[0] is something like: +# "/home/user/.local/share/letsencrypt/bin/letsencrypt" +# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before running +# letsencrypt-auto (and sudo stops us from seeing if they did), so it should only be used +# for purposes where inability to detect letsencrypt-auto fails safely + +fragment = os.path.join(".local", "share", "letsencrypt") +cli_command = "letsencrypt-auto" if fragment in sys.argv[0] else "letsencrypt" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want # to replace as much of it as we can... @@ -51,7 +74,7 @@ logger = logging.getLogger(__name__) # This is the stub to include in help generated by argparse SHORT_USAGE = """ - letsencrypt [SUBCOMMAND] [options] [-d domain] [-d domain] ... + {0} [SUBCOMMAND] [options] [-d domain] [-d domain] ... The Let's Encrypt agent can obtain and install HTTPS/TLS/SSL certificates. By default, it will attempt to use a webserver both for obtaining and installing @@ -60,12 +83,13 @@ the cert. Major SUBCOMMANDS are: (default) run Obtain & install a cert in your current webserver certonly Obtain cert, but do not install it (aka "auth") install Install a previously obtained cert in a server + renew Renew previously obtained certs that are near expiry revoke Revoke a previously obtained certificate rollback Rollback server configuration changes made during install config_changes Show changes made to server config during installation plugins Display information about installed plugins -""" +""".format(cli_command) # This is the short help for letsencrypt --help, where we disable argparse # altogether @@ -103,11 +127,15 @@ def usage_strings(plugins): return USAGE % (apache_doc, nginx_doc), SHORT_USAGE -def _find_domains(args, installer): - if not args.domains: +def _find_domains(config, installer): + if not config.domains: domains = display_ops.choose_names(installer) + # record in config.domains (so that it can be serialised in renewal config files), + # and set webroot_map entries if applicable + for d in domains: + _process_domain(config, d) else: - domains = args.domains + domains = config.domains if not domains: raise errors.Error("Please specify --domains, or --installer that " @@ -116,14 +144,14 @@ def _find_domains(args, installer): return domains -def _determine_account(args, config): +def _determine_account(config): """Determine which account to use. In order to make the renewer (configuration de/serialization) happy, - if ``args.account`` is ``None``, it will be updated based on the - user input. Same for ``args.email``. + if ``config.account`` is ``None``, it will be updated based on the + user input. Same for ``config.email``. - :param argparse.Namespace args: CLI arguments + :param argparse.Namespace config: CLI arguments :param letsencrypt.interface.IConfig config: Configuration object :param .AccountStorage account_storage: Account storage. @@ -136,8 +164,8 @@ def _determine_account(args, config): account_storage = account.AccountFileStorage(config) acme = None - if args.account is not None: - acc = account_storage.load(args.account) + if config.account is not None: + acc = account_storage.load(config.account) else: accounts = account_storage.find_all() if len(accounts) > 1: @@ -145,35 +173,37 @@ def _determine_account(args, config): elif len(accounts) == 1: acc = accounts[0] else: # no account registered yet - if args.email is None and not args.register_unsafely_without_email: - args.email = display_ops.get_email() + if config.email is None and not config.register_unsafely_without_email: + config.namespace.email = display_ops.get_email() def _tos_cb(regr): - if args.tos: + if config.tos: return True msg = ("Please read the Terms of Service at {0}. You " "must agree in order to register with the ACME " "server at {1}".format( regr.terms_of_service, config.server)) - return zope.component.getUtility(interfaces.IDisplay).yesno( - msg, "Agree", "Cancel") + obj = zope.component.getUtility(interfaces.IDisplay) + return obj.yesno(msg, "Agree", "Cancel", cli_flag="--agree-tos") try: acc, acme = client.register( config, account_storage, tos_cb=_tos_cb) + except errors.MissingCommandlineFlag: + raise except errors.Error as error: logger.debug(error, exc_info=True) raise errors.Error( "Unable to register an account with ACME server") - args.account = acc.id + config.namespace.account = acc.id return acc, acme -def _init_le_client(args, config, authenticator, installer): +def _init_le_client(config, authenticator, installer): if authenticator is not None: # if authenticator was given, then we will need account... - acc, acme = _determine_account(args, config) + acc, acme = _determine_account(config) logger.debug("Picked account: %r", acc) # XXX #crypto_util.validate_key_csr(acc.key) @@ -193,13 +223,12 @@ def _find_duplicative_certs(config, domains): # Verify the directory is there le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - for renewal_file in os.listdir(configs_dir): + for renewal_file in _renewal_conf_files(cli_config): try: - full_path = os.path.join(configs_dir, renewal_file) - candidate_lineage = storage.RenewableCert(full_path, cli_config) + candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): - logger.warning("Renewal configuration file %s is broken. " - "Skipping.", full_path) + logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file) + logger.debug("Traceback was:\n%s", traceback.format_exc()) continue # TODO: Handle these differently depending on whether they are # expired or still valid? @@ -218,7 +247,10 @@ def _find_duplicative_certs(config, domains): def _treat_as_renewal(config, domains): - """Determine whether there are duplicated names and how to handle them. + """Determine whether there are duplicated names and how to handle + them (renew, reinstall, newcert, or raising an error to stop + the client run if the user chooses to cancel the operation when + prompted). :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -245,8 +277,24 @@ def _treat_as_renewal(config, domains): elif subset_names_cert is not None: return _handle_subset_cert_request(config, domains, subset_names_cert) + +def _should_renew(config, lineage): + "Return true if any of the circumstances for automatic renewal apply." + if config.renew_by_default: + logger.info("Auto-renewal forced with --force-renewal...") + return True + if lineage.should_autorenew(interactive=True): + logger.info("Cert is due for renewal, auto-renewing...") + return True + if config.dry_run: + logger.info("Cert not due for renewal, but simulating renewal for dry run") + return True + logger.info("Cert not yet due for renewal") + return False + + def _handle_identical_cert_request(config, cert): - """Figure out what to do if a cert has the same names as a perviously obtained one + """Figure out what to do if a cert has the same names as a previously obtained one :param storage.RenewableCert cert: @@ -254,17 +302,12 @@ def _handle_identical_cert_request(config, cert): :rtype: tuple """ - if config.renew_by_default: - logger.info("Auto-renewal forced with --renew-by-default...") - return "renew", cert - if cert.should_autorenew(interactive=True): - logger.info("Cert is due for renewal, auto-renewing...") + if _should_renew(config, cert): return "renew", cert if config.reinstall: # Set with --reinstall, force an identical certificate to be # reinstalled without further prompting. return "reinstall", cert - question = ( "You have an existing certificate that contains exactly the same " "domains you requested and isn't close to expiry." @@ -276,12 +319,11 @@ def _handle_identical_cert_request(config, cert): elif config.verb == "certonly": keep_opt = "Keep the existing certificate for now" choices = [keep_opt, - "Renew & replace the cert (limit ~5 per 7 days)", - "Cancel this operation and do nothing"] + "Renew & replace the cert (limit ~5 per 7 days)"] display = zope.component.getUtility(interfaces.IDisplay) - response = display.menu(question, choices, "OK", "Cancel") - if response[0] == "cancel" or response[1] == 2: + response = display.menu(question, choices, "OK", "Cancel", default=0) + if response[0] == display_util.CANCEL: # TODO: Add notification related to command-line options for # skipping the menu for this case. raise errors.Error( @@ -294,6 +336,7 @@ def _handle_identical_cert_request(config, cert): else: assert False, "This is impossible" + def _handle_subset_cert_request(config, domains, cert): """Figure out what to do if a previous cert had a subset of the names now requested @@ -315,7 +358,8 @@ def _handle_subset_cert_request(config, domains, cert): ", ".join(domains), br=os.linesep) if config.expand or config.renew_by_default or zope.component.getUtility( - interfaces.IDisplay).yesno(question, "Expand", "Cancel"): + interfaces.IDisplay).yesno(question, "Expand", "Cancel", + cli_flag="--expand (or in some cases, --duplicate)"): return "renew", cert else: reporter_util = zope.component.getUtility(interfaces.IReporter) @@ -360,8 +404,14 @@ def _report_new_cert(cert_path, fullchain_path): .format(and_chain, path, expiry)) reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY) -def _suggest_donate(): - "Suggest a donation to support Let's Encrypt" + +def _suggest_donation_if_appropriate(config, action): + """Potentially suggest a donation to support Let's Encrypt.""" + if config.staging or config.verb == "renew": + # --dry-run implies --staging + return + if action not in ["renew", "newcert"]: + return reporter_util = zope.component.getUtility(interfaces.IReporter) msg = ("If you like Let's Encrypt, please consider supporting our work by:\n\n" "Donating to ISRG / Let's Encrypt: https://letsencrypt.org/donate\n" @@ -369,7 +419,14 @@ def _suggest_donate(): reporter_util.add_message(msg, reporter_util.LOW_PRIORITY) -def _auth_from_domains(le_client, config, domains): +def _report_successful_dry_run(config): + reporter_util = zope.component.getUtility(interfaces.IReporter) + if config.verb != "renew": + reporter_util.add_message("The dry run was successful.", + reporter_util.HIGH_PRIORITY, on_crash=False) + + +def _auth_from_domains(le_client, config, domains, lineage=None): """Authenticate and enroll certificate.""" # Note: This can raise errors... caught above us though. This is now # a three-way case: reinstall (which results in a no-op here because @@ -378,11 +435,20 @@ def _auth_from_domains(le_client, config, domains): # (which results in treating the request as a renewal), or newcert # (which results in treating the request as a new certificate request). - action, lineage = _treat_as_renewal(config, domains) + # If lineage is specified, use that one instead of looking around for + # a matching one. + if lineage is None: + # This will find a relevant matching lineage that exists + action, lineage = _treat_as_renewal(config, domains) + else: + # Renewal, where we already know the specific lineage we're + # interested in + action = "renew" + if action == "reinstall": # The lineage already exists; allow the caller to try installing # it without getting a new certificate at all. - return lineage + return lineage, "reinstall" elif action == "renew": original_server = lineage.configuration["renewalparams"]["server"] _avoid_invalidating_lineage(config, lineage, original_server) @@ -390,24 +456,30 @@ def _auth_from_domains(le_client, config, domains): # https://github.com/letsencrypt/letsencrypt/pull/777/files#r40498574 new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) # TODO: Check whether it worked! <- or make sure errors are thrown (jdk) - lineage.save_successor( - lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), - new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) - - lineage.update_all_links_to(lineage.latest_common_version()) + if config.dry_run: + logger.info("Dry run: skipping updating lineage at %s", + os.path.dirname(lineage.cert)) + else: + lineage.save_successor( + lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), + new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain), + configuration.RenewerConfiguration(config.namespace)) + lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor # TODO: Also update lineage renewal config with any relevant # configuration values from this attempt? <- Absolutely (jdkasten) elif action == "newcert": # TREAT AS NEW REQUEST lineage = le_client.obtain_and_enroll_certificate(domains) - if not lineage: + if lineage is False: raise errors.Error("Certificate could not be obtained") - _report_new_cert(lineage.cert, lineage.fullchain) + if not config.dry_run and not config.verb == "renew": + _report_new_cert(lineage.cert, lineage.fullchain) + + return lineage, action - return lineage def _avoid_invalidating_lineage(config, lineage, original_server): "Do not renew a valid cert with one from a staging server!" @@ -420,7 +492,7 @@ def _avoid_invalidating_lineage(config, lineage, original_server): open(lineage.cert).read()) # all our test certs are from happy hacker fake CA, though maybe one day # we should test more methodically - now_valid = not "fake" in repr(latest_cert.get_issuer()).lower() + now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if _is_staging(config.server): if not _is_staging(original_server) or now_valid: @@ -431,21 +503,6 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "a test certificate (domains: {0}). We will not do that " "unless you use the --break-my-certs flag!".format(names)) -def set_configurator(previously, now): - """ - Setting configurators multiple ways is okay, as long as they all agree - :param str previously: previously identified request for the installer/authenticator - :param str requested: the request currently being processed - """ - if now is None: - # we're not actually setting anything - return previously - if previously: - if previously != now: - msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" - raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) - return now - def diagnose_configurator_problem(cfg_type, requested, plugins): """ @@ -479,39 +536,85 @@ def diagnose_configurator_problem(cfg_type, requested, plugins): raise errors.PluginSelectionError(msg) -def choose_configurator_plugins(args, config, plugins, verb): # pylint: disable=too-many-branches +def set_configurator(previously, now): """ - Figure out which configurator we're going to use - :raises error.PluginSelectionError if there was a problem + Setting configurators multiple ways is okay, as long as they all agree + :param str previously: previously identified request for the installer/authenticator + :param str requested: the request currently being processed + """ + if now is None: + # we're not actually setting anything + return previously + if previously: + if previously != now: + msg = "Too many flags setting configurators/installers/authenticators {0} -> {1}" + raise errors.PluginSelectionError(msg.format(repr(previously), repr(now))) + return now + + +def cli_plugin_requests(config): + """ + Figure out which plugins the user requested with CLI and config options + + :returns: (requested authenticator string or None, requested installer string or None) + :rtype: tuple + """ + req_inst = req_auth = config.configurator + req_inst = set_configurator(req_inst, config.installer) + req_auth = set_configurator(req_auth, config.authenticator) + if config.nginx: + req_inst = set_configurator(req_inst, "nginx") + req_auth = set_configurator(req_auth, "nginx") + if config.apache: + req_inst = set_configurator(req_inst, "apache") + req_auth = set_configurator(req_auth, "apache") + if config.standalone: + req_auth = set_configurator(req_auth, "standalone") + if config.webroot: + req_auth = set_configurator(req_auth, "webroot") + if config.manual: + req_auth = set_configurator(req_auth, "manual") + logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) + return req_auth, req_inst + + +noninstaller_plugins = ["webroot", "manual", "standalone"] + + +def choose_configurator_plugins(config, plugins, verb): + """ + Figure out which configurator we're going to use, modifies + config.authenticator and config.istaller strings to reflect that choice if + necessary. + + :raises errors.PluginSelectionError if there was a problem + + :returns: (an `IAuthenticator` or None, an `IInstaller` or None) + :rtype: tuple """ + req_auth, req_inst = cli_plugin_requests(config) + # Which plugins do we need? - need_inst = need_auth = (verb == "run") + if verb == "run": + need_inst = need_auth = True + if req_auth in noninstaller_plugins and not req_inst: + msg = ('With the {0} plugin, you probably want to use the "certonly" command, eg:{1}' + '{1} {2} certonly --{0}{1}{1}' + '(Alternatively, add a --installer flag. See https://eff.org/letsencrypt-plugins' + '{1} and "--help plugins" for more information.)'.format( + req_auth, os.linesep, cli_command)) + + raise errors.MissingCommandlineFlag(msg) + else: + need_inst = need_auth = False if verb == "certonly": need_auth = True if verb == "install": need_inst = True - if args.authenticator: + if config.authenticator: logger.warn("Specifying an authenticator doesn't make sense in install mode") - # Which plugins did the user request? - req_inst = req_auth = args.configurator - req_inst = set_configurator(req_inst, args.installer) - req_auth = set_configurator(req_auth, args.authenticator) - if args.nginx: - req_inst = set_configurator(req_inst, "nginx") - req_auth = set_configurator(req_auth, "nginx") - if args.apache: - req_inst = set_configurator(req_inst, "apache") - req_auth = set_configurator(req_auth, "apache") - if args.standalone: - req_auth = set_configurator(req_auth, "standalone") - if args.webroot: - req_auth = set_configurator(req_auth, "webroot") - if args.manual: - req_auth = set_configurator(req_auth, "manual") - logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) - # Try to meet the user's request and/or ask them to pick plugins authenticator = installer = None if verb == "run" and req_auth == req_inst: @@ -543,20 +646,20 @@ def record_chosen_plugins(config, plugins, auth, inst): # TODO: Make run as close to auth + install as possible -# Possible difficulties: args.csr was hacked into auth -def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-locals +# Possible difficulties: config.csr was hacked into auth +def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals """Obtain a certificate and install.""" try: - installer, authenticator = choose_configurator_plugins(args, config, plugins, "run") - except errors.PluginSelectionError, e: + installer, authenticator = choose_configurator_plugins(config, plugins, "run") + except errors.PluginSelectionError as e: return e.message - domains = _find_domains(args, installer) + domains = _find_domains(config, installer) # TODO: Handle errors from _init_le_client? - le_client = _init_le_client(args, config, authenticator, installer) + le_client = _init_le_client(config, authenticator, installer) - lineage = _auth_from_domains(le_client, config, domains) + lineage, action = _auth_from_domains(le_client, config, domains) le_client.deploy_certificate( domains, lineage.privkey, lineage.cert, @@ -567,84 +670,390 @@ def run(args, config, plugins): # pylint: disable=too-many-branches,too-many-lo if len(lineage.available_versions("cert")) == 1: display_ops.success_installation(domains) else: - display_ops.success_renewal(domains) + display_ops.success_renewal(domains, action) - _suggest_donate() + _suggest_donation_if_appropriate(config, action) -def obtain_cert(args, config, plugins): - """Authenticate & obtain cert, but do not install it.""" - if args.domains and args.csr is not None: - # TODO: --csr could have a priority, when --domains is - # supplied, check if CSR matches given domains? - return "--domains and --csr are mutually exclusive" - +def obtain_cert(config, plugins, lineage=None): + """Implements "certonly": authenticate & obtain cert, but do not install it.""" + # pylint: disable=too-many-locals try: # installers are used in auth mode to determine domain names - installer, authenticator = choose_configurator_plugins(args, config, plugins, "certonly") - except errors.PluginSelectionError, e: - return e.message + installer, authenticator = choose_configurator_plugins(config, plugins, "certonly") + except errors.PluginSelectionError as e: + logger.info("Could not choose appropriate plugin: %s", e) + raise # TODO: Handle errors from _init_le_client? - le_client = _init_le_client(args, config, authenticator, installer) + le_client = _init_le_client(config, authenticator, installer) + action = "newcert" # This is a special case; cert and chain are simply saved - if args.csr is not None: - certr, chain = le_client.obtain_certificate_from_csr(le_util.CSR( - file=args.csr[0], data=args.csr[1], form="der")) - cert_path, _, cert_fullchain = le_client.save_certificate( - certr, chain, args.cert_path, args.chain_path, args.fullchain_path) - _report_new_cert(cert_path, cert_fullchain) + if config.csr is not None: + assert lineage is None, "Did not expect a CSR with a RenewableCert" + csr, typ = config.actual_csr + certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) + if config.dry_run: + logger.info( + "Dry run: skipping saving certificate to %s", config.cert_path) + else: + cert_path, _, cert_fullchain = le_client.save_certificate( + certr, chain, config.cert_path, config.chain_path, config.fullchain_path) + _report_new_cert(cert_path, cert_fullchain) else: - domains = _find_domains(args, installer) - _auth_from_domains(le_client, config, domains) + domains = _find_domains(config, installer) + _, action = _auth_from_domains(le_client, config, domains, lineage) - _suggest_donate() + if config.dry_run: + _report_successful_dry_run(config) + elif config.verb == "renew": + if installer is None: + # Tell the user that the server was not restarted. + print("new certificate deployed without reload, fullchain is", + lineage.fullchain) + else: + # In case of a renewal, reload server to pick up new certificate. + # In principle we could have a configuration option to inhibit this + # from happening. + installer.restart() + print("new certificate deployed with reload of", + config.installer, "server; fullchain is", lineage.fullchain) + _suggest_donation_if_appropriate(config, action) -def install(args, config, plugins): +def install(config, plugins): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert + # FIXME: be consistent about whether errors are raised or returned from + # this function ... try: - installer, _ = choose_configurator_plugins(args, config, - plugins, "install") - except errors.PluginSelectionError, e: + installer, _ = choose_configurator_plugins(config, plugins, "install") + except errors.PluginSelectionError as e: return e.message - domains = _find_domains(args, installer) - le_client = _init_le_client( - args, config, authenticator=None, installer=installer) - assert args.cert_path is not None # required=True in the subparser + domains = _find_domains(config, installer) + le_client = _init_le_client(config, authenticator=None, installer=installer) + assert config.cert_path is not None # required=True in the subparser le_client.deploy_certificate( - domains, args.key_path, args.cert_path, args.chain_path, - args.fullchain_path) + domains, config.key_path, config.cert_path, config.chain_path, + config.fullchain_path) le_client.enhance_config(domains, config) -def revoke(args, config, unused_plugins): # TODO: coop with renewal config +def _set_by_cli(var): + """ + Return True if a particular config variable has been set by the user + (CLI or config file) including if the user explicitly set it to the + default. Returns False if the variable was assigned a default value. + """ + detector = _set_by_cli.detector + if detector is None: + # Setup on first run: `detector` is a weird version of config in which + # the default value of every attribute is wrangled to be boolean-false + plugins = plugins_disco.PluginsRegistry.find_all() + # reconstructed_args == sys.argv[1:], or whatever was passed to main() + reconstructed_args = _parser.args + [_parser.verb] + detector = _set_by_cli.detector = prepare_and_parse_args( + plugins, reconstructed_args, detect_defaults=True) + # propagate plugin requests: eg --standalone modifies config.authenticator + auth, inst = cli_plugin_requests(detector) + detector.authenticator = auth if auth else "" + detector.installer = inst if inst else "" + logger.debug("Default Detector is %r", detector) + + try: + # Is detector.var something that isn't false? + change_detected = getattr(detector, var) + except AttributeError: + logger.warning("Missing default analysis for %r", var) + return False + + if change_detected: + return True + # Special case: we actually want account to be set to "" if the server + # the account was on has changed + elif var == "account" and (detector.server or detector.dry_run or detector.staging): + return True + # Special case: vars like --no-redirect that get set True -> False + # default to None; False means they were set + elif var in detector.store_false_vars and change_detected is not None: + return True + else: + return False +# static housekeeping var +_set_by_cli.detector = None + +def _restore_required_config_elements(config, renewalparams): + """Sets non-plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: parameters from the renewal + configuration file that defines this lineage + + """ + # string-valued items to add if they're present + for config_item in STR_CONFIG_ITEMS: + if config_item in renewalparams and not _set_by_cli(config_item): + value = renewalparams[config_item] + # Unfortunately, we've lost type information from ConfigObj, + # so we don't know if the original was NoneType or str! + if value == "None": + value = None + setattr(config.namespace, config_item, value) + # int-valued items to add if they're present + for config_item in INT_CONFIG_ITEMS: + if config_item in renewalparams and not _set_by_cli(config_item): + try: + value = int(renewalparams[config_item]) + setattr(config.namespace, config_item, value) + except ValueError: + raise errors.Error( + "Expected a numeric value for {0}".format(config_item)) + + +def _restore_plugin_configs(config, renewalparams): + """Sets plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ + # Now use parser to get plugin-prefixed items with correct types + # XXX: the current approach of extracting only prefixed items + # related to the actually-used installer and authenticator + # works as long as plugins don't need to read plugin-specific + # variables set by someone else (e.g., assuming Apache + # configurator doesn't need to read webroot_ variables). + # Note: if a parameter that used to be defined in the parser is no + # longer defined, stored copies of that parameter will be + # deserialized as strings by this logic even if they were + # originally meant to be some other type. + if renewalparams["authenticator"] == "webroot": + _restore_webroot_config(config, renewalparams) + plugin_prefixes = [] + else: + plugin_prefixes = [renewalparams["authenticator"]] + + if renewalparams.get("installer", None) is not None: + plugin_prefixes.append(renewalparams["installer"]) + for plugin_prefix in set(plugin_prefixes): + for config_item, config_value in renewalparams.iteritems(): + if config_item.startswith(plugin_prefix + "_") and not _set_by_cli(config_item): + # Values None, True, and False need to be treated specially, + # As they don't get parsed correctly based on type + if config_value in ("None", "True", "False"): + # bool("False") == True + # pylint: disable=eval-used + setattr(config.namespace, config_item, eval(config_value)) + continue + for action in _parser.parser._actions: # pylint: disable=protected-access + if action.type is not None and action.dest == config_item: + setattr(config.namespace, config_item, + action.type(config_value)) + break + else: + setattr(config.namespace, config_item, str(config_value)) + +def _restore_webroot_config(config, renewalparams): + """ + webroot_map is, uniquely, a dict, and the general-purpose configuration + restoring logic is not able to correctly parse it from the serialized + form. + """ + if "webroot_map" in renewalparams: + # if the user does anything that would create a new webroot map on the + # CLI, don't use the old one + if not (_set_by_cli("webroot_map") or _set_by_cli("webroot_path")): + setattr(config.namespace, "webroot_map", renewalparams["webroot_map"]) + elif "webroot_path" in renewalparams: + logger.info("Ancient renewal conf file without webroot-map, restoring webroot-path") + setattr(config.namespace, "webroot_path", renewalparams["webroot_path"]) + + +def _reconstitute(config, full_path): + """Try to instantiate a RenewableCert, updating config with relevant items. + + This is specifically for use in renewal and enforces several checks + and policies to ensure that we can try to proceed with the renwal + request. The config argument is modified by including relevant options + read from the renewal configuration file. + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param str full_path: Absolute path to the configuration file that + defines this lineage + + :returns: the RenewableCert object or None if a fatal error occurred + :rtype: `storage.RenewableCert` or NoneType + + """ + try: + renewal_candidate = storage.RenewableCert( + full_path, configuration.RenewerConfiguration(config)) + except (errors.CertStorageError, IOError): + logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + return None + if "renewalparams" not in renewal_candidate.configuration: + logger.warning("Renewal configuration file %s lacks " + "renewalparams. Skipping.", full_path) + return None + renewalparams = renewal_candidate.configuration["renewalparams"] + if "authenticator" not in renewalparams: + logger.warning("Renewal configuration file %s does not specify " + "an authenticator. Skipping.", full_path) + return None + # Now restore specific values along with their data types, if + # those elements are present. + try: + _restore_required_config_elements(config, renewalparams) + _restore_plugin_configs(config, renewalparams) + except (ValueError, errors.Error) as error: + logger.warning( + "An error occured while parsing %s. The error was %s. " + "Skipping the file.", full_path, error.message) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + return None + + try: + for d in renewal_candidate.names(): + _process_domain(config, d) + except errors.ConfigurationError as error: + logger.warning("Renewal configuration file %s references a cert " + "that contains an invalid domain name. The problem " + "was: %s. Skipping.", full_path, error) + return None + + return renewal_candidate + +def _renewal_conf_files(config): + """Return /path/to/*.conf in the renewal conf directory""" + return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + + +def _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures): + status = lambda x, msg: " " + "\n ".join(i + " (" + msg +")" for i in x) + if config.dry_run: + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates below have not been saved.)") + print() + if renew_skipped: + print("The following certs are not due for renewal yet:") + print(status(renew_skipped, "skipped")) + if not renew_successes and not renew_failures: + print("No renewals were attempted.") + elif renew_successes and not renew_failures: + print("Congratulations, all renewals succeeded. The following certs " + "have been renewed:") + print(status(renew_successes, "success")) + elif renew_failures and not renew_successes: + print("All renewal attempts failed. The following certs could not be " + "renewed:") + print(status(renew_failures, "failure")) + elif renew_failures and renew_successes: + print("The following certs were successfully renewed:") + print(status(renew_successes, "success")) + print("\nThe following certs could not be renewed:") + print(status(renew_failures, "failure")) + + if parse_failures: + print("\nAdditionally, the following renewal configuration files " + "were invalid: ") + print(status(parse_failures, "parsefail")) + + if config.dry_run: + print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry") + print("** (The test certificates above have not been saved.)") + + +def renew(config, unused_plugins): + """Renew previously-obtained certificates.""" + + if config.domains != []: + raise errors.Error("Currently, the renew verb is only capable of " + "renewing all installed certificates that are due " + "to be renewed; individual domains cannot be " + "specified with this action. If you would like to " + "renew specific certificates, use the certonly " + "command. The renew verb may provide other options " + "for selecting certificates to renew in the future.") + if config.csr is not None: + raise errors.Error("Currently, the renew verb cannot be used when " + "specifying a CSR file. Please try the certonly " + "command instead.") + renewer_config = configuration.RenewerConfiguration(config) + renew_successes = [] + renew_failures = [] + renew_skipped = [] + parse_failures = [] + for renewal_file in _renewal_conf_files(renewer_config): + print("Processing " + renewal_file) + lineage_config = copy.deepcopy(config) + + # Note that this modifies config (to add back the configuration + # elements from within the renewal configuration file). + try: + renewal_candidate = _reconstitute(lineage_config, renewal_file) + except Exception as e: # pylint: disable=broad-except + logger.warning("Renewal configuration file %s produced an " + "unexpected error: %s. Skipping.", renewal_file, e) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + parse_failures.append(renewal_file) + continue + + try: + if renewal_candidate is None: + parse_failures.append(renewal_file) + else: + # XXX: ensure that each call here replaces the previous one + zope.component.provideUtility(lineage_config) + if _should_renew(lineage_config, renewal_candidate): + plugins = plugins_disco.PluginsRegistry.find_all() + obtain_cert(lineage_config, plugins, renewal_candidate) + renew_successes.append(renewal_candidate.fullchain) + else: + renew_skipped.append(renewal_candidate.fullchain) + except Exception as e: # pylint: disable=broad-except + # obtain_cert (presumably) encountered an unanticipated problem. + logger.warning("Attempting to renew cert from %s produced an " + "unexpected error: %s. Skipping.", renewal_file, e) + logger.debug("Traceback was:\n%s", traceback.format_exc()) + renew_failures.append(renewal_candidate.fullchain) + + # Describe all the results + _renew_describe_results(config, renew_successes, renew_failures, + renew_skipped, parse_failures) + + +def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction config.namespace.installer = config.namespace.authenticator = "none" - if args.key_path is not None: # revocation by cert key + if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", - args.cert_path[0], args.key_path[0]) - key = jose.JWK.load(args.key_path[1]) + config.cert_path[0], config.key_path[0]) + key = jose.JWK.load(config.key_path[1]) else: # revocation by account key - logger.debug("Revoking %s using Account Key", args.cert_path[0]) - acc, _ = _determine_account(args, config) + logger.debug("Revoking %s using Account Key", config.cert_path[0]) + acc, _ = _determine_account(config) key = acc.key acme = client.acme_from_config_key(config, key) - cert = crypto_util.pyopenssl_load_certificate(args.cert_path[1])[0] + cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] acme.revoke(jose.ComparableX509(cert)) -def rollback(args, config, plugins): +def rollback(config, plugins): """Rollback server configuration changes made during install.""" - client.rollback(args.installer, args.checkpoints, config, plugins) + client.rollback(config.installer, config.checkpoints, config, plugins) -def config_changes(unused_args, config, unused_plugins): +def config_changes(config, unused_plugins): """Show changes made to server config during installation View checkpoints and associated configuration changes. @@ -653,15 +1062,15 @@ def config_changes(unused_args, config, unused_plugins): client.view_config_changes(config) -def plugins_cmd(args, config, plugins): # TODO: Use IDisplay rather than print +def plugins_cmd(config, plugins): # TODO: Use IDisplay rather than print """List server software plugins.""" - logger.debug("Expected interfaces: %s", args.ifaces) + logger.debug("Expected interfaces: %s", config.ifaces) - ifaces = [] if args.ifaces is None else args.ifaces + ifaces = [] if config.ifaces is None else config.ifaces filtered = plugins.visible().ifaces(ifaces) logger.debug("Filtered plugins: %r", filtered) - if not args.init and not args.prepare: + if not config.init and not config.prepare: print(str(filtered)) return @@ -669,7 +1078,7 @@ def plugins_cmd(args, config, plugins): # TODO: Use IDisplay rather than print verified = filtered.verify(ifaces) logger.debug("Verified plugins: %r", verified) - if not args.prepare: + if not config.prepare: print(str(verified)) return @@ -740,14 +1149,14 @@ class HelpfulArgumentParser(object): # Maps verbs/subcommands to the functions that implement them VERBS = {"auth": obtain_cert, "certonly": obtain_cert, "config_changes": config_changes, "everything": run, - "install": install, "plugins": plugins_cmd, + "install": install, "plugins": plugins_cmd, "renew": renew, "revoke": revoke, "rollback": rollback, "run": run} # List of topics for which additional help can be provided HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + VERBS.keys() - def __init__(self, args, plugins): + def __init__(self, args, plugins, detect_defaults=False): plugin_names = [name for name, _p in plugins.iteritems()] self.help_topics = self.HELP_TOPICS + plugin_names + [None] usage, short_usage = usage_strings(plugins) @@ -761,6 +1170,14 @@ class HelpfulArgumentParser(object): self.parser._add_config_file_help = False # pylint: disable=protected-access self.silent_parser = SilentParser(self.parser) + # This setting attempts to force all default values to things that are + # pythonically false; it is used to detect when values have been + # explicitly set by the user, including when they are set to their + # normal default value + self.detect_defaults = detect_defaults + if detect_defaults: + self.store_false_vars = {} # vars that use "store_false" + self.args = args self.determine_verb() help1 = self.prescan_for_flag("-h", self.help_topics) @@ -772,7 +1189,7 @@ class HelpfulArgumentParser(object): print(usage) sys.exit(0) self.visible_topics = self.determine_help_topics(self.help_arg) - self.groups = {} # elements are added by .add_group() + self.groups = {} # elements are added by .add_group() def parse_args(self): """Parses command line arguments and returns the result. @@ -787,14 +1204,82 @@ class HelpfulArgumentParser(object): # Do any post-parsing homework here - # argparse seemingly isn't flexible enough to give us this behaviour easily... - if parsed_args.staging: + # we get domains from -d, but also from the webroot map... + if parsed_args.webroot_map: + for domain in parsed_args.webroot_map.keys(): + if domain not in parsed_args.domains: + parsed_args.domains.append(domain) + + if parsed_args.staging or parsed_args.dry_run: if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): - raise errors.Error("--server value conflicts with --staging") + conflicts = ["--staging"] if parsed_args.staging else [] + conflicts += ["--dry-run"] if parsed_args.dry_run else [] + if not self.detect_defaults: + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) + parsed_args.server = constants.STAGING_URI + if parsed_args.dry_run: + if self.verb not in ["certonly", "renew"]: + raise errors.Error("--dry-run currently only works with the " + "'certonly' or 'renew' subcommands (%r)" % self.verb) + parsed_args.break_my_certs = parsed_args.staging = True + if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): + # The user has a prod account, but might not have a staging + # one; we don't want to start trying to perform interactive registration + parsed_args.agree_tos = True + parsed_args.register_unsafely_without_email = True + + if parsed_args.csr: + self.handle_csr(parsed_args) + + if self.detect_defaults: # plumbing + parsed_args.store_false_vars = self.store_false_vars + return parsed_args + def handle_csr(self, parsed_args): + """ + Process a --csr flag. This needs to happen early enough that the + webroot plugin can know about the calls to _process_domain + """ + try: + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") + typ = OpenSSL.crypto.FILETYPE_ASN1 + domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) + except OpenSSL.crypto.Error: + try: + e1 = traceback.format_exc() + typ = OpenSSL.crypto.FILETYPE_PEM + csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="pem") + domains = crypto_util.get_sans_from_csr(csr.data, typ) + except OpenSSL.crypto.Error: + logger.debug("DER CSR parse error %s", e1) + logger.debug("PEM CSR parse error %s", traceback.format_exc()) + raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) + for d in domains: + _process_domain(parsed_args, d) + + for d in domains: + sanitised = le_util.enforce_domain_sanity(d) + if d.lower() != sanitised: + raise errors.ConfigurationError( + "CSR domain {0} needs to be sanitised to {1}.".format(d, sanitised)) + + if not domains: + # TODO: add CN to domains instead: + raise errors.Error( + "Unfortunately, your CSR %s needs to have a SubjectAltName for every domain" + % parsed_args.csr[0]) + + parsed_args.actual_csr = (csr, typ) + csr_domains, config_domains = set(domains), set(parsed_args.domains) + if csr_domains != config_domains: + raise errors.ConfigurationError( + "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" + .format(", ".join(csr_domains), ", ".join(config_domains))) + def determine_verb(self): """Determines the verb/subcommand provided by the user. @@ -844,10 +1329,16 @@ class HelpfulArgumentParser(object): def add(self, topic, *args, **kwargs): """Add a new command line argument. - @topic is required, to indicate which part of the help will document - it, but can be None for `always documented'. + :param str: help topic this should be listed under, can be None for + "always documented" + :param list *args: the names of this argument flag + :param dict **kwargs: various argparse settings for this argument """ + + if self.detect_defaults: + kwargs = self.modify_arg_for_default_detection(self, *args, **kwargs) + if self.visible_topics[topic]: if topic in self.groups: group = self.groups[topic] @@ -858,6 +1349,39 @@ class HelpfulArgumentParser(object): kwargs["help"] = argparse.SUPPRESS self.parser.add_argument(*args, **kwargs) + + def modify_arg_for_default_detection(self, *args, **kwargs): + """ + Adding an arg, but ensure that it has a default that evaluates to false, + so that _set_by_cli can tell if it was set. Only called if detect_defaults==True. + + :param list *args: the names of this argument flag + :param dict **kwargs: various argparse settings for this argument + + :returns: a modified versions of kwargs + """ + # argument either doesn't have a default, or the default doesn't + # isn't Pythonically false + if kwargs.get("default", True): + arg_type = kwargs.get("type", None) + if arg_type == int or kwargs.get("action", "") == "count": + kwargs["default"] = 0 + elif arg_type == read_file or "-c" in args: + kwargs["default"] = "" + kwargs["type"] = str + else: + kwargs["default"] = "" + # This doesn't matter at present (none of the store_false args + # are renewal-relevant), but implement it for future sanity: + # detect the setting of args whose presence causes True -> False + if kwargs.get("action", "") == "store_false": + kwargs["default"] = None + for var in args: + self.store_false_vars[var] = True + + return kwargs + + def add_deprecated_argument(self, argument_name, num_args): """Adds a deprecated argument with the name argument_name. @@ -925,7 +1449,7 @@ class HelpfulArgumentParser(object): return dict([(t, t == chosen_topic) for t in self.help_topics]) -def prepare_and_parse_args(plugins, args): +def prepare_and_parse_args(plugins, args, detect_defaults=False): """Returns parsed command line arguments. :param .PluginsRegistry plugins: available plugins @@ -935,7 +1459,7 @@ def prepare_and_parse_args(plugins, args): :rtype: argparse.Namespace """ - helpful = HelpfulArgumentParser(args, plugins) + helpful = HelpfulArgumentParser(args, plugins, detect_defaults) # --help is automatically provided by argparse helpful.add( @@ -946,6 +1470,17 @@ def prepare_and_parse_args(plugins, args): helpful.add( None, "-t", "--text", dest="text_mode", action="store_true", help="Use the text output instead of the curses UI.") + helpful.add( + None, "-n", "--non-interactive", "--noninteractive", + dest="noninteractive_mode", action="store_true", + help="Run without ever asking for user input. This may require " + "additional command line flags; the client will try to explain " + "which ones are required if it finds one missing") + helpful.add( + None, "--dry-run", action="store_true", dest="dry_run", + help="Perform a test run of the client, obtaining test (invalid) certs" + " but not saving them to disk. This can currently only be used" + " with the 'certonly' subcommand.") helpful.add( None, "--register-unsafely-without-email", action="store_true", help="Specifying this flag enables registering an account with no " @@ -984,10 +1519,12 @@ def prepare_and_parse_args(plugins, args): version="%(prog)s {0}".format(letsencrypt.__version__), help="show program's version number and exit") helpful.add( - "automation", "--renew-by-default", action="store_true", - help="Select renewal by default when domains are a superset of a " - "previously attained cert (often --keep-until-expiring is " - "more appropriate). Implies --expand.") + "automation", "--force-renewal", "--renew-by-default", + action="store_true", dest="renew_by_default", help="If a certificate " + "already exists for the requested domains, renew it now, " + "regardless of whether it is near expiry. (Often " + "--keep-until-expiring is more appropriate). Also implies " + "--expand.") helpful.add( "automation", "--agree-tos", dest="tos", action="store_true", help="Agree to the Let's Encrypt Subscriber Agreement") @@ -998,6 +1535,13 @@ def prepare_and_parse_args(plugins, args): "automation", "--duplicate", dest="duplicate", action="store_true", help="Allow making a certificate lineage that duplicates an existing one " "(both can be renewed in parallel)") + helpful.add( + "automation", "--os-packages-only", action="store_true", + help="(letsencrypt-auto only) install OS package dependencies and then stop") + helpful.add( + "automation", "--no-self-upgrade", action="store_true", + help="(letsencrypt-auto only) prevent the letsencrypt-auto script from" + " upgrading itself to newer released versions") helpful.add_group( "testing", description="The following flags are meant for " @@ -1059,6 +1603,16 @@ def prepare_and_parse_args(plugins, args): help="Require that all configuration files are owned by the current " "user; only needed if your config is somewhere unsafe like /tmp/") + helpful.add_group( + "renew", description="The 'renew' subcommand will attempt to renew all" + " certificates (or more precisely, certificate lineages) you have" + " previously obtained if they are close to expiry, and print a" + " summary of the results. By default, 'renew' will reuse the options" + " used to create obtain or most recently successfully renew each" + " certificate lineage. You can try it with `--dry-run` first. For" + " more fine-grained control, you can renew individual lineages with" + " the `certonly` subcommand.") + helpful.add_deprecated_argument("--agree-dev-preview", 0) _create_subparsers(helpful) @@ -1067,6 +1621,9 @@ def prepare_and_parse_args(plugins, args): # parser (--help should display plugin-specific options last) _plugins_parsing(helpful, plugins) + if not detect_defaults: + global _parser # pylint: disable=global-statement + _parser = helpful return helpful.parse_args() @@ -1086,7 +1643,8 @@ def _create_subparsers(helpful): "--csr", type=read_file, help="Path to a Certificate Signing Request (CSR) in DER" " format; note that the .csr file *must* contain a Subject" - " Alternative Name field for each domain you want certified.") + " Alternative Name field for each domain you want certified." + " Currently --csr only works with the 'certonly' subcommand'") helpful.add("rollback", "--checkpoints", type=int, metavar="N", default=flag_default("rollback_checkpoints"), @@ -1159,9 +1717,8 @@ def _plugins_parsing(helpful, plugins): "plugins", description="Let's Encrypt client supports an " "extensible plugins architecture. See '%(prog)s plugins' for a " "list of all installed plugins and their names. You can force " - "a particular plugin by setting options provided below. Further " - "down this help message you will find plugin-specific options " - "(prefixed by --{plugin_name}).") + "a particular plugin by setting options provided below. Running " + "--help will list flags specific to that plugin.") helpful.add( "plugins", "-a", "--authenticator", help="Authenticator plugin name.") helpful.add( @@ -1191,61 +1748,87 @@ def _plugins_parsing(helpful, plugins): # they are parsed in conjunction with --domains, they live here for # legibility. helpful.add_plugin_ags must be called first to add the # "webroot" topic - helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor, + helpful.add("webroot", "-w", "--webroot-path", default=[], action=WebrootPathProcessor, help="public_html / webroot path. This can be specified multiple times to " "handle different domains; each domain will have the webroot path that" " preceded it. For instance: `-w /var/www/example -d example.com -d " "www.example.com -w /var/www/thing -d thing.net -d m.thing.net`") - parse_dict = lambda s: dict(json.loads(s)) # --webroot-map still has some awkward properties, so it is undocumented - helpful.add("webroot", "--webroot-map", default={}, type=parse_dict, - help=argparse.SUPPRESS) + helpful.add("webroot", "--webroot-map", default={}, action=WebrootMapProcessor, + help="JSON dictionary mapping domains to webroot paths; this " + "implies -d for each entry. You may need to escape this " + "from your shell. E.g.: --webroot-map " + """'{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' """ + "This option is merged with, but takes precedence over, " + "-w / -d entries. At present, if you put webroot-map in " + "a config file, it needs to be on a single line, like: " + 'webroot-map = {"example.com":"/var/www"}.') -class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring +class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring def __init__(self, *args, **kwargs): self.domain_before_webroot = False argparse.Action.__init__(self, *args, **kwargs) - def __call__(self, parser, config, webroot, option_string=None): + def __call__(self, parser, args, webroot, option_string=None): """ Keep a record of --webroot-path / -w flags during processing, so that we know which apply to which -d flags """ - if config.webroot_path is None: # first -w flag encountered - config.webroot_path = [] + if not args.webroot_path: # first -w flag encountered # if any --domain flags preceded the first --webroot-path flag, # apply that webroot path to those; subsequent entries in - # config.webroot_map are filled in by cli.DomainFlagProcessor - if config.domains: + # args.webroot_map are filled in by cli.DomainFlagProcessor + if args.domains: self.domain_before_webroot = True - for d in config.domains: - config.webroot_map.setdefault(d, webroot) + for d in args.domains: + args.webroot_map.setdefault(d, webroot) elif self.domain_before_webroot: - # FIXME if you set domains in a config file, you should get a different error + # FIXME if you set domains in a args file, you should get a different error # here, pointing you to --webroot-map raise errors.Error("If you specify multiple webroot paths, one of " "them must precede all domain flags") - config.webroot_path.append(webroot) + args.webroot_path.append(webroot) -class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring - def __call__(self, parser, config, domain_arg, option_string=None): - """ - Process a new -d flag, helping the webroot plugin construct a map of - {domain : webrootpath} if -w / --webroot-path is in use - """ - for domain in (d.strip() for d in domain_arg.split(",")): - if domain not in config.domains: - config.domains.append(domain) - # Each domain has a webroot_path of the most recent -w flag - if config.webroot_path: - config.webroot_map[domain] = config.webroot_path[-1] +def _process_domain(args_or_config, domain_arg, webroot_path=None): + """ + Process a new -d flag, helping the webroot plugin construct a map of + {domain : webrootpath} if -w / --webroot-path is in use + + :param args_or_config: may be an argparse args object, or a NamespaceConfig object + :param str domain_arg: a string representing 1+ domains, eg: "eg.is, example.com" + :param str webroot_path: (optional) the webroot_path for these domains + + """ + webroot_path = webroot_path if webroot_path else args_or_config.webroot_path + + for domain in (d.strip() for d in domain_arg.split(",")): + domain = le_util.enforce_domain_sanity(domain) + if domain not in args_or_config.domains: + args_or_config.domains.append(domain) + # Each domain has a webroot_path of the most recent -w flag + # unless it was explicitly included in webroot_map + if webroot_path: + args_or_config.webroot_map.setdefault(domain, webroot_path[-1]) -def setup_log_file_handler(args, logfile, fmt): +class WebrootMapProcessor(argparse.Action): # pylint: disable=missing-docstring + def __call__(self, parser, args, webroot_map_arg, option_string=None): + webroot_map = json.loads(webroot_map_arg) + for domains, webroot_path in webroot_map.iteritems(): + _process_domain(args, domains, [webroot_path]) + + +class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring + def __call__(self, parser, args, domain_arg, option_string=None): + """Just wrap _process_domain in argparseese.""" + _process_domain(args, domain_arg) + + +def setup_log_file_handler(config, logfile, fmt): """Setup file debug logging.""" - log_file_path = os.path.join(args.logs_dir, logfile) + log_file_path = os.path.join(config.logs_dir, logfile) handler = logging.handlers.RotatingFileHandler( log_file_path, maxBytes=2 ** 20, backupCount=10) # rotate on each invocation, rollover only possible when maxBytes @@ -1259,8 +1842,8 @@ def setup_log_file_handler(args, logfile, fmt): return handler, log_file_path -def _cli_log_handler(args, level, fmt): - if args.text_mode: +def _cli_log_handler(config, level, fmt): + if config.text_mode or config.noninteractive_mode or config.verb == "renew": handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) else: @@ -1271,13 +1854,13 @@ def _cli_log_handler(args, level, fmt): return handler -def setup_logging(args, cli_handler_factory, logfile): +def setup_logging(config, cli_handler_factory, logfile): """Setup logging.""" fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" - level = -args.verbose_count * 10 + level = -config.verbose_count * 10 file_handler, log_file_path = setup_log_file_handler( - args, logfile=logfile, fmt=fmt) - cli_handler = cli_handler_factory(args, level, fmt) + config, logfile=logfile, fmt=fmt) + cli_handler = cli_handler_factory(config, level, fmt) # TODO: use fileConfig? @@ -1290,12 +1873,12 @@ def setup_logging(args, cli_handler_factory, logfile): logger.info("Saving debug log to %s", log_file_path) -def _handle_exception(exc_type, exc_value, trace, args): +def _handle_exception(exc_type, exc_value, trace, config): """Logs exceptions and reports them to the user. - Args is used to determine how to display exceptions to the user. In - general, if args.debug is True, then the full exception and traceback is - shown to the user, otherwise it is suppressed. If args itself is None, + Config is used to determine how to display exceptions to the user. In + general, if config.debug is True, then the full exception and traceback is + shown to the user, otherwise it is suppressed. If config itself is None, then the traceback and exception is attempted to be written to a logfile. If this is successful, the traceback is suppressed, otherwise it is shown to the user. sys.exit is always called with a nonzero status. @@ -1306,8 +1889,8 @@ def _handle_exception(exc_type, exc_value, trace, args): os.linesep, "".join(traceback.format_exception(exc_type, exc_value, trace))) - if issubclass(exc_type, Exception) and (args is None or not args.debug): - if args is None: + if issubclass(exc_type, Exception) and (config is None or not config.debug): + if config is None: logfile = "letsencrypt.log" try: with open(logfile, "w") as logfd: @@ -1328,15 +1911,15 @@ def _handle_exception(exc_type, exc_value, trace, args): # acme.messages.Error: urn:acme:error:malformed :: The request message was # malformed :: Error creating new registration :: Validation of contact # mailto:none@longrandomstring.biz failed: Server failure at resolver - if ("urn:acme" in err and ":: " in err - and args.verbose_count <= flag_default("verbose_count")): + if (("urn:acme" in err and ":: " in err and + config.verbose_count <= flag_default("verbose_count"))): # prune ACME error code, we have a human description _code, _sep, err = err.partition(":: ") msg = "An unexpected error occurred:\n" + err + "Please see the " - if args is None: + if config is None: msg += "logfile '{0}' for more details.".format(logfile) else: - msg += "logfiles in {0} for more details.".format(args.logs_dir) + msg += "logfiles in {0} for more details.".format(config.logs_dir) sys.exit(msg) else: sys.exit("".join( @@ -1345,10 +1928,10 @@ def _handle_exception(exc_type, exc_value, trace, args): def main(cli_args=sys.argv[1:]): """Command line argument parsing and main script execution.""" - sys.excepthook = functools.partial(_handle_exception, args=None) + sys.excepthook = functools.partial(_handle_exception, config=None) + plugins = plugins_disco.PluginsRegistry.find_all() # note: arg parser internally handles --help (and exits afterwards) - plugins = plugins_disco.PluginsRegistry.find_all() args = prepare_and_parse_args(plugins, cli_args) config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) @@ -1362,19 +1945,24 @@ def main(cli_args=sys.argv[1:]): # TODO: logs might contain sensitive data such as contents of the # private key! #525 le_util.make_or_verify_dir( - args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) - setup_logging(args, _cli_log_handler, logfile='letsencrypt.log') + config.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) + setup_logging(config, _cli_log_handler, logfile='letsencrypt.log') logger.debug("letsencrypt version: %s", letsencrypt.__version__) - # do not log `args`, as it contains sensitive data (e.g. revoke --key)! + # do not log `config`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) logger.debug("Discovered plugins: %r", plugins) - sys.excepthook = functools.partial(_handle_exception, args=args) + sys.excepthook = functools.partial(_handle_exception, config=config) # Displayer - if args.text_mode: + if config.noninteractive_mode: + displayer = display_util.NoninteractiveDisplay(sys.stdout) + elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) + elif config.verb == "renew": + config.noninteractive_mode = True + displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() zope.component.provideUtility(displayer) @@ -1384,18 +1972,7 @@ def main(cli_args=sys.argv[1:]): zope.component.provideUtility(report) atexit.register(report.atexit_print_messages) - if not os.geteuid() == 0: - logger.warning( - "Root (sudo) is required to run most of letsencrypt functionality.") - # check must be done after arg parsing as --help should work - # w/o root; on the other hand, e.g. "letsencrypt run - # --authenticator dns" or "letsencrypt plugins" does not - # require root as well - #return ( - # "{0}Root is required to run letsencrypt. Please use sudo.{0}" - # .format(os.linesep)) - - return args.func(args, config, plugins) + return config.func(config, plugins) if __name__ == "__main__": err_string = main() diff --git a/letsencrypt/client.py b/letsencrypt/client.py index c2dfca1bf..9dfa70e8d 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -146,7 +146,7 @@ def perform_registration(acme, config): """ try: return acme.register(messages.NewRegistration.from_data(email=config.email)) - except messages.Error, e: + except messages.Error as e: err = repr(e) if "MX record" in err or "Validation of contact mailto" in err: config.namespace.email = display_ops.get_email(more=True, invalid=True) @@ -195,7 +195,8 @@ class Client(object): else: self.auth_handler = None - def _obtain_certificate(self, domains, csr): + def obtain_certificate_from_csr(self, domains, csr, + typ=OpenSSL.crypto.FILETYPE_ASN1): """Obtain certificate. Internal function with precondition that `domains` are @@ -223,33 +224,18 @@ class Client(object): authzr = self.auth_handler.get_authorizations(domains) certr = self.acme.request_issuance( - jose.ComparableX509(OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, csr.data)), + jose.ComparableX509( + OpenSSL.crypto.load_certificate_request(typ, csr.data)), authzr) return certr, self.acme.fetch_chain(certr) - def obtain_certificate_from_csr(self, csr): - """Obtain certficiate from CSR. - - :param .le_util.CSR csr: DER-encoded Certificate Signing - Request. - - :returns: `.CertificateResource` and certificate chain (as - returned by `.fetch_chain`). - :rtype: tuple - - """ - return self._obtain_certificate( - # TODO: add CN to domains? - crypto_util.get_sans_from_csr( - csr.data, OpenSSL.crypto.FILETYPE_ASN1), csr) def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. `.register` must be called before `.obtain_certificate` - :param set domains: domains to get a certificate + :param list domains: domains to get a certificate :returns: `.CertificateResource`, certificate chain (as returned by `.fetch_chain`), and newly generated private key @@ -263,7 +249,7 @@ class Client(object): self.config.rsa_key_size, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) - return self._obtain_certificate(domains, csr) + (key, csr) + return self.obtain_certificate_from_csr(domains, csr) + (key, csr) def obtain_and_enroll_certificate(self, domains): """Obtain and enroll certificate. @@ -276,34 +262,28 @@ class Client(object): :param plugins: A PluginsFactory object. :returns: A new :class:`letsencrypt.storage.RenewableCert` instance - referred to the enrolled cert lineage, or False if the cert could - not be obtained. + referred to the enrolled cert lineage, False if the cert could not + be obtained, or None if doing a successful dry run. """ certr, chain, key, _ = self.obtain_certificate(domains) - # XXX: We clearly need a more general and correct way of getting - # options into the configobj for the RenewableCert instance. - # This is a quick-and-dirty way to do it to allow integration - # testing to start. (Note that the config parameter to new_lineage - # ideally should be a ConfigObj, but in this case a dict will be - # accepted in practice.) - params = vars(self.config.namespace) - config = {} - cli_config = configuration.RenewerConfiguration(self.config.namespace) - - if (cli_config.config_dir != constants.CLI_DEFAULTS["config_dir"] or - cli_config.work_dir != constants.CLI_DEFAULTS["work_dir"]): + if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or + self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): logger.warning( "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - lineage = storage.RenewableCert.new_lineage( - domains[0], OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), - key.pem, crypto_util.dump_pyopenssl_chain(chain), - params, config, cli_config) - return lineage + if self.config.dry_run: + logger.info("Dry run: Skipping creating new lineage for %s", + domains[0]) + return None + else: + return storage.RenewableCert.new_lineage( + domains[0], OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), + key.pem, crypto_util.dump_pyopenssl_chain(chain), + configuration.RenewerConfiguration(self.config.namespace)) def save_certificate(self, certr, chain_cert, cert_path, chain_path, fullchain_path): diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index afd5edbe4..c49751a6c 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -1,7 +1,8 @@ """Let's Encrypt user-supplied configuration.""" +import copy import os -import urlparse +from six.moves.urllib import parse # pylint: disable=import-error import zope.interface from letsencrypt import constants @@ -49,7 +50,7 @@ class NamespaceConfig(object): @property def server_path(self): """File path based on ``server``.""" - parsed = urlparse.urlparse(self.namespace.server) + parsed = parse.urlparse(self.namespace.server) return (parsed.netloc + parsed.path).replace('/', os.path.sep) @property @@ -78,6 +79,12 @@ class NamespaceConfig(object): return os.path.join( self.namespace.work_dir, constants.TEMP_CHECKPOINT_DIR) + def __deepcopy__(self, _memo): + # Work around https://bugs.python.org/issue1515 for py26 tests :( :( + # https://travis-ci.org/letsencrypt/letsencrypt/jobs/106900743#L3276 + new_ns = copy.deepcopy(self.namespace) + return type(self)(new_ns) + class RenewerConfiguration(object): """Configuration wrapper for renewer.""" @@ -124,4 +131,5 @@ def check_config_sanity(config): # Domain checks if config.namespace.domains is not None: for domain in config.namespace.domains: - le_util.check_domain_sanity(domain) + # This may be redundant, but let's be paranoid + le_util.enforce_domain_sanity(domain) diff --git a/letsencrypt/constants.py b/letsencrypt/constants.py index a1dccd1ea..402f5e9a1 100644 --- a/letsencrypt/constants.py +++ b/letsencrypt/constants.py @@ -37,7 +37,9 @@ STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" RENEWER_DEFAULTS = dict( renewer_enabled="yes", renew_before_expiry="30 days", - deploy_before_expiry="20 days", + # This value should ensure that there is never a deployment delay by + # default. + deploy_before_expiry="99 years", ) """Defaults for renewer script.""" diff --git a/letsencrypt/crypto_util.py b/letsencrypt/crypto_util.py index 730c32398..76265a739 100644 --- a/letsencrypt/crypto_util.py +++ b/letsencrypt/crypto_util.py @@ -53,8 +53,8 @@ def init_save_key(key_size, key_dir, keyname="key-letsencrypt.pem"): config.strict_permissions) key_f, key_path = le_util.unique_file( os.path.join(key_dir, keyname), 0o600) - key_f.write(key_pem) - key_f.close() + with key_f: + key_f.write(key_pem) logger.info("Generating key (%d bits): %s", key_size, key_path) diff --git a/letsencrypt/display/enhancements.py b/letsencrypt/display/enhancements.py index c56198161..39def1651 100644 --- a/letsencrypt/display/enhancements.py +++ b/letsencrypt/display/enhancements.py @@ -48,7 +48,7 @@ def redirect_by_default(): code, selection = util(interfaces.IDisplay).menu( "Please choose whether HTTPS access is required or optional.", - choices) + choices, default=0, cli_flag="--redirect / --no-redirect") if code != display_util.OK: return False diff --git a/letsencrypt/display/ops.py b/letsencrypt/display/ops.py index 5ceb7fcfc..f0dec8b06 100644 --- a/letsencrypt/display/ops.py +++ b/letsencrypt/display/ops.py @@ -31,8 +31,8 @@ def choose_plugin(prepared, question): for plugin_ep in prepared] while True: - code, index = util(interfaces.IDisplay).menu( - question, opts, help_label="More Info") + disp = util(interfaces.IDisplay) + code, index = disp.menu(question, opts, help_label="More Info") if code == display_util.OK: plugin_ep = prepared[index] @@ -74,6 +74,17 @@ def pick_plugin(config, default, plugins, question, ifaces): # throw more UX-friendly error if default not in plugins filtered = plugins.filter(lambda p_ep: p_ep.name == default) else: + if config.noninteractive_mode: + # it's really bad to auto-select the single available plugin in + # non-interactive mode, because an update could later add a second + # available plugin + raise errors.MissingCommandlineFlag( + "Missing command line flags. For non-interactive execution, " + "you will need to specify a plugin on the command line. Run " + "with '--help plugins' to see a list of options, and see " + "https://eff.org/letsencrypt-plugins for more detail on what " + "the plugins do and how to use them.") + filtered = plugins.visible().ifaces(ifaces) filtered.init(config) @@ -143,7 +154,12 @@ def get_email(more=False, invalid=False): msg += ('\n\nIf you really want to skip this, you can run the client with ' '--register-unsafely-without-email but make sure you backup your ' 'account key from /etc/letsencrypt/accounts\n\n') - code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + try: + code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) + except errors.MissingCommandlineFlag: + msg = ("You should register before running non-interactively, or provide --agree-tos" + " and --email flags") + raise errors.MissingCommandlineFlag(msg) if code == display_util.OK: if le_util.safe_email(email): @@ -197,7 +213,8 @@ def choose_names(installer): "specify ServerNames in your config files in order to allow for " "accurate installation of your certificate.{0}" "If you do use the default vhost, you may specify the name " - "manually. Would you like to continue?{0}".format(os.linesep)) + "manually. Would you like to continue?{0}".format(os.linesep), + default=True) if manual: return _choose_names_manually() @@ -222,8 +239,7 @@ def get_valid_domains(domains): valid_domains = [] for domain in domains: try: - le_util.check_domain_sanity(domain) - valid_domains.append(domain) + valid_domains.append(le_util.enforce_domain_sanity(domain)) except errors.ConfigurationError: continue return valid_domains @@ -242,7 +258,7 @@ def _filter_names(names): """ code, names = util(interfaces.IDisplay).checklist( "Which names would you like to activate HTTPS for?", - tags=names) + tags=names, cli_flag="--domains") return code, [str(s) for s in names] @@ -250,7 +266,8 @@ def _choose_names_manually(): """Manually input names for those without an installer.""" code, input_ = util(interfaces.IDisplay).input( - "Please enter in your domain name(s) (comma and/or space separated) ") + "Please enter in your domain name(s) (comma and/or space separated) ", + cli_flag="--domains") if code == display_util.OK: invalid_domains = dict() @@ -264,9 +281,9 @@ def _choose_names_manually(): "supported.{0}{0}Would you like to re-enter the " "names?{0}").format(os.linesep) - for domain in domain_list: + for i, domain in enumerate(domain_list): try: - le_util.check_domain_sanity(domain) + domain_list[i] = le_util.enforce_domain_sanity(domain) except errors.ConfigurationError as e: invalid_domains[domain] = e.message @@ -309,22 +326,24 @@ def success_installation(domains): pause=False) -def success_renewal(domains): +def success_renewal(domains, action): """Display a box confirming the renewal of an existing certificate. .. todo:: This should be centered on the screen :param list domains: domain names which were renewed + :param str action: can be "reinstall" or "renew" """ util(interfaces.IDisplay).notification( - "Your existing certificate has been successfully renewed, and the " + "Your existing certificate has been successfully {3}ed, and the " "new certificate has been installed.{1}{1}" "The new certificate covers the following domains: {0}{1}{1}" "You should test your configuration at:{1}{2}".format( _gen_https_names(domains), os.linesep, - os.linesep.join(_gen_ssl_lab_urls(domains))), + os.linesep.join(_gen_ssl_lab_urls(domains)), + action), height=(14 + len(domains)), pause=False) diff --git a/letsencrypt/display/util.py b/letsencrypt/display/util.py index 01a8cbc92..976a2afdf 100644 --- a/letsencrypt/display/util.py +++ b/letsencrypt/display/util.py @@ -6,7 +6,7 @@ import dialog import zope.interface from letsencrypt import interfaces - +from letsencrypt import errors WIDTH = 72 HEIGHT = 20 @@ -21,6 +21,20 @@ CANCEL = "cancel" HELP = "help" """Display exit code when for when the user requests more help.""" +def _wrap_lines(msg): + """Format lines nicely to 80 chars. + + :param str msg: Original message + + :returns: Formatted message respecting newlines in message + :rtype: str + + """ + lines = msg.splitlines() + fixed_l = [] + for line in lines: + fixed_l.append(textwrap.fill(line, 80)) + return os.linesep.join(fixed_l) class NcursesDisplay(object): """Ncurses-based display.""" @@ -49,8 +63,8 @@ class NcursesDisplay(object): """ self.dialog.msgbox(message, height, width=self.width) - def menu(self, message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(self, message, choices, ok_label="OK", cancel_label="Cancel", + help_label="", **unused_kwargs): """Display a menu. :param str message: title of menu @@ -61,10 +75,11 @@ class NcursesDisplay(object): :param str ok_label: label of the OK button :param str help_label: label of the help button + :param dict unused_kwargs: absorbs default / cli_args - :returns: tuple of the form (`code`, `tag`) where - `code` - `str` display_util exit code - `tag` - `int` index corresponding to the item chosen + :returns: tuple of the form (`code`, `index`) where + `code` - int display exit code + `int` - index of the selected item :rtype: tuple """ @@ -97,20 +112,21 @@ class NcursesDisplay(object): (str(i), choice) for i, choice in enumerate(choices, 1) ] # pylint: disable=star-args - code, tag = self.dialog.menu(message, **menu_options) + code, index = self.dialog.menu(message, **menu_options) if code == CANCEL: return code, -1 - return code, int(tag) - 1 + return code, int(index) - 1 - def input(self, message): + def input(self, message, **unused_kwargs): """Display an input box to the user. :param str message: Message to display that asks for input. + :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, string) where + :returns: tuple of the form (`code`, `string`) where `code` - int display exit code `string` - input entered by the user @@ -122,7 +138,7 @@ class NcursesDisplay(object): return self.dialog.inputbox(message, width=self.width, height=height) - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Display a Yes/No dialog box. Yes and No label must begin with different letters. @@ -130,6 +146,7 @@ class NcursesDisplay(object): :param str message: message to display to user :param str yes_label: label on the "yes" button :param str no_label: label on the "no" button + :param dict _kwargs: absorbs default / cli_args :returns: if yes_label was selected :rtype: bool @@ -139,16 +156,17 @@ class NcursesDisplay(object): message, self.height, self.width, yes_label=yes_label, no_label=no_label) - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **unused_kwargs): """Displays a checklist. :param message: Message to display before choices :param list tags: where each is of type :class:`str` len(tags) > 0 :param bool default_status: If True, items are in a selected state by default. + :param dict _kwargs: absorbs default / cli_args - :returns: tuple of the form (code, list_tags) where + :returns: tuple of the form (`code`, `list_tags`) where `code` - int display exit code `list_tags` - list of str tags selected by the user @@ -178,15 +196,15 @@ class FileDisplay(object): """ side_frame = "-" * 79 - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write( "{line}{frame}{line}{msg}{line}{frame}{line}".format( line=os.linesep, frame=side_frame, msg=message)) if pause: raw_input("Press Enter to Continue") - def menu(self, message, choices, - ok_label="", cancel_label="", help_label=""): + def menu(self, message, choices, ok_label="", cancel_label="", + help_label="", **unused_kwargs): # pylint: disable=unused-argument """Display a menu. @@ -197,10 +215,12 @@ class FileDisplay(object): :param choices: Menu lines, len must be > 0 :type choices: list of tuples (tag, item) or list of descriptions (tags will be enumerated) + :param dict _kwargs: absorbs default / cli_args + + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection - :returns: tuple of the form (code, tag) where - code - int display exit code - tag - str corresponding to the item chosen :rtype: tuple """ @@ -210,11 +230,12 @@ class FileDisplay(object): return code, selection - 1 - def input(self, message): + def input(self, message, **unused_kwargs): # pylint: disable=no-self-use """Accept input from the user. :param str message: message to display to the user + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `input`) where `code` - str display exit code @@ -230,7 +251,7 @@ class FileDisplay(object): else: return OK, ans - def yesno(self, message, yes_label="Yes", no_label="No"): + def yesno(self, message, yes_label="Yes", no_label="No", **unused_kwargs): """Query the user with a yes/no question. Yes and No label must begin with different letters, and must contain at @@ -239,6 +260,7 @@ class FileDisplay(object): :param str message: question for the user :param str yes_label: Label of the "Yes" parameter :param str no_label: Label of the "No" parameter + :param dict _kwargs: absorbs default / cli_args :returns: True for "Yes", False for "No" :rtype: bool @@ -246,7 +268,7 @@ class FileDisplay(object): """ side_frame = ("-" * 79) + os.linesep - message = self._wrap_lines(message) + message = _wrap_lines(message) self.outfile.write("{0}{frame}{msg}{0}{frame}".format( os.linesep, frame=side_frame, msg=message)) @@ -265,13 +287,14 @@ class FileDisplay(object): ans.startswith(no_label[0].upper())): return False - def checklist(self, message, tags, default_status=True): + def checklist(self, message, tags, default_status=True, **unused_kwargs): # pylint: disable=unused-argument """Display a checklist. :param str message: Message to display to user :param list tags: `str` tags to select, len(tags) > 0 :param bool default_status: Not used for FileDisplay + :param dict _kwargs: absorbs default / cli_args :returns: tuple of (`code`, `tags`) where `code` - str display exit code @@ -352,21 +375,6 @@ class FileDisplay(object): self.outfile.write(side_frame) - def _wrap_lines(self, msg): # pylint: disable=no-self-use - """Format lines nicely to 80 chars. - - :param str msg: Original message - - :returns: Formatted message respecting newlines in message - :rtype: str - - """ - lines = msg.splitlines() - fixed_l = [] - for line in lines: - fixed_l.append(textwrap.fill(line, 80)) - - return os.linesep.join(fixed_l) def _get_valid_int_ans(self, max_): """Get a numerical selection. @@ -403,6 +411,118 @@ class FileDisplay(object): return OK, selection +class NoninteractiveDisplay(object): + """An iDisplay implementation that never asks for interactive user input""" + + zope.interface.implements(interfaces.IDisplay) + + def __init__(self, outfile): + super(NoninteractiveDisplay, self).__init__() + self.outfile = outfile + + def _interaction_fail(self, message, cli_flag, extra=""): + "Error out in case of an attempt to interact in noninteractive mode" + msg = "Missing command line flag or config entry for this setting:\n" + msg += message + if extra: + msg += "\n" + extra + if cli_flag: + msg += "\n\n(You can set this with the {0} flag)".format(cli_flag) + raise errors.MissingCommandlineFlag(msg) + + def notification(self, message, height=10, pause=False): + # pylint: disable=unused-argument + """Displays a notification without waiting for user acceptance. + + :param str message: Message to display to stdout + :param int height: No effect for NoninteractiveDisplay + :param bool pause: The NoninteractiveDisplay waits for no keyboard + + """ + side_frame = "-" * 79 + message = _wrap_lines(message) + self.outfile.write( + "{line}{frame}{line}{msg}{line}{frame}{line}".format( + line=os.linesep, frame=side_frame, msg=message)) + + def menu(self, message, choices, ok_label=None, cancel_label=None, + help_label=None, default=None, cli_flag=None): + # pylint: disable=unused-argument,too-many-arguments + """Avoid displaying a menu. + + :param str message: title of menu + :param choices: Menu lines, len must be > 0 + :type choices: list of tuples (tag, item) or + list of descriptions (tags will be enumerated) + :param int default: the default choice + :param dict kwargs: absorbs various irrelevant labelling arguments + + :returns: tuple of (`code`, `index`) where + `code` - str display exit code + `index` - int index of the user's selection + :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default + + """ + if default is None: + self._interaction_fail(message, cli_flag, "Choices: " + repr(choices)) + + return OK, default + + def input(self, message, default=None, cli_flag=None): + """Accept input from the user. + + :param str message: message to display to the user + + :returns: tuple of (`code`, `input`) where + `code` - str display exit code + `input` - str of the user's input + :rtype: tuple + :raises errors.MissingCommandlineFlag: if there was no default + + """ + if default is None: + self._interaction_fail(message, cli_flag) + else: + return OK, default + + + def yesno(self, message, yes_label=None, no_label=None, default=None, cli_flag=None): + # pylint: disable=unused-argument + """Decide Yes or No, without asking anybody + + :param str message: question for the user + :param dict kwargs: absorbs yes_label, no_label + + :raises errors.MissingCommandlineFlag: if there was no default + :returns: True for "Yes", False for "No" + :rtype: bool + + """ + if default is None: + self._interaction_fail(message, cli_flag) + else: + return default + + def checklist(self, message, tags, default=None, cli_flag=None, **kwargs): + # pylint: disable=unused-argument + """Display a checklist. + + :param str message: Message to display to user + :param list tags: `str` tags to select, len(tags) > 0 + :param dict kwargs: absorbs default_status arg + + :returns: tuple of (`code`, `tags`) where + `code` - str display exit code + `tags` - list of selected tags + :rtype: tuple + + """ + if default is None: + self._interaction_fail(message, cli_flag, "? ".join(tags)) + else: + return OK, default + def separate_list_input(input_): """Separate a comma or space separated list. diff --git a/letsencrypt/errors.py b/letsencrypt/errors.py index 1358d1048..99bb29d9d 100644 --- a/letsencrypt/errors.py +++ b/letsencrypt/errors.py @@ -102,3 +102,8 @@ class StandaloneBindError(Error): class ConfigurationError(Error): """Configuration sanity error.""" + +# NoninteractiveDisplay iDisplay plugin error: + +class MissingCommandlineFlag(Error): + """A command line argument was missing in noninteractive usage""" diff --git a/letsencrypt/interfaces.py b/letsencrypt/interfaces.py index c8a725fde..1921b1e54 100644 --- a/letsencrypt/interfaces.py +++ b/letsencrypt/interfaces.py @@ -169,7 +169,9 @@ class IAuthenticator(IPlugin): Authenticator will never be able to perform (error). :rtype: :class:`list` of - :class:`acme.challenges.ChallengeResponse` + :class:`acme.challenges.ChallengeResponse`, + where responses are required to be returned in + the same order as corresponding input challenges :raises .PluginError: If challenges cannot be performed @@ -365,8 +367,8 @@ class IDisplay(zope.interface.Interface): """ - def menu(message, choices, - ok_label="OK", cancel_label="Cancel", help_label=""): + def menu(message, choices, ok_label="OK", # pylint: disable=too-many-arguments + cancel_label="Cancel", help_label="", default=None, cli_flag=None): """Displays a generic menu. :param str message: message to display @@ -377,14 +379,19 @@ class IDisplay(zope.interface.Interface): :param str ok_label: label for OK button :param str cancel_label: label for Cancel button :param str help_label: label for Help button + :param int default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--keep" :returns: tuple of (`code`, `index`) where `code` - str display exit code `index` - int index of the user's selection + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def input(message): + def input(message, default=None, cli_args=None): """Accept input from the user. :param str message: message to display to the user @@ -394,27 +401,45 @@ class IDisplay(zope.interface.Interface): `input` - str of the user's input :rtype: tuple + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def yesno(message, yes_label="Yes", no_label="No"): + def yesno(message, yes_label="Yes", no_label="No", default=None, + cli_args=None): """Query the user with a yes/no question. Yes and No label must begin with different letters. :param str message: question for the user + :param str default: default (non-interactive) choice from the menu + :param str cli_flag: to automate choice from the menu, eg "--redirect / --no-redirect" :returns: True for "Yes", False for "No" :rtype: bool + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set + """ - def checklist(message, tags, default_state): + def checklist(message, tags, default_state, default=None, cli_args=None): """Allow for multiple selections from a menu. :param str message: message to display to the user :param list tags: where each is of type :class:`str` len(tags) > 0 - :param bool default_status: If True, items are in a selected state by - default. + :param bool default_status: If True, items are in a selected state by default. + :param str default: default (non-interactive) state of the checklist + :param str cli_flag: to automate choice from the menu, eg "--domains" + + :returns: tuple of the form (code, list_tags) where + `code` - int display exit code + `list_tags` - list of str tags selected by the user + :rtype: tuple + + :raises errors.MissingCommandlineFlag: if called in non-interactive + mode without a default set """ diff --git a/letsencrypt/le_util.py b/letsencrypt/le_util.py index 64295a80f..c8a9d24c2 100644 --- a/letsencrypt/le_util.py +++ b/letsencrypt/le_util.py @@ -6,6 +6,7 @@ import logging import os import platform import re +import socket import stat import subprocess import sys @@ -285,31 +286,49 @@ def add_deprecated_argument(add_argument, argument_name, nargs): help=argparse.SUPPRESS, nargs=nargs) -def check_domain_sanity(domain): +def enforce_domain_sanity(domain): """Method which validates domain value and errors out if the requirements are not met. :param domain: Domain to check - :type domains: `string` + :type domains: `str` or `unicode` :raises ConfigurationError: for invalid domains and cases where Let's Encrypt currently will not issue certificates + :returns: The domain cast to `str`, with ASCII-only contents + :rtype: str """ # Check if there's a wildcard domain if domain.startswith("*."): raise errors.ConfigurationError( - "Wildcard domains are not supported") + "Wildcard domains are not supported: {0}".format(domain)) # Punycode if "xn--" in domain: raise errors.ConfigurationError( - "Punycode domains are not presently supported") + "Punycode domains are not presently supported: {0}".format(domain)) # Unicode try: - domain.encode('ascii') + domain = domain.encode('ascii').lower() except UnicodeDecodeError: raise errors.ConfigurationError( - "Internationalized domain names are not presently supported") + "Internationalized domain names are not presently supported: {0}" + .format(domain)) + + # Remove trailing dot + domain = domain[:-1] if domain.endswith('.') else domain + + # Explain separately that IP addresses aren't allowed (apart from not + # being FQDNs) because hope springs eternal concerning this point + try: + socket.inet_aton(domain) + raise errors.ConfigurationError( + "Requested name {0} is an IP address. The Let's Encrypt " + "certificate authority will not issue certificates for a " + "bare IP address.".format(domain)) + except socket.error: + # It wasn't an IP address, so that's good + pass # FQDN checks from # http://www.mkyong.com/regular-expressions/domain-name-regular-expression-example/ @@ -317,4 +336,5 @@ def check_domain_sanity(domain): # first and last char is not "-" fqdn = re.compile("^((?!-)[A-Za-z0-9-]{1,63}(? +staging = True +prepare = False +work_dir = /var/lib/letsencrypt +tos = False +init = False +http01_port = 80 +duplicate = False +noninteractive_mode = True +key_path = None +nginx = False +nginx_server_root = /etc/nginx +fullchain_path = /home/ubuntu/letsencrypt/chain.pem +email = None +csr = None +agree_dev_preview = None +redirect = None +verb = certonly +verbose_count = -3 +config_file = None +renew_by_default = False +hsts = False +apache_handle_sites = True +authenticator = standalone +domains = isnot.org, +rsa_key_size = 2048 +apache_challenge_location = /etc/apache2 +checkpoints = 1 +manual_test_mode = False +apache = False +cert_path = /home/ubuntu/letsencrypt/cert.pem +webroot_path = None +reinstall = False +expand = False +strict_permissions = False +apache_server_root = /etc/apache2 +account = None +dry_run = False +manual_public_ip_logging_ok = False +chain_path = /home/ubuntu/letsencrypt/chain.pem +break_my_certs = False +standalone = True +manual = False +server = https://acme-staging.api.letsencrypt.org/directory +standalone_supported_challenges = "tls-sni-01,http-01" +webroot = False +os_packages_only = False +apache_init_script = None +user_agent = None +apache_le_vhost_ext = -le-ssl.conf +debug = False +tls_sni_01_port = 443 +logs_dir = /var/log/letsencrypt +apache_vhost_root = /etc/apache2/sites-available +configurator = None +[[webroot_map]] diff --git a/letsencrypt/tests/testdata/webrootconftest.ini b/letsencrypt/tests/testdata/webrootconftest.ini new file mode 100644 index 000000000..de3bd98a6 --- /dev/null +++ b/letsencrypt/tests/testdata/webrootconftest.ini @@ -0,0 +1,3 @@ +webroot +webroot-path = /tmp +domains = eg.com, eg2.com diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index d487e556d..fff8dcfc3 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.2.0.dev0' +version = '0.5.0.dev0' install_requires = [ 'setuptools', # pkg_resources @@ -55,4 +55,5 @@ setup( 'letshelp-letsencrypt-apache = letshelp_letsencrypt.apache:main', ], }, + test_suite='letshelp_letsencrypt', ) diff --git a/setup.cfg b/setup.cfg index ca4c1b1ca..1ea06661e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,6 @@ [easy_install] zip_ok = false -[aliases] -dev = develop easy_install letsencrypt[dev,docs,testing] - [nosetests] nocapture=1 cover-package=letsencrypt,acme,letsencrypt_apache,letsencrypt_nginx diff --git a/setup.py b/setup.py index ad7fb6909..cbf0ff89d 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,10 @@ version = meta['version'] # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), + # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but + # saying so here causes a runtime error against our temporary fork of 0.9.3 + # in which we added 2.6 support (see #2243), so we relax the requirement. + 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime', @@ -48,23 +52,25 @@ install_requires = [ ] # env markers in extras_require cause problems with older pip: #517 +# Keep in sync with conditional_requirements.py. if sys.version_info < (2, 7): install_requires.extend([ # only some distros recognize stdlib argparse as already satisfying 'argparse', - 'ConfigArgParse>=0.10.0', # python2.6 support, upstream #17 'mock<1.1.0', ]) else: - install_requires.extend([ - 'ConfigArgParse', - 'mock', - ]) + install_requires.append('mock') dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', + 'coverage', + 'nose', + 'nosexcover', + 'pep8', 'pylint==1.4.2', # upstream #248 + 'tox', 'twine', 'wheel', ] @@ -76,14 +82,6 @@ docs_extras = [ 'sphinxcontrib-programoutput', ] -testing_extras = [ - 'coverage', - 'nose', - 'nosexcover', - 'pep8', - 'tox', -] - setup( name='letsencrypt', version=version, @@ -119,10 +117,8 @@ setup( extras_require={ 'dev': dev_extras, 'docs': docs_extras, - 'testing': testing_extras, }, - tests_require=install_requires, # to test all packages run "python setup.py test -s # {acme,letsencrypt_apache,letsencrypt_nginx}" test_suite='letsencrypt', @@ -130,7 +126,6 @@ setup( entry_points={ 'console_scripts': [ 'letsencrypt = letsencrypt.cli:main', - 'letsencrypt-renewer = letsencrypt.renewer:main', ], 'letsencrypt.plugins': [ 'manual = letsencrypt.plugins.manual:Authenticator', diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 53996cd20..32c292e90 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -20,16 +20,22 @@ else readlink="readlink" fi -common() { - letsencrypt_test \ +common_no_force_renew() { + letsencrypt_test_no_force_renew \ --authenticator standalone \ --installer null \ "$@" } +common() { + common_no_force_renew \ + --renew-by-default \ + "$@" +} + common --domains le1.wtf --standalone-supported-challenges tls-sni-01 auth common --domains le2.wtf --standalone-supported-challenges http-01 run -common -a manual -d le.wtf auth +common -a manual -d le.wtf auth --rsa-key-size 4096 export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf @@ -44,20 +50,39 @@ common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ --key-path "${root}/csr/key.pem" -# the following assumes that Boulder issues certificates for less than -# 10 years, otherwise renewal will not take place -cat < "$root/conf/renewer.conf" -renew_before_expiry = 10 years -deploy_before_expiry = 10 years -EOF -letsencrypt-renewer $store_flags -dir="$root/conf/archive/le1.wtf" -for x in cert chain fullchain privkey; -do - latest="$(ls -1t $dir/ | grep -e "^${x}" | head -n1)" - live="$($readlink -f "$root/conf/live/le1.wtf/${x}.pem")" - [ "${dir}/${latest}" = "$live" ] # renewer fails this test -done +CheckCertCount() { + CERTCOUNT=`ls "${root}/conf/archive/le.wtf/cert"* | wc -l` + if [ "$CERTCOUNT" -ne "$1" ] ; then + echo Wrong cert count, not "$1" `ls "${root}/conf/archive/le.wtf/"*` + exit 1 + fi +} + +CheckCertCount 1 +# This won't renew (because it's not time yet) +common_no_force_renew renew +CheckCertCount 1 + +# --renew-by-default is used, so renewal should occur +common renew +CheckCertCount 2 + +# This will renew because the expiry is less than 10 years from now +sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le.wtf.conf" +common_no_force_renew renew --rsa-key-size 2048 +CheckCertCount 3 + +# The 4096 bit setting should persist to the first renewal, but be overriden in the second + +size1=`wc -c ${root}/conf/archive/le.wtf/privkey1.pem | cut -d" " -f1` +size2=`wc -c ${root}/conf/archive/le.wtf/privkey2.pem | cut -d" " -f1` +size3=`wc -c ${root}/conf/archive/le.wtf/privkey3.pem | cut -d" " -f1` +# 4096 bit PEM keys are about ~3270 bytes, 2048 ones are about 1700 bytes +if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; then + echo key sizes violate assumptions: + ls -l "${root}/conf/archive/le.wtf/privkey"* + exit 1 +fi # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 4572b0fb3..e86d087cb 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -12,6 +12,12 @@ store_flags="$store_flags --logs-dir $root/logs" export root store_flags letsencrypt_test () { + letsencrypt_test_no_force_renew \ + --renew-by-default \ + "$@" +} + +letsencrypt_test_no_force_renew () { letsencrypt \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ @@ -19,11 +25,10 @@ letsencrypt_test () { --http-01-port 5002 \ --manual-test-mode \ $store_flags \ - --text \ + --non-interactive \ --no-redirect \ --agree-tos \ --register-unsafely-without-email \ - --renew-by-default \ --debug \ -vvvvvvv \ "$@" diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index dee6968c3..e27385002 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -139,7 +139,15 @@ def make_instance(instance_name, time.sleep(1.0) # give instance a name - new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + try: + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + except botocore.exceptions.ClientError as e: + if "InvalidInstanceID.NotFound" in str(e): + # This seems to be ephemeral... retry + time.sleep(1) + new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}]) + else: + raise return new_instance def terminate_and_clean(instances): @@ -233,21 +241,21 @@ def local_git_clone(repo_url): "clones master of repo_url" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s'% repo_url) + local('git clone %s letsencrypt'% repo_url) local('tar czf le.tar.gz letsencrypt') def local_git_branch(repo_url, branch_name): "clones branch of repo_url" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s --branch %s --single-branch'%(repo_url, branch_name)) + local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name)) local('tar czf le.tar.gz letsencrypt') def local_git_PR(repo_url, PRnumstr, merge_master=True): "clones specified pull request from repo_url and optionally merges into master" with lcd(LOGDIR): local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s'% repo_url) + local('git clone %s letsencrypt'% repo_url) local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) local('cd letsencrypt && git co lePRtest') if merge_master: diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4032e2195..940cc36c6 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -35,22 +35,16 @@ then #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/ fi -# run letsencrypt-apache2 via letsencrypt-auto +# Run letsencrypt-apache2. cd letsencrypt -export SUDO=sudo -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO bootstrap/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO bootstrap/_rpm_common.sh -else - echo "Dont have bootstrapping for this OS!" - exit 1 +echo "Bootstrapping dependencies..." +letsencrypt-auto-source/letsencrypt-auto --os-packages-only +if [ $? -ne 0 ] ; then + exit 1 fi -bootstrap/dev/venv.sh +tools/venv.sh sudo venv/bin/letsencrypt -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index b7849755a..0ad0d6081 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -8,12 +8,28 @@ cd letsencrypt SAVE="$PIP_EXTRA_INDEX_URL" unset PIP_EXTRA_INDEX_URL export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" -./letsencrypt-auto -v --debug --version + +#OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" + + +if ! command -v git ; then + if [ "$OS_TYPE" = "ubuntu" ] ; then + sudo apt-get update + fi + if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then + echo git installation failed! + exit 1 + fi +fi +BRANCH=`git rev-parse --abbrev-ref HEAD` +git checkout v0.1.0 +./letsencrypt-auto -v --debug --version unset PIP_INDEX_URL export PIP_EXTRA_INDEX_URL="$SAVE" -if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then +git checkout -f "$BRANCH" +if ! ./letsencrypt-auto -v --debug --version | grep 0.3.0 ; then echo upgrade appeared to fail exit 1 fi diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 10d7c3b5e..f4aef11fe 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -8,7 +8,8 @@ #private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) cd letsencrypt -./letsencrypt-auto certonly -v --standalone --debug \ +./letsencrypt-auto --os-packages-only --debug --version +./letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect \ --register-unsafely-without-email \ diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh new file mode 100755 index 000000000..d90ae9ab6 --- /dev/null +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -0,0 +1,55 @@ +#!/bin/bash -x + +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution + +# run letsencrypt-apache2 via letsencrypt-auto +cd letsencrypt + +export SUDO=sudo +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO bootstrap/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO bootstrap/_rpm_common.sh +else + echo "Dont have bootstrapping for this OS!" + exit 1 +fi + +bootstrap/dev/venv.sh +sudo venv/bin/letsencrypt certonly --debug --standalone -t --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -v +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +if [ "$OS_TYPE" = "ubuntu" ] ; then + venv/bin/tox -e apacheconftest +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +sudo venv/bin/letsencrypt renew --renew-by-default + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + +ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed +if [ "$FAIL" = 1 ] ; then + exit 1 +fi diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index f7f325d5c..4c2eb429e 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -6,71 +6,15 @@ VENV_NAME="venv" LEA_PATH=./letsencrypt/ VENV_PATH=${LEA_PATH/$VENV_NAME} VENV_BIN=${VENV_PATH}/bin -BOOTSTRAP=${LEA_PATH}/bootstrap -SUDO=sudo - -ExperimentalBootstrap() { - # Arguments: Platform name, boostrap script name, SUDO command (iff needed) - if [ "$2" != "" ] ; then - echo "Bootstrapping dependencies for $1..." - if [ "$3" != "" ] ; then - "$3" "$BOOTSTRAP/$2" - else - "$BOOTSTRAP/$2" - fi - fi -} # virtualenv call is not idempotent: it overwrites pip upgraded in # later steps, causing "ImportError: cannot import name unpack_url" -if [ ! -f $BOOTSTRAP/debian.sh ] ; then - echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP" - exit 1 -fi -if [ -f /etc/debian_version ] ; then - echo "Bootstrapping dependencies for Debian-based OSes..." - $SUDO $BOOTSTRAP/_deb_common.sh -elif [ -f /etc/redhat-release ] ; then - echo "Bootstrapping dependencies for RedHat-based OSes..." - $SUDO $BOOTSTRAP/_rpm_common.sh -elif `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." - $SUDO $BOOTSTRAP/_suse_common.sh -elif [ -f /etc/arch-release ] ; then - if [ "$DEBUG" = 1 ] ; then - echo "Bootstrapping dependencies for Archlinux..." - $SUDO $BOOTSTRAP/archlinux.sh - else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." - exit 1 - fi -elif [ -f /etc/manjaro-release ] ; then - ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO" -elif [ -f /etc/gentoo-release ] ; then - ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO" -elif uname | grep -iq FreeBSD ; then - ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO" -elif uname | grep -iq Darwin ; then - ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root -elif grep -iq "Amazon Linux" /etc/issue ; then - ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO" -else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run a pip install manually" - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info" -fi -echo "Bootstrapped!" +"$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./bootstrap/dev/venv.sh +./tools/venv.sh PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ $PYVER -eq 26 ] ; then diff --git a/bootstrap/dev/_venv_common.sh b/tools/_venv_common.sh similarity index 100% rename from bootstrap/dev/_venv_common.sh rename to tools/_venv_common.sh diff --git a/tools/half-sign.c b/tools/half-sign.c index 454201799..e56bc397c 100644 --- a/tools/half-sign.c +++ b/tools/half-sign.c @@ -9,18 +9,21 @@ // This program can be used to perform RSA public key signatures given only // the hash of the file to be signed as input. -// Sign with SHA1 -#define HASH_SIZE 20 +// To compile: +// gcc half-sign.c -lssl -lcrypto -o half-sign + +// Sign with SHA256 +#define HASH_SIZE 32 void usage() { printf("half-sign [binary hash file]\n"); printf("\n"); - printf(" Computes and prints a binary RSA signature over data given the SHA1 hash of\n"); + printf(" Computes and prints a binary RSA signature over data given the SHA256 hash of\n"); printf(" the data as input.\n"); printf("\n"); printf(" should be PEM encoded.\n"); printf("\n"); - printf(" The input SHA1 hash should be %d bytes in length. If no binary hash file is\n", HASH_SIZE); + printf(" The input SHA256 hash should be %d bytes in length. If no binary hash file is\n", HASH_SIZE); printf(" specified, it will be read from stdin.\n"); exit(1); } @@ -38,7 +41,7 @@ void sign_hashed_data(EVP_PKEY *signing_key, unsigned char *md, size_t mdlen) { if ((!ctx) || (EVP_PKEY_sign_init(ctx) <= 0) || (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) - || (EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha1()) <= 0)) { + || (EVP_PKEY_CTX_set_signature_md(ctx, EVP_sha256()) <= 0)) { fprintf(stderr, "Failure establishing ctx for signature\n"); exit(1); } @@ -105,7 +108,7 @@ int main(int argc, char *argv[]) { exit(1); } if (fread(buffer, HASH_SIZE, 1, input) != 1) { - perror("half-sign: Failed to read SHA1 from input\n"); + perror("half-sign: Failed to read SHA256 from input\n"); exit(1); } diff --git a/tools/offline-sigrequest.sh b/tools/offline-sigrequest.sh new file mode 100755 index 000000000..7706796ef --- /dev/null +++ b/tools/offline-sigrequest.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -o errexit + +if ! `which festival > /dev/null` ; then + echo Please install \'festival\'! + exit 1 +fi + +function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL + while read -p "Press Enter to read the hash aloud or type 'done': " INP && [ "$INP" = "" ] ; do + cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \ + echo -n '(SayText "'; \ + sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \ + echo '")' ) | festival + done + + echo 'Paste in the data from the QR code, then type Ctrl-D:' + cat > $2 +} + +function offlinesign { # $1 <-- INPFILE ; $2 <---SIGFILE + echo HASH FOR SIGNING: + SIGFILEBALL="$2.lzma.base64" + #echo "(place the resulting raw binary signature in $SIGFILEBALL)" + sha256sum $1 + echo metahash for confirmation only $(sha256sum $1 |cut -d' ' -f1 | tr -d '\n' | sha256sum | cut -c1-6) ... + echo + sayhash $1 $SIGFILEBALL +} + +function oncesigned { # $1 <-- INPFILE ; $2 <--SIGFILE + SIGFILEBALL="$2.lzma.base64" + cat $SIGFILEBALL | tr -d '\r' | base64 -d | unlzma -c > $2 || exit 1 + if ! [ -f $2 ] ; then + echo "Failed to find $2"'!' + exit 1 + fi + + if file $2 | grep -qv " data" ; then + echo "WARNING WARNING $2 does not look like a binary signature:" + echo `file $2` + exit 1 + fi +} + +HERE=`dirname $0` +LEAUTO="`realpath $HERE`/../letsencrypt-auto-source/letsencrypt-auto" +SIGFILE="$LEAUTO".sig +offlinesign $LEAUTO $SIGFILE +oncesigned $LEAUTO $SIGFILE diff --git a/tools/release.sh b/tools/release.sh index 172f6fea1..6ec83053f 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -34,6 +34,9 @@ else echo Releasing developer version "$version"... fi +if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then + RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" +fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -80,18 +83,18 @@ git checkout "$RELEASE_BRANCH" SetVersion() { ver="$1" - for pkg_dir in $SUBPKGS + for pkg_dir in $SUBPKGS letsencrypt-compatibility-test do sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py + + # interactive user input + git add -p letsencrypt $SUBPKGS letsencrypt-compatibility-test - git add -p letsencrypt $SUBPKGS # interactive user input } + SetVersion "$version" -git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" -git tag --local-user "$RELEASE_GPG_KEY" \ - --sign --message "Release $version" "$tag" echo "Preparing sdists and wheels" for pkg_dir in . $SUBPKGS @@ -112,6 +115,7 @@ do cd - done + mkdir "dist.$version" mv dist "dist.$version/letsencrypt" for pkg_dir in $SUBPKGS @@ -129,8 +133,12 @@ virtualenv --no-site-packages ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip -# Now, use our local PyPI +# Now, use our local PyPI. Disable cache so we get the correct KGS even if we +# (or our dependencies) have conditional dependencies implemented with if +# statements in setup.py and we have cached wheels lying around that would +# cause those ifs to not be evaluated. pip install \ + --no-cache-dir \ --extra-index-url http://localhost:$PORT \ letsencrypt $SUBPKGS # stop local PyPI @@ -153,6 +161,21 @@ for module in letsencrypt $subpkgs_modules ; do done deactivate +# ensure we have the latest built version of leauto +letsencrypt-auto-source/build.py + +# and that it's signed correctly +while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ + letsencrypt-auto-source/letsencrypt-auto.sig \ + letsencrypt-auto-source/letsencrypt-auto ; do + read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" +done + +git add letsencrypt-auto-source +git diff --cached +git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" +git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" + cd .. echo Now in $PWD name=${root_without_le%.*} diff --git a/bootstrap/dev/venv.sh b/tools/venv.sh similarity index 72% rename from bootstrap/dev/venv.sh rename to tools/venv.sh index 11ab417dd..73c3bb110 100755 --- a/bootstrap/dev/venv.sh +++ b/tools/venv.sh @@ -3,9 +3,9 @@ export VENV_ARGS="--python python2" -./bootstrap/dev/_venv_common.sh \ - -e acme[testing] \ - -e .[dev,docs,testing] \ +./tools/_venv_common.sh \ + -e acme[dev] \ + -e .[dev,docs] \ -e letsencrypt-apache \ -e letsencrypt-nginx \ -e letshelp-letsencrypt \ diff --git a/bootstrap/dev/venv3.sh b/tools/venv3.sh similarity index 72% rename from bootstrap/dev/venv3.sh rename to tools/venv3.sh index ccffffb83..645ed0d47 100755 --- a/bootstrap/dev/venv3.sh +++ b/tools/venv3.sh @@ -4,5 +4,5 @@ export VENV_NAME="${VENV_NAME:-venv3}" export VENV_ARGS="--python python3" -./bootstrap/dev/_venv_common.sh \ - -e acme[testing] \ +./tools/_venv_common.sh \ + -e acme[dev] \ diff --git a/tox.ini b/tox.ini index c6cefb764..57359cd86 100644 --- a/tox.ini +++ b/tox.ini @@ -12,12 +12,12 @@ envlist = py{26,27,33,34,35},py{26,27}-oldest,cover,lint # loops, especially on Travis [testenv] -# packages installed separately to ensure that dowstream deps problems +# packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 commands = - pip install -e acme[testing] + pip install -e acme[dev] nosetests -v acme - pip install -e .[testing] + pip install -e .[dev] nosetests -v letsencrypt pip install -e letsencrypt-apache nosetests -v letsencrypt_apache @@ -40,23 +40,23 @@ deps = [testenv:py33] commands = - pip install -e acme[testing] + pip install -e acme[dev] nosetests -v acme [testenv:py34] commands = - pip install -e acme[testing] + pip install -e acme[dev] nosetests -v acme [testenv:py35] commands = - pip install -e acme[testing] + pip install -e acme[dev] nosetests -v acme [testenv:cover] basepython = python2.7 commands = - pip install -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt + pip install -e acme[dev] -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt ./tox.cover.sh [testenv:lint] @@ -66,7 +66,7 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt + pip install -e acme[dev] -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt ./pep8.travis.sh pylint --rcfile=.pylintrc letsencrypt pylint --rcfile=acme/.pylintrc acme/acme @@ -82,3 +82,13 @@ setenv = commands = pip install -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letsencrypt-compatibility-test -e letshelp-letsencrypt sudo ./letsencrypt-apache/letsencrypt_apache/tests/apache-conf-files/apache-conf-test --debian-modules + +[testenv:le_auto] +# At the moment, this tests under Python 2.7 only, as only that version is +# readily available on the Trusty Docker image. +commands = + docker build -t lea letsencrypt-auto-source + docker run --rm -t -i lea +whitelist_externals = + docker +passenv = DOCKER_* \ No newline at end of file