diff --git a/.travis.yml b/.travis.yml index 0d59ec9e1..cabd3cf73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,141 +18,163 @@ branches: - /^\d+\.\d+\.x$/ - /^test-.*$/ +# Jobs for the main test suite are always executed (including on PRs) except for pushes on master. +not-on-master: ¬-on-master + if: NOT (type = push AND branch = master) + +# Jobs for the extended test suite are executed for cron jobs and pushes on non-master branches. +extended-test-suite: &extended-test-suite + if: type = cron OR (type = push AND branch != master) + matrix: include: - # These environments are always executed + # Main test suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install sudo: required services: docker + <<: *not-on-master + + # This job is always executed, including on master - python: "2.7" env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - sudo: required env: TOXENV=nginx_compat services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=lint + <<: *not-on-master - python: "3.4" env: TOXENV=mypy + <<: *not-on-master - python: "3.5" env: TOXENV=mypy + <<: *not-on-master - python: "2.7" env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest' sudo: required services: docker + <<: *not-on-master - python: "3.4" env: TOXENV=py34 sudo: required services: docker + <<: *not-on-master - python: "3.7" dist: xenial env: TOXENV=py37 sudo: required services: docker + <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker before_install: addons: + <<: *not-on-master - sudo: required env: TOXENV=le_auto_trusty services: docker before_install: addons: + <<: *not-on-master - python: "2.7" env: TOXENV=apacheconftest-with-pebble sudo: required services: docker + <<: *not-on-master - python: "2.7" env: TOXENV=nginxroundtrip + <<: *not-on-master - # These environments are executed on cron events and commits to tested - # branches other than master. Which branches are tested is controlled by - # the "branches" section earlier in this file. + # Extended test suite on cron jobs and pushes to tested branches other than master - python: "3.7" dist: xenial env: TOXENV=py37 CERTBOT_NO_PIN=1 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "2.7" env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.4" env: TOXENV=py34 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.5" env: TOXENV=py35 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.6" env: TOXENV=py36 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v1 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - python: "3.7" dist: xenial env: TOXENV=py37 BOULDER_INTEGRATION=v2 sudo: required services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_xenial services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_jessie services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=le_auto_centos6 services: docker - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - sudo: required env: TOXENV=docker_dev services: docker @@ -160,7 +182,7 @@ matrix: apt: packages: # don't install nginx and apache - libaugeas0 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py27 os: osx @@ -169,7 +191,7 @@ matrix: packages: - augeas - python2 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite - language: generic env: TOXENV=py3 os: osx @@ -178,7 +200,7 @@ matrix: packages: - augeas - python3 - if: type = cron OR (type = push AND branch != master) + <<: *extended-test-suite # container-based infrastructure sudo: false @@ -197,10 +219,10 @@ addons: - nginx-light - openssl -install: "travis_retry $(command -v pip || command -v pip3) install codecov tox" +install: "$(command -v pip || command -v pip3) install codecov tox" script: - - travis_retry tox - - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' + - tox + - '[ -z "${BOULDER_INTEGRATION+x}" ] || (tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e484536..9d2130168 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,20 +2,60 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.32.0 - master +## 0.33.0 - master ### Added +* Fedora 29+ is now supported by certbot-auto. Since Python 2.x is on a deprecation + path in Fedora, certbot-auto will install and use Python 3.x on Fedora 29+. + +### Changed + * +### Fixed + +* Certbot uses the Python library cryptography for OCSP when cryptography>=2.5 + is installed. We fixed a bug in Certbot causing it to interpret timestamps in + the OCSP response as being in the local timezone rather than UTC. +* Issue causing the default CentOS 6 TLS configuration to ignore some of the HTTPS VirtualHosts created by Certbot. mod_ssl loading is now moved to main http.conf for this environment where possible. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* certbot +* certbot-apache + +More details about these changes can be found on our GitHub repo. + +## 0.32.0 - 2019-03-06 + +### Added + +* If possible, Certbot uses built-in support for OCSP from recent cryptography + versions instead of the OpenSSL binary: as a consequence Certbot does not need + the OpenSSL binary to be installed anymore if cryptography>=2.5 is installed. + ### Changed * Certbot and its acme module now depend on josepy>=1.1.0 to avoid printing the warnings described at https://github.com/certbot/josepy/issues/13. - -### Fixed - -* Issue causing the default CentOS 6 TLS configuration to ignore some of the HTTPS VirtualHosts created by Certbot. mod_ssl loading is now moved to main http.conf for this environment. +* Apache plugin now respects CERTBOT_DOCS environment variable when adding + command line defaults. +* The running of manual plugin hooks is now always included in Certbot's log + output. +* Tests execution for certbot, certbot-apache and certbot-nginx packages now relies on pytest. +* An ACME CA server may return a "Retry-After" HTTP header on authorization polling, as + specified in the ACME protocol, to indicate when the next polling should occur. Certbot now + reads this header if set and respect its value. +* The `acme` module avoids sending the `keyAuthorization` field in the JWS + payload when responding to a challenge as the field is not included in the + current ACME protocol. To ease the migration path for ACME CA servers, + Certbot and its `acme` module will first try the request without the + `keyAuthorization` field but will temporarily retry the request with the + field included if a `malformed` error is received. This fallback will be + removed in version 0.34.0. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only @@ -24,6 +64,7 @@ package with changes other than its version number was: * acme * certbot * certbot-apache +* certbot-nginx More details about these changes can be found on our GitHub repo. diff --git a/Dockerfile b/Dockerfile index 8f434e89c..828f5ec94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,13 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ + +# Generate constraints file to pin dependency versions COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . +COPY tools /opt/certbot/tools +RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt' +RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt' + COPY acme src/acme COPY certbot src/certbot @@ -23,7 +29,7 @@ RUN apk add --no-cache --virtual .build-deps \ musl-dev \ libffi-dev \ && pip install -r /opt/certbot/dependency-requirements.txt \ - && pip install --no-cache-dir \ + && pip install --no-cache-dir --no-deps \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ && apk del .build-deps diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 501f74881..6f2b3757b 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -108,6 +108,10 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): key_authorization = jose.Field("keyAuthorization") thumbprint_hash_function = hashes.SHA256 + def __init__(self, *args, **kwargs): + super(KeyAuthorizationChallengeResponse, self).__init__(*args, **kwargs) + self._dump_authorization_key(False) + def verify(self, chall, account_public_key): """Verify the key authorization. @@ -140,6 +144,22 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True + def _dump_authorization_key(self, dump): + # type: (bool) -> None + """ + Set if keyAuthorization is dumped in the JSON representation of this ChallengeResponse. + NB: This method is declared as private because it will eventually be removed. + :param bool dump: True to dump the keyAuthorization, False otherwise + """ + object.__setattr__(self, '_dump_auth_key', dump) + + def to_partial_json(self): + jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + if not self._dump_auth_key: # pylint: disable=no-member + jobj.pop('keyAuthorization', None) + + return jobj + @six.add_metaclass(abc.ABCMeta) class KeyAuthorizationChallenge(_TokenChallenge): diff --git a/acme/acme/challenges_test.py b/acme/acme/challenges_test.py index 81d39058e..4b905c1e5 100644 --- a/acme/acme/challenges_test.py +++ b/acme/acme/challenges_test.py @@ -94,6 +94,9 @@ class DNS01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -165,6 +168,9 @@ class HTTP01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): @@ -285,6 +291,9 @@ class TLSSNI01ResponseTest(unittest.TestCase): self.assertEqual(self.z_domain, self.response.z_domain) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.response.to_partial_json()) + self.response._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.response.to_partial_json()) def test_from_json(self): @@ -419,6 +428,9 @@ class TLSALPN01ResponseTest(unittest.TestCase): self.response = self.chall.response(KEY) def test_to_partial_json(self): + self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, + self.msg.to_partial_json()) + self.msg._dump_authorization_key(True) # pylint: disable=protected-access self.assertEqual(self.jmsg, self.msg.to_partial_json()) def test_from_json(self): diff --git a/acme/acme/client.py b/acme/acme/client.py index 76b6a7788..d74713a7e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -17,6 +17,7 @@ import requests from requests.adapters import HTTPAdapter import sys +from acme import challenges from acme import crypto_util from acme import errors from acme import jws @@ -155,7 +156,23 @@ class ClientBase(object): # pylint: disable=too-many-instance-attributes :raises .UnexpectedUpdate: """ - response = self._post(challb.uri, response) + # Because sending keyAuthorization in a response challenge has been removed from the ACME + # spec, it is not included in the KeyAuthorizationResponseChallenge JSON by default. + # However as a migration path, we temporarily expect a malformed error from the server, + # and fallback by resending the challenge response with the keyAuthorization field. + # TODO: Remove this fallback for Certbot 0.34.0 + try: + response = self._post(challb.uri, response) + except messages.Error as error: + if (error.code == 'malformed' + and isinstance(response, challenges.KeyAuthorizationChallengeResponse)): + logger.debug('Error while responding to a challenge without keyAuthorization ' + 'in the JWS, your ACME CA server may not support it:\n%s', error) + logger.debug('Retrying request with keyAuthorization set.') + response._dump_authorization_key(True) # pylint: disable=protected-access + response = self._post(challb.uri, response) + else: + raise try: authzr_uri = response.links['up']['url'] except KeyError: @@ -781,7 +798,7 @@ class ClientV2(ClientBase): except messages.Error as error: if error.code == 'malformed': logger.debug('Error during a POST-as-GET request, ' - 'your ACME CA may not support it:\n%s', error) + 'your ACME CA server may not support it:\n%s', error) logger.debug('Retrying request with GET.') else: # pragma: no cover raise @@ -1191,10 +1208,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, acme_version=1, **kwargs): - try: - new_nonce_url = kwargs.pop('new_nonce_url') - except KeyError: - new_nonce_url = None + new_nonce_url = kwargs.pop('new_nonce_url', None) data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version) kwargs.setdefault('headers', {'Content-Type': content_type}) response = self._send_request('POST', url, data=data, **kwargs) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index b3d0f1921..dc8cc0c93 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -463,6 +463,34 @@ class ClientTest(ClientTestBase): errors.ClientError, self.client.answer_challenge, self.challr.body, challenges.DNSResponse(validation=None)) + def test_answer_challenge_key_authorization_fallback(self): + self.response.links['up'] = {'url': self.challr.authzr_uri} + self.response.json.return_value = self.challr.body.to_json() + + def _wrapper_post(url, obj, *args, **kwargs): # pylint: disable=unused-argument + """ + Simulate an old ACME CA server, that would respond a 'malformed' + error if keyAuthorization is missing. + """ + jobj = obj.to_partial_json() + if 'keyAuthorization' not in jobj: + raise messages.Error.with_code('malformed') + return self.response + self.net.post.side_effect = _wrapper_post + + # This challenge response is of type KeyAuthorizationChallengeResponse, so the fallback + # should be triggered, and avoid an exception. + http_chall_response = challenges.HTTP01Response(key_authorization='test', + resource=mock.MagicMock()) + self.client.answer_challenge(self.challr.body, http_chall_response) + + # This challenge response is not of type KeyAuthorizationChallengeResponse, so the fallback + # should not be triggered, leading to an exception. + dns_chall_response = challenges.DNSResponse(validation=None) + self.assertRaises( + errors.Error, self.client.answer_challenge, + self.challr.body, dns_chall_response) + def test_retry_after_date(self): self.response.headers['Retry-After'] = 'Fri, 31 Dec 1999 23:59:59 GMT' self.assertEqual( diff --git a/acme/examples/http01_example.py b/acme/examples/http01_example.py new file mode 100644 index 000000000..79508f1b4 --- /dev/null +++ b/acme/examples/http01_example.py @@ -0,0 +1,240 @@ +"""Example ACME-V2 API for HTTP-01 challenge. + +Brief: + +This a complete usage example of the python-acme API. + +Limitations of this example: + - Works for only one Domain name + - Performs only HTTP-01 challenge + - Uses ACME-v2 + +Workflow: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account +""" +from contextlib import contextmanager +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +import OpenSSL + +from acme import challenges +from acme import client +from acme import crypto_util +from acme import errors +from acme import messages +from acme import standalone +import josepy as jose + +# Constants: + +# This is the staging point for ACME-V2 within Let's Encrypt. +DIRECTORY_URL = 'https://acme-staging-v02.api.letsencrypt.org/directory' + +USER_AGENT = 'python-acme-example' + +# Account key size +ACC_KEY_BITS = 2048 + +# Certificate private key size +CERT_PKEY_BITS = 2048 + +# Domain name for the certificate. +DOMAIN = 'client.example.com' + +# If you are running Boulder locally, it is possible to configure any port +# number to execute the challenge, but real CA servers will always use port +# 80, as described in the ACME specification. +PORT = 80 + + +# Useful methods and classes: + + +def new_csr_comp(domain_name, pkey_pem=None): + """Create certificate signing request.""" + if pkey_pem is None: + # Create private key. + pkey = OpenSSL.crypto.PKey() + pkey.generate_key(OpenSSL.crypto.TYPE_RSA, CERT_PKEY_BITS) + pkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, + pkey) + csr_pem = crypto_util.make_csr(pkey_pem, [domain_name]) + return pkey_pem, csr_pem + + +def select_http01_chall(orderr): + """Extract authorization resource from within order resource.""" + # Authorization Resource: authz. + # This object holds the offered challenges by the server and their status. + authz_list = orderr.authorizations + + for authz in authz_list: + # Choosing challenge. + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + return i + + raise Exception('HTTP-01 challenge was not offered by the CA server.') + + +@contextmanager +def challenge_server(http_01_resources): + """Manage standalone server set up and shutdown.""" + + # Setting up a fake server that binds at PORT and any address. + address = ('', PORT) + try: + servers = standalone.HTTP01DualNetworkedServers(address, + http_01_resources) + # Start client standalone web server. + servers.serve_forever() + yield servers + finally: + # Shutdown client web server and unbind from PORT + servers.shutdown_and_server_close() + + +def perform_http01(client_acme, challb, orderr): + """Set up standalone webserver and perform HTTP-01 challenge.""" + + response, validation = challb.response_and_validation(client_acme.net.key) + + resource = standalone.HTTP01RequestHandler.HTTP01Resource( + chall=challb.chall, response=response, validation=validation) + + with challenge_server({resource}): + # Let the CA server know that we are ready for the challenge. + client_acme.answer_challenge(challb, response) + + # Wait for challenge status and then issue a certificate. + # It is possible to set a deadline time. + finalized_orderr = client_acme.poll_and_finalize(orderr) + + return finalized_orderr.fullchain_pem + + +# Main examples: + + +def example_http(): + """This example executes the whole process of fulfilling a HTTP-01 + challenge for one specific domain. + + The workflow consists of: + (Account creation) + - Create account key + - Register account and accept TOS + (Certificate actions) + - Select HTTP-01 within offered challenges by the CA server + - Set up http challenge resource + - Set up standalone web server + - Create domain private key and CSR + - Issue certificate + - Renew certificate + - Revoke certificate + (Account update actions) + - Change contact information + - Deactivate Account + + """ + # Create account key + + acc_key = jose.JWKRSA( + key=rsa.generate_private_key(public_exponent=65537, + key_size=ACC_KEY_BITS, + backend=default_backend())) + + # Register account and accept TOS + + net = client.ClientNetwork(acc_key, user_agent=USER_AGENT) + directory = messages.Directory.from_json(net.get(DIRECTORY_URL).json()) + client_acme = client.ClientV2(directory, net=net) + + # Terms of Service URL is in client_acme.directory.meta.terms_of_service + # Registration Resource: regr + # Creates account with contact information. + email = ('fake@example.com') + regr = client_acme.new_account( + messages.NewRegistration.from_data( + email=email, terms_of_service_agreed=True)) + + # Create domain private key and CSR + pkey_pem, csr_pem = new_csr_comp(DOMAIN) + + # Issue certificate + + orderr = client_acme.new_order(csr_pem) + + # Select HTTP-01 within offered challenges by the CA server + challb = select_http01_chall(orderr) + + # The certificate is ready to be used in the variable "fullchain_pem". + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Renew certificate + + _, csr_pem = new_csr_comp(DOMAIN, pkey_pem) + + orderr = client_acme.new_order(csr_pem) + + challb = select_http01_chall(orderr) + + # Performing challenge + fullchain_pem = perform_http01(client_acme, challb, orderr) + + # Revoke certificate + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, fullchain_pem)) + + try: + client_acme.revoke(fullchain_com, 0) # revocation reason = 0 + except errors.ConflictError: + # Certificate already revoked. + pass + + # Query registration status. + client_acme.net.account = regr + try: + regr = client_acme.query_registration(regr) + except errors.Error as err: + if err.typ == messages.OLD_ERROR_PREFIX + 'unauthorized' \ + or err.typ == messages.ERROR_PREFIX + 'unauthorized': + # Status is deactivated. + pass + raise + + # Change contact information + + email = 'newfake@example.com' + regr = client_acme.update_registration( + regr.update( + body=regr.body.update( + contact=('mailto:' + email,) + ) + ) + ) + + # Deactivate account/registration + + regr = client_acme.deactivate_registration(regr) + + +if __name__ == "__main__": + example_http() diff --git a/acme/setup.py b/acme/setup.py index 79d6d3389..cd4ea3ef5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -36,6 +36,7 @@ docs_extras = [ 'sphinx_rtd_theme', ] + class PyTest(TestCommand): user_options = [] @@ -50,6 +51,7 @@ class PyTest(TestCommand): errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) + setup( name='acme', version=version, @@ -82,7 +84,7 @@ setup( 'dev': dev_extras, 'docs': docs_extras, }, - tests_require=["pytest"], test_suite='acme', + tests_require=["pytest"], cmdclass={"test": PyTest}, ) diff --git a/appveyor.yml b/appveyor.yml index 2b6b82747..12d882973 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,6 +11,14 @@ branches: - /^\d+\.\d+\.x$/ # Version branches like X.X.X - /^test-.*$/ +init: + # Since master can receive only commits from PR that have already been tested, following + # condition avoid to launch all jobs except the coverage one for commits pushed to master. + - ps: | + if (-Not $Env:APPVEYOR_PULL_REQUEST_NUMBER -And $Env:APPVEYOR_REPO_BRANCH -Eq 'master' ` + -And -Not ($Env:TOXENV -Like '*-cover')) + { $Env:APPVEYOR_SKIP_FINALIZE_ON_EXIT = 'true'; Exit-AppVeyorBuild } + install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index efd766e63..f7f4ff925 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -92,6 +92,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ description = "Apache Web Server plugin" + if os.environ.get("CERTBOT_DOCS") == "1": + description += ( # pragma: no cover + " (Please note that the default values of the Apache plugin options" + " change depending on the operating system Certbot is run on.)" + ) OS_DEFAULTS = dict( server_root="/etc/apache2", @@ -141,28 +146,36 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # When adding, modifying or deleting command line arguments, be sure to # include the changes in the list used in method _prepare_options() to # ensure consistent behavior. - add("enmod", default=cls.OS_DEFAULTS["enmod"], + + # Respect CERTBOT_DOCS environment variable and use default values from + # base class regardless of the underlying distribution (overrides). + if os.environ.get("CERTBOT_DOCS") == "1": + DEFAULTS = ApacheConfigurator.OS_DEFAULTS + else: + # cls.OS_DEFAULTS can be distribution specific, see override classes + DEFAULTS = cls.OS_DEFAULTS + add("enmod", default=DEFAULTS["enmod"], help="Path to the Apache 'a2enmod' binary") - add("dismod", default=cls.OS_DEFAULTS["dismod"], + add("dismod", default=DEFAULTS["dismod"], help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], help="SSL vhost configuration extension") - add("server-root", default=cls.OS_DEFAULTS["server_root"], + add("server-root", default=DEFAULTS["server_root"], help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=cls.OS_DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS["logs_root"], help="Apache server logs directory") add("challenge-location", - default=cls.OS_DEFAULTS["challenge_location"], + default=DEFAULTS["challenge_location"], help="Directory path for challenge configuration") - add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS["handle_modules"], help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS["handle_sites"], help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=cls.OS_DEFAULTS["ctl"], + add("ctl", default=DEFAULTS["ctl"], help="Full path to Apache control script") util.add_deprecated_argument( add, argument_name="init-script", nargs=1) diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf index 0918e5669..dbfae3765 100644 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/finalize-1243.conf @@ -1,7 +1,7 @@ #LoadModule ssl_module modules/mod_ssl.so -Listen 443 - +Listen 4443 + # The ServerName directive sets the request scheme, hostname and port that # the server uses to identify itself. This is used when creating # redirection URLs. In the context of virtual hosts, the ServerName diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index ff93682b6..7c281d707 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -115,6 +115,37 @@ class MultipleVhostsTest(util.ApacheTest): # Weak test.. ApacheConfigurator.add_parser_arguments(mock.MagicMock()) + def test_docs_parser_arguments(self): + os.environ["CERTBOT_DOCS"] = "1" + from certbot_apache.configurator import ApacheConfigurator + mock_add = mock.MagicMock() + ApacheConfigurator.add_parser_arguments(mock_add) + parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", + "vhost_root", "logs_root", "challenge_location", + "handle_modules", "handle_sites", "ctl"] + exp = dict() + + for k in ApacheConfigurator.OS_DEFAULTS: + if k in parserargs: + exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + # Special cases + exp["vhost-root"] = None + exp["init-script"] = None + + found = set() + for call in mock_add.call_args_list: + # init-script is a special case: deprecated argument + if call[0][0] != "init-script": + self.assertEqual(exp[call[0][0]], call[1]['default']) + found.add(call[0][0]) + + # Make sure that all (and only) the expected values exist + self.assertEqual(len(mock_add.call_args_list), len(found)) + for e in exp: + self.assertTrue(e in found) + + del os.environ["CERTBOT_DOCS"] + def test_add_parser_arguments_all_configurators(self): # pylint: disable=no-self-use from certbot_apache.entrypoint import OVERRIDE_CLASSES for cls in OVERRIDE_CLASSES.values(): diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 52528a536..dfedc3f0d 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,8 +1,10 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-apache', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_apache', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot-auto b/certbot-auto index 03c2994e3..0c82a7437 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/.coveragerc b/certbot-ci/certbot_integration_tests/.coveragerc new file mode 100644 index 000000000..de36d4e02 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/.coveragerc @@ -0,0 +1,8 @@ +[run] +# Avoid false warnings because certbot packages are not installed in the thread that executes +# the coverage: indeed, certbot is launched as a CLI from a subprocess. +disable_warnings = module-not-imported,no-data-collected + +[report] +# Exclude unit tests in coverage during integration tests. +omit = **/*_test.py,**/tests/*,**/certbot_nginx/parser_obj.py diff --git a/certbot-ci/certbot_integration_tests/__init__.py b/certbot-ci/certbot_integration_tests/__init__.py new file mode 100644 index 000000000..434a85a23 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/__init__.py @@ -0,0 +1 @@ +"""Package certbot_integration_test is for tests that require a live acme ca server instance""" diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py new file mode 100644 index 000000000..9045cd37d --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -0,0 +1,16 @@ +class IntegrationTestsContext(object): + """General fixture describing a certbot integration tests context""" + def __init__(self, request): + self.request = request + if hasattr(request.config, 'slaveinput'): # Worker node + self.worker_id = request.config.slaveinput['slaveid'] + self.acme_xdist = request.config.slaveinput['acme_xdist'] + else: # Primary node + self.worker_id = 'primary' + self.acme_xdist = request.config.acme_xdist + self.directory_url = self.acme_xdist['directory_url'] + self.tls_alpn_01_port = self.acme_xdist['https_port'][self.worker_id] + self.http_01_port = self.acme_xdist['http_port'][self.worker_id] + + def cleanup(self): + pass diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py new file mode 100644 index 000000000..5b0981b36 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -0,0 +1,40 @@ +import requests +import urllib3 + +import pytest + +from certbot_integration_tests.certbot_tests import context as certbot_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = certbot_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello_1(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass + + +def test_hello_2(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py new file mode 100644 index 000000000..892c16266 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -0,0 +1,92 @@ +""" +General conftest for pytest execution of all integration tests lying +in the certbot_integration tests package. +As stated by pytest documentation, conftest module is used to set on +for a directory a specific configuration using built-in pytest hooks. + +See https://docs.pytest.org/en/latest/reference.html#hook-reference +""" +import contextlib +import sys +import subprocess + +from certbot_integration_tests.utils import acme_server as acme_lib + + +def pytest_addoption(parser): + """ + Standard pytest hook to add options to the pytest parser. + :param parser: current pytest parser that will be used on the CLI + """ + parser.addoption('--acme-server', default='pebble', + choices=['boulder-v1', 'boulder-v2', 'pebble'], + help='select the ACME server to use (boulder-v1, boulder-v2, ' + 'pebble), defaulting to pebble') + + +def pytest_configure(config): + """ + Standard pytest hook used to add a configuration logic for each node of a pytest run. + :param config: the current pytest configuration + """ + if not hasattr(config, 'slaveinput'): # If true, this is the primary node + with _print_on_err(): + config.acme_xdist = _setup_primary_node(config) + + +def pytest_configure_node(node): + """ + Standard pytest-xdist hook used to configure a worker node. + :param node: current worker node + """ + node.slaveinput['acme_xdist'] = node.config.acme_xdist + + +@contextlib.contextmanager +def _print_on_err(): + """ + During pytest-xdist setup, stdout is used for nodes communication, so print is useless. + However, stderr is still available. This context manager transfers stdout to stderr + for the duration of the context, allowing to display prints to the user. + """ + old_stdout = sys.stdout + sys.stdout = sys.stderr + try: + yield + finally: + sys.stdout = old_stdout + + +def _setup_primary_node(config): + """ + Setup the environment for integration tests. + Will: + - check runtime compatiblity (Docker, docker-compose, Nginx) + - create a temporary workspace and the persistent GIT repositories space + - configure and start paralleled ACME CA servers using Docker + - transfer ACME CA servers configurations to pytest nodes using env variables + :param config: Configuration of the pytest primary node + """ + # Check for runtime compatibility: some tools are required to be available in PATH + try: + subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + try: + subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + # Parameter numprocesses is added to option by pytest-xdist + workers = ['primary'] if not config.option.numprocesses\ + else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] + + # By calling setup_acme_server we ensure that all necessary acme server instances will be + # fully started. This runtime is reflected by the acme_xdist returned. + acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers) + print('ACME xdist config:\n{0}'.format(acme_xdist)) + + return acme_xdist diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py b/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py new file mode 100644 index 000000000..6d7d6012b --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -0,0 +1,5 @@ +from certbot_integration_tests.certbot_tests import context as certbot_context + + +class IntegrationTestsContext(certbot_context.IntegrationTestsContext): + """General fixture describing a certbot-nginx integration tests context""" diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py new file mode 100644 index 000000000..472e5e7b7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -0,0 +1,17 @@ +import pytest + +from certbot_integration_tests.nginx_tests import context as nginx_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = nginx_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello(context): + print(context.directory_url) diff --git a/certbot-ci/certbot_integration_tests/utils/__init__.py b/certbot-ci/certbot_integration_tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py new file mode 100644 index 000000000..33ef05194 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -0,0 +1,194 @@ +"""Module to setup an ACME CA server environment able to run multiple tests in parallel""" +from __future__ import print_function +import tempfile +import atexit +import os +import subprocess +import shutil +import sys +from os.path import join + +import requests +import json +import yaml + +from certbot_integration_tests.utils import misc + +# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble. +CHALLTESTSRV_PORT = 8055 +HTTP_01_PORT = 5002 + + +def setup_acme_server(acme_server, nodes): + """ + This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel + execution of integration tests against the unique http-01 port expected by the ACME CA server. + Instances are properly closed and cleaned when the Python process exits using atexit. + Typically all pytest integration tests will be executed in this context. + This method returns an object describing ports and directory url to use for each pytest node + with the relevant pytest xdist node. + :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) + :param str[] nodes: list of node names that will be setup by pytest xdist + :return: a dict describing the challenge ports that have been setup for the nodes + :rtype: dict + """ + acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' + acme_xdist = _construct_acme_xdist(acme_server, nodes) + workspace = _construct_workspace(acme_type) + + _prepare_traefik_proxy(workspace, acme_xdist) + _prepare_acme_server(workspace, acme_type, acme_xdist) + + return acme_xdist + + +def _construct_acme_xdist(acme_server, nodes): + """Generate and return the acme_xdist dict""" + acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} + + # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + if acme_server == 'pebble': + acme_xdist['directory_url'] = 'https://localhost:14000/dir' + else: # boulder + port = 4001 if acme_server == 'boulder-v2' else 4000 + acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port) + + acme_xdist['http_port'] = {node: port for (node, port) + in zip(nodes, range(5200, 5200 + len(nodes)))} + acme_xdist['https_port'] = {node: port for (node, port) + in zip(nodes, range(5100, 5100 + len(nodes)))} + + return acme_xdist + + +def _construct_workspace(acme_type): + """Create a temporary workspace for integration tests stack""" + workspace = tempfile.mkdtemp() + + def cleanup(): + """Cleanup function to call that will teardown relevant dockers and their configuration.""" + for instance in [acme_type, 'traefik']: + print('=> Tear down the {0} instance...'.format(instance)) + instance_path = join(workspace, instance) + try: + if os.path.isfile(join(instance_path, 'docker-compose.yml')): + _launch_command(['docker-compose', 'down'], cwd=instance_path) + except subprocess.CalledProcessError: + pass + print('=> Finished tear down of {0} instance.'.format(acme_type)) + + shutil.rmtree(workspace) + + # Here with atexit we ensure that clean function is called no matter what. + atexit.register(cleanup) + + return workspace + + +def _prepare_acme_server(workspace, acme_type, acme_xdist): + """Configure and launch the ACME server, Boulder or Pebble""" + print('=> Starting {0} instance deployment...'.format(acme_type)) + instance_path = join(workspace, acme_type) + try: + # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production. + _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type), + '--single-branch', '--depth=1', instance_path]) + if acme_type == 'boulder': + # Allow Boulder to ignore usual limit rate policies, useful for tests. + os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), + join(instance_path, 'test/rate-limit-policies.yml')) + if acme_type == 'pebble': + # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid + # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. + with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler: + config = yaml.load(file_handler.read()) + + config['services']['pebble'].setdefault('environment', [])\ + .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0']) + with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler: + file_handler.write(yaml.dump(config)) + + # Launch the ACME CA server. + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + # Wait for the ACME CA server to be up. + print('=> Waiting for {0} instance to respond...'.format(acme_type)) + misc.check_until_timeout(acme_xdist['directory_url']) + + # Configure challtestsrv to answer any A record request with ip of the docker host. + acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50' + response = requests.post('http://localhost:{0}/set-default-ipv4' + .format(acme_xdist['challtestsrv_port']), + json={'ip': '{0}.1'.format(acme_subnet)}) + response.raise_for_status() + + print('=> Finished {0} instance deployment.'.format(acme_type)) + except BaseException: + print('Error while setting up {0} instance.'.format(acme_type)) + raise + + +def _prepare_traefik_proxy(workspace, acme_xdist): + """Configure and launch Traefik, the HTTP reverse proxy""" + print('=> Starting traefik instance deployment...') + instance_path = join(workspace, 'traefik') + traefik_subnet = '10.33.33' + traefik_api_port = 8056 + try: + os.mkdir(instance_path) + + with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h: + file_h.write('''\ +version: '3' +services: + traefik: + image: traefik + command: --api --rest + ports: + - {http_01_port}:80 + - {traefik_api_port}:8080 + networks: + traefiknet: + ipv4_address: {traefik_subnet}.2 +networks: + traefiknet: + ipam: + config: + - subnet: {traefik_subnet}.0/24 +'''.format(traefik_subnet=traefik_subnet, + traefik_api_port=traefik_api_port, + http_01_port=HTTP_01_PORT)) + + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port)) + config = { + 'backends': { + node: { + 'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}} + } for node, port in acme_xdist['http_port'].items() + }, + 'frontends': { + node: { + 'backend': node, 'passHostHeader': True, + 'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}} + } for node in acme_xdist['http_port'].keys() + } + } + response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port), + data=json.dumps(config)) + response.raise_for_status() + + print('=> Finished traefik instance deployment.') + except BaseException: + print('Error while setting up traefik instance.') + raise + + +def _launch_command(command, cwd=os.getcwd()): + """Launch silently an OS command, output will be displayed in case of failure""" + try: + subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True) + except subprocess.CalledProcessError as e: + sys.stderr.write(e.output) + raise diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py new file mode 100644 index 000000000..a3b134788 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -0,0 +1,45 @@ +""" +Misc module contains stateless functions that could be used during pytest execution, +or outside during setup/teardown of the integration tests environment. +""" +import os +import time +import contextlib + +import requests + + +def check_until_timeout(url): + """ + Wait and block until given url responds with status 200, or raise an exception + after 150 attempts. + :param str url: the URL to test + :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL + """ + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + for _ in range(0, 150): + time.sleep(1) + try: + if requests.get(url, verify=False).status_code == 200: + return + except requests.exceptions.ConnectionError: + pass + + raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url)) + + +@contextlib.contextmanager +def execute_in_given_cwd(cwd): + """ + Context manager that will execute any command in the given cwd after entering context, + and restore current cwd when context is destroyed. + :param str cwd: the path to use as the temporary current workspace for python execution + """ + current_cwd = os.getcwd() + try: + os.chdir(cwd) + yield + finally: + os.chdir(current_cwd) diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py new file mode 100644 index 000000000..595bba69e --- /dev/null +++ b/certbot-ci/setup.py @@ -0,0 +1,45 @@ +from setuptools import setup +from setuptools import find_packages + + +version = '0.32.0.dev0' + +install_requires = [ + 'pytest', + 'pytest-cov', + 'pytest-xdist', + 'pytest-sugar', + 'coverage', + 'requests', + 'pyyaml', +] + +setup( + name='certbot-ci', + version=version, + description="Certbot continuous integration framework", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, +) diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e1c82b063..858cef831 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile index 27dcc8751..adbf715fa 100644 --- a/certbot-dns-cloudflare/Dockerfile +++ b/certbot-dns-cloudflare/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudflare -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index d6a16f468..84414e0c0 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile index cc84ea65b..48c88c35c 100644 --- a/certbot-dns-cloudxns/Dockerfile +++ b/certbot-dns-cloudxns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-cloudxns -RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index f0f2446e1..4b2fe15be 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile index 8bdd0619f..342e0e876 100644 --- a/certbot-dns-digitalocean/Dockerfile +++ b/certbot-dns-digitalocean/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-digitalocean -RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index c61f05339..1de51f7ca 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile index 38d2be80e..724675339 100644 --- a/certbot-dns-dnsimple/Dockerfile +++ b/certbot-dns-dnsimple/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsimple -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 7d39a10d0..113e55e12 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile index ff7936925..1480baf4f 100644 --- a/certbot-dns-dnsmadeeasy/Dockerfile +++ b/certbot-dns-dnsmadeeasy/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-dnsmadeeasy -RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 8a5f234b5..6e882a4f7 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile index 48ad902b5..7dce0e521 100644 --- a/certbot-dns-gehirn/Dockerfile +++ b/certbot-dns-gehirn/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-gehirn -RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 67ae1d4eb..bdbd198d1 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile index 4a258d0ee..5750b31d9 100644 --- a/certbot-dns-google/Dockerfile +++ b/certbot-dns-google/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-google -RUN pip install --no-cache-dir --editable src/certbot-dns-google +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 429702d75..c2556797d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile index 2e237b521..6db8b59fb 100644 --- a/certbot-dns-linode/Dockerfile +++ b/certbot-dns-linode/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-linode -RUN pip install --no-cache-dir --editable src/certbot-dns-linode +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 6797501b9..c494d6d4e 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile index 6efb4d777..efc9f36d6 100644 --- a/certbot-dns-luadns/Dockerfile +++ b/certbot-dns-luadns/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-luadns -RUN pip install --no-cache-dir --editable src/certbot-dns-luadns +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 07d9decdf..4c57c5709 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile index 88fc13c57..de541e850 100644 --- a/certbot-dns-nsone/Dockerfile +++ b/certbot-dns-nsone/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-nsone -RUN pip install --no-cache-dir --editable src/certbot-dns-nsone +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 314b0a16c..9c9f233cc 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile index e8da96d95..37e488dc4 100644 --- a/certbot-dns-ovh/Dockerfile +++ b/certbot-dns-ovh/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-ovh -RUN pip install --no-cache-dir --editable src/certbot-dns-ovh +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index a878eac47..cf6fee5d7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile index 1b8feb2f8..3ebb6a72e 100644 --- a/certbot-dns-rfc2136/Dockerfile +++ b/certbot-dns-rfc2136/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-rfc2136 -RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-rfc2136 diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 39f35e953..dacf41101 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile index a1b8d6caf..e1825c11d 100644 --- a/certbot-dns-route53/Dockerfile +++ b/certbot-dns-route53/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-route53 -RUN pip install --no-cache-dir --editable src/certbot-dns-route53 +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-route53 diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e915c6b86..321ad3b5d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile index 694773f61..9fa9b3c22 100644 --- a/certbot-dns-sakuracloud/Dockerfile +++ b/certbot-dns-sakuracloud/Dockerfile @@ -2,4 +2,4 @@ FROM certbot/certbot COPY . src/certbot-dns-sakuracloud -RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud +RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-sakuracloud diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 429f960f7..46d92a6ff 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index dd0bf9e8b..ffe1ddac7 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -13,6 +13,7 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util +from certbot import compat from certbot import constants as core_constants from certbot import crypto_util from certbot import errors @@ -164,9 +165,7 @@ class NginxConfigurator(common.Installer): util.lock_dir_until_exit(self.conf('server-root')) except (OSError, errors.LockError): logger.debug('Encountered error:', exc_info=True) - raise errors.PluginError( - 'Unable to lock %s', self.conf('server-root')) - + raise errors.PluginError('Unable to lock {0}'.format(self.conf('server-root'))) # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, @@ -899,7 +898,7 @@ class NginxConfigurator(common.Installer): have permissions of root. """ - uid = os.geteuid() + uid = compat.os_geteuid() util.make_or_verify_dir( self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) util.make_or_verify_dir( diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 622eb8d55..c5f780d94 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -81,9 +81,9 @@ class NginxParser(object): """ if not os.path.isabs(path): - return os.path.join(self.root, path) + return os.path.normpath(os.path.join(self.root, path)) else: - return path + return os.path.normpath(path) def _build_addr_to_ssl(self): """Builds a map from address to whether it listens on ssl in any server block diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 957588e2a..706e2637a 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -1,7 +1,6 @@ # pylint: disable=too-many-public-methods """Test for certbot_nginx.configurator.""" import os -import shutil import unittest import mock @@ -33,12 +32,6 @@ class NginxConfiguratorTest(util.NginxTest): self.config = util.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - shutil.rmtree(self.logs_dir) - @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): mock_exe_exists.return_value = False @@ -69,8 +62,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare_locked(self): server_root = self.config.conf("server-root") + + from certbot import util as certbot_util + certbot_util._LOCKS[server_root].release() # pylint: disable=protected-access + self.config.config_test = mock.Mock() - os.remove(os.path.join(server_root, ".certbot.lock")) certbot_test_util.lock_and_call(self._test_prepare_locked, server_root) @mock.patch("certbot_nginx.configurator.util.exe_exists") @@ -88,11 +84,11 @@ class NginxConfiguratorTest(util.NginxTest): def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) names = self.config.get_all_names() - self.assertEqual(names, set( - ["155.225.50.69.nephoscale.net", "www.example.org", "another.alias", + self.assertEqual(names, { + "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com"])) + "headers.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -171,6 +167,7 @@ class NginxConfiguratorTest(util.NginxTest): 'abc.www.foo.com': "etc_nginx/foo.conf", 'www.bar.co.uk': "etc_nginx/nginx.conf", 'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"} + conf_path = {key: os.path.normpath(value) for key, value in conf_path.items()} vhost = self.config.choose_vhosts(name)[0] path = os.path.relpath(vhost.filep, self.temp_dir) diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py index ed3c257ee..41c4b95fc 100644 --- a/certbot-nginx/certbot_nginx/tests/http_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.http_01""" import unittest -import shutil import mock import six @@ -54,11 +53,6 @@ class HttpPerformTest(util.NginxTest): from certbot_nginx import http_01 self.http01 = http_01.NginxHttp01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - def test_perform0(self): responses = self.http01.perform() self.assertEqual([], responses) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index f6f28e42b..3a68f7f24 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -67,9 +67,15 @@ class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods def test_abs_path(self): nparser = parser.NginxParser(self.config_path) - self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) - self.assertEqual(os.path.join(self.config_path, 'foo/bar/'), - nparser.abs_path('foo/bar/')) + if os.name != 'nt': + self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) + self.assertEqual(os.path.join(self.config_path, 'foo/bar'), + nparser.abs_path('foo/bar')) + else: + self.assertEqual('C:\\etc\\nginx\\*', nparser.abs_path('C:\\etc\\nginx\\*')) + self.assertEqual(os.path.join(self.config_path, 'foo\\bar'), + nparser.abs_path('foo\\bar')) + def test_filedump(self): nparser = parser.NginxParser(self.config_path) diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 72b65911c..62ca085ef 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -1,6 +1,5 @@ """Tests for certbot_nginx.tls_sni_01""" import unittest -import shutil import mock import six @@ -55,11 +54,6 @@ class TlsSniPerformTest(util.NginxTest): from certbot_nginx import tls_sni_01 self.sni = tls_sni_01.NginxTlsSni01(config) - def tearDown(self): - shutil.rmtree(self.temp_dir) - shutil.rmtree(self.config_dir) - shutil.rmtree(self.work_dir) - @mock.patch("certbot_nginx.configurator" ".NginxConfigurator.choose_vhosts") def test_perform(self, mock_choose): diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index ad1af2b96..ef669dac0 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -4,6 +4,8 @@ import os import pkg_resources import tempfile import unittest +import shutil +import warnings import josepy as jose import mock @@ -33,6 +35,22 @@ class NginxTest(unittest.TestCase): # pylint: disable=too-few-public-methods self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( "rsa512_key.pem")) + def tearDown(self): + # On Windows we have various files which are not correctly closed at the time of tearDown. + # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. + def onerror_handler(_, path, excinfo): + """On error handler""" + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting path {0}' + 'during tearDown process: {1}'.format(path, str(excinfo))) + warnings.warn(message) + + shutil.rmtree(self.temp_dir, onerror=onerror_handler) + shutil.rmtree(self.config_dir, onerror=onerror_handler) + shutil.rmtree(self.work_dir, onerror=onerror_handler) + shutil.rmtree(self.logs_dir, onerror=onerror_handler) + def get_data_filename(filename): """Gets the filename of a test data file.""" diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index bcd02d197..2108298ae 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.26.0 -certbot[dev]==0.22.0 +acme[dev]==0.29.0 +certbot[dev]==0.32.0 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index f78473ada..7494bf6bf 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,8 +1,10 @@ from setuptools import setup from setuptools import find_packages +from setuptools.command.test import test as TestCommand +import sys -version = '0.32.0.dev0' +version = '0.33.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -21,6 +23,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot-nginx', version=version, @@ -64,4 +82,6 @@ setup( ], }, test_suite='certbot_nginx', + tests_require=["pytest"], + cmdclass={"test": PyTest}, ) diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 470eab28e..80cca2682 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -3,16 +3,22 @@ # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf +# USAGE: ./boulder-integration.conf.sh /path/to/root cert.key cert.pem >> nginx.conf + +ROOT=$1 +CERT_KEY_PATH=$2 +CERT_PATH=$3 + cat < $nginx_conf diff --git a/certbot/__init__.py b/certbot/__init__.py index 767448a12..38e25f3a4 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.32.0.dev0' +__version__ = '0.33.0.dev0' diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 3dfaaf26f..517b56351 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -1,29 +1,23 @@ """ACME AuthHandler.""" -import collections import logging import time +import datetime -import six import zope.component from acme import challenges from acme import messages # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import DefaultDict, Dict, List, Set, Collection +from acme.magic_typing import Dict, List # pylint: enable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot import error_handler from certbot import interfaces - logger = logging.getLogger(__name__) -AnnotatedAuthzr = collections.namedtuple("AnnotatedAuthzr", ["authzr", "achalls"]) -"""Stores an authorization resource and its active annotated challenges.""" - - class AuthHandler(object): """ACME Authorization Handler for a client. @@ -47,242 +41,151 @@ class AuthHandler(object): self.account = account self.pref_challs = pref_challs - def handle_authorizations(self, orderr, best_effort=False): - """Retrieve all authorizations for challenges. - - :param acme.messages.OrderResource orderr: must have - authorizations filled in - :param bool best_effort: Whether or not all authorizations are - required (this is useful in renewal) - - :returns: List of authorization resources - :rtype: list - - :raises .AuthorizationError: If unable to retrieve all - authorizations - + def handle_authorizations(self, orderr, best_effort=False, max_retries=30): """ - aauthzrs = [AnnotatedAuthzr(authzr, []) - for authzr in orderr.authorizations] + Retrieve all authorizations, perform all challenges required to validate + these authorizations, then poll and wait for the authorization to be checked. + :param acme.messages.OrderResource orderr: must have authorizations filled in + :param bool best_effort: if True, not all authorizations need to be validated (eg. renew) + :param int max_retries: maximum number of retries to poll authorizations + :returns: list of all validated authorizations + :rtype: List - self._choose_challenges(aauthzrs) - config = zope.component.getUtility(interfaces.IConfig) - notify = zope.component.getUtility(interfaces.IDisplay).notification + :raises .AuthorizationError: If unable to retrieve all authorizations + """ + authzrs = orderr.authorizations[:] + if not authzrs: + raise errors.AuthorizationError('No authorization to handle.') - # While there are still challenges remaining... - while self._has_challenges(aauthzrs): - with error_handler.ExitHandler(self._cleanup_challenges, aauthzrs): - resp = self._solve_challenges(aauthzrs) - logger.info("Waiting for verification...") + # Retrieve challenges that need to be performed to validate authorizations. + achalls = self._choose_challenges(authzrs) + if not achalls: + return authzrs + + # Starting now, challenges will be cleaned at the end no matter what. + with error_handler.ExitHandler(self._cleanup_challenges, achalls): + # To begin, let's ask the authenticator plugin to perform all challenges. + try: + resps = self.auth.perform(achalls) + + # If debug is on, wait for user input before starting the verification process. + logger.info('Waiting for verification...') + config = zope.component.getUtility(interfaces.IConfig) if config.debug_challenges: + notify = zope.component.getUtility(interfaces.IDisplay).notification notify('Challenges loaded. Press continue to submit to CA. ' 'Pass "-v" for more info about challenges.', pause=True) + except errors.AuthorizationError as error: + logger.critical('Failure in setting up challenges.') + logger.info('Attempting to clean up outstanding challenges...') + raise error + # All challenges should have been processed by the authenticator. + assert len(resps) == len(achalls), 'Some challenges have not been performed.' - # Send all Responses - this modifies achalls - self._respond(aauthzrs, resp, best_effort) + # Inform the ACME CA server that challenges are available for validation. + for achall, resp in zip(achalls, resps): + self.acme.answer_challenge(achall.challb, resp) - # Just make sure all decisions are complete. - self.verify_authzr_complete(aauthzrs) + # Wait for authorizations to be checked. + self._poll_authorizations(authzrs, max_retries, best_effort) - # Only return valid authorizations - ret_val = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + # Keep validated authorizations only. If there is none, no certificate can be issued. + authzrs_validated = [authzr for authzr in authzrs + if authzr.body.status == messages.STATUS_VALID] + if not authzrs_validated: + raise errors.AuthorizationError('All challenges have failed.') - if not ret_val: - raise errors.AuthorizationError( - "Challenges failed for all domains") + return authzrs_validated - return ret_val + def _poll_authorizations(self, authzrs, max_retries, best_effort): + """ + Poll the ACME CA server, to wait for confirmation that authorizations have their challenges + all verified. The poll may occur several times, until all authorizations are checked + (valid or invalid), or after a maximum of retries. + """ + authzrs_to_check = {index: (authzr, None) + for index, authzr in enumerate(authzrs)} + authzrs_failed_to_report = [] + # Give an initial second to the ACME CA server to check the authorizations + sleep_seconds = 1 + for _ in range(max_retries): + # Wait for appropriate time (from Retry-After, initial wait, or no wait) + if sleep_seconds > 0: + time.sleep(sleep_seconds) + # Poll all updated authorizations. + authzrs_to_check = {index: self.acme.poll(authzr) for index, (authzr, _) + in authzrs_to_check.items()} + # Update the original list of authzr with the updated authzrs from server. + for index, (authzr, _) in authzrs_to_check.items(): + authzrs[index] = authzr - def _choose_challenges(self, aauthzrs): + # Gather failed authorizations + authzrs_failed = [authzr for authzr, _ in authzrs_to_check.values() + if authzr.body.status == messages.STATUS_INVALID] + for authzr_failed in authzrs_failed: + logger.warning('Challenge failed for domain %s', + authzr_failed.body.identifier.value) + # Accumulating all failed authzrs to build a consolidated report + # on them at the end of the polling. + authzrs_failed_to_report.extend(authzrs_failed) + + # Extract out the authorization already checked for next poll iteration. + # Poll may stop here because there is no pending authorizations anymore. + authzrs_to_check = {index: (authzr, resp) for index, (authzr, resp) + in authzrs_to_check.items() + if authzr.body.status == messages.STATUS_PENDING} + if not authzrs_to_check: + # Polling process is finished, we can leave the loop + break + + # Be merciful with the ACME server CA, check the Retry-After header returned, + # and wait this time before polling again in next loop iteration. + # From all the pending authorizations, we take the greatest Retry-After value + # to avoid polling an authorization before its relevant Retry-After value. + retry_after = max(self.acme.retry_after(resp, 3) + for _, resp in authzrs_to_check.values()) + sleep_seconds = (retry_after - datetime.datetime.now()).total_seconds() + + # In case of failed authzrs, create a report to the user. + if authzrs_failed_to_report: + _report_failed_authzrs(authzrs_failed_to_report, self.account.key) + if not best_effort: + # Without best effort, having failed authzrs is critical and fail the process. + raise errors.AuthorizationError('Some challenges have failed.') + + if authzrs_to_check: + # Here authzrs_to_check is still not empty, meaning we exceeded the max polling attempt. + raise errors.AuthorizationError('All authorizations were not finalized by the CA.') + + def _choose_challenges(self, authzrs): """ Retrieve necessary and pending challenges to satisfy server. NB: Necessary and already validated challenges are not retrieved, as they can be reused for a certificate issuance. """ - pending_authzrs = [aauthzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status != messages.STATUS_VALID] + pending_authzrs = [authzr for authzr in authzrs + if authzr.body.status != messages.STATUS_VALID] + achalls = [] # type: List[achallenges.AnnotatedChallenge] if pending_authzrs: logger.info("Performing the following challenges:") - for aauthzr in pending_authzrs: - aauthzr_challenges = aauthzr.authzr.body.challenges + for authzr in pending_authzrs: + authzr_challenges = authzr.body.challenges if self.acme.acme_version == 1: - combinations = aauthzr.authzr.body.combinations + combinations = authzr.body.combinations else: - combinations = tuple((i,) for i in range(len(aauthzr_challenges))) + combinations = tuple((i,) for i in range(len(authzr_challenges))) path = gen_challenge_path( - aauthzr_challenges, - self._get_chall_pref(aauthzr.authzr.body.identifier.value), + authzr_challenges, + self._get_chall_pref(authzr.body.identifier.value), combinations) - aauthzr_achalls = self._challenge_factory( - aauthzr.authzr, path) - aauthzr.achalls.extend(aauthzr_achalls) + achalls.extend(self._challenge_factory(authzr, path)) - for aauthzr in aauthzrs: - for achall in aauthzr.achalls: - if isinstance(achall.chall, challenges.TLSSNI01): - logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - return + if any(isinstance(achall.chall, challenges.TLSSNI01) for achall in achalls): + logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.") - def _has_challenges(self, aauthzrs): - """Do we have any challenges to perform?""" - return any(aauthzr.achalls for aauthzr in aauthzrs) - - def _solve_challenges(self, aauthzrs): - """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[challenges.ChallengeResponse] - all_achalls = self._get_all_achalls(aauthzrs) - try: - if all_achalls: - resp = self.auth.perform(all_achalls) - except errors.AuthorizationError: - logger.critical("Failure in setting up challenges.") - logger.info("Attempting to clean up outstanding challenges...") - raise - - assert len(resp) == len(all_achalls) - - return resp - - def _get_all_achalls(self, aauthzrs): - """Return all active challenges.""" - all_achalls = [] # type: Collection[challenges.ChallengeResponse] - for aauthzr in aauthzrs: - all_achalls.extend(aauthzr.achalls) - return all_achalls - - def _respond(self, aauthzrs, resp, best_effort): - """Send/Receive confirmation of all challenges. - - .. note:: This method also cleans up the auth_handler state. - - """ - # TODO: chall_update is a dirty hack to get around acme-spec #105 - chall_update = dict() \ - # type: Dict[int, List[achallenges.KeyAuthorizationAnnotatedChallenge]] - self._send_responses(aauthzrs, resp, chall_update) - - # Check for updated status... - self._poll_challenges(aauthzrs, chall_update, best_effort) - - def _send_responses(self, aauthzrs, resps, chall_update): - """Send responses and make sure errors are handled. - - :param aauthzrs: authorizations and the selected annotated challenges - to try and perform - :type aauthzrs: `list` of `AnnotatedAuthzr` - :param resps: challenge responses from the authenticator where - each response at index i corresponds to the annotated - challenge at index i in the list returned by - :func:`_get_all_achalls` - :type resps: `collections.abc.Iterable` of - :class:`~acme.challenges.ChallengeResponse` or `False` or - `None` - :param dict chall_update: parameter that is updated to hold - aauthzr index to list of outstanding solved annotated challenges - - """ - active_achalls = [] - resps_iter = iter(resps) - for i, aauthzr in enumerate(aauthzrs): - for achall in aauthzr.achalls: - # This line needs to be outside of the if block below to - # ensure failed challenges are cleaned up correctly - active_achalls.append(achall) - - resp = next(resps_iter) - # Don't send challenges for None and False authenticator responses - if resp: - self.acme.answer_challenge(achall.challb, resp) - # TODO: answer_challenge returns challr, with URI, - # that can be used in _find_updated_challr - # comparisons... - chall_update.setdefault(i, []).append(achall) - - return active_achalls - - def _poll_challenges(self, aauthzrs, chall_update, - best_effort, min_sleep=3, max_rounds=30): - """Wait for all challenge results to be determined.""" - indices_to_check = set(chall_update.keys()) - comp_indices = set() - rounds = 0 - - while indices_to_check and rounds < max_rounds: - # TODO: Use retry-after... - time.sleep(min_sleep) - all_failed_achalls = set() # type: Set[achallenges.KeyAuthorizationAnnotatedChallenge] - for index in indices_to_check: - comp_achalls, failed_achalls = self._handle_check( - aauthzrs, index, chall_update[index]) - - if len(comp_achalls) == len(chall_update[index]): - comp_indices.add(index) - elif not failed_achalls: - for achall, _ in comp_achalls: - chall_update[index].remove(achall) - # We failed some challenges... damage control - else: - if best_effort: - comp_indices.add(index) - logger.warning( - "Challenge failed for domain %s", - aauthzrs[index].authzr.body.identifier.value) - else: - all_failed_achalls.update( - updated for _, updated in failed_achalls) - - if all_failed_achalls: - _report_failed_challs(all_failed_achalls) - raise errors.FailedChallenges(all_failed_achalls) - - indices_to_check -= comp_indices - comp_indices.clear() - rounds += 1 - - def _handle_check(self, aauthzrs, index, achalls): - """Returns tuple of ('completed', 'failed').""" - completed = [] - failed = [] - - original_aauthzr = aauthzrs[index] - updated_authzr, _ = self.acme.poll(original_aauthzr.authzr) - aauthzrs[index] = AnnotatedAuthzr(updated_authzr, original_aauthzr.achalls) - if updated_authzr.body.status == messages.STATUS_VALID: - return achalls, [] - - # Note: if the whole authorization is invalid, the individual failed - # challenges will be determined here... - for achall in achalls: - updated_achall = achall.update(challb=self._find_updated_challb( - updated_authzr, achall)) - - # This does nothing for challenges that have yet to be decided yet. - if updated_achall.status == messages.STATUS_VALID: - completed.append((achall, updated_achall)) - elif updated_achall.status == messages.STATUS_INVALID: - failed.append((achall, updated_achall)) - - return completed, failed - - def _find_updated_challb(self, authzr, achall): # pylint: disable=no-self-use - """Find updated challenge body within Authorization Resource. - - .. warning:: This assumes only one instance of type of challenge in - each challenge resource. - - :param .AuthorizationResource authzr: Authorization Resource - :param .AnnotatedChallenge achall: Annotated challenge for which - to get status - - """ - for authzr_challb in authzr.body.challenges: - if type(authzr_challb.chall) is type(achall.challb.chall): # noqa - return authzr_challb - raise errors.AuthorizationError( - "Target challenge not found in authorization resource") + return achalls def _get_chall_pref(self, domain): """Return list of challenge preferences. @@ -306,43 +209,15 @@ class AuthHandler(object): chall_prefs.extend(plugin_pref) return chall_prefs - def _cleanup_challenges(self, aauthzrs, achalls=None): + def _cleanup_challenges(self, achalls): """Cleanup challenges. - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` :param achalls: annotated challenges to cleanup :type achalls: `list` of :class:`certbot.achallenges.AnnotatedChallenge` """ logger.info("Cleaning up challenges") - if achalls is None: - achalls = self._get_all_achalls(aauthzrs) - if achalls: - self.auth.cleanup(achalls) - for achall in achalls: - for aauthzr in aauthzrs: - if achall in aauthzr.achalls: - aauthzr.achalls.remove(achall) - break - - def verify_authzr_complete(self, aauthzrs): - """Verifies that all authorizations have been decided. - - :param aauthzrs: authorizations and their selected annotated - challenges - :type aauthzrs: `list` of `AnnotatedAuthzr` - - :returns: Whether all authzr are complete - :rtype: bool - - """ - for aauthzr in aauthzrs: - authzr = aauthzr.authzr - if (authzr.body.status != messages.STATUS_VALID and - authzr.body.status != messages.STATUS_INVALID): - raise errors.AuthorizationError("Incomplete authorizations") + self.auth.cleanup(achalls) def _challenge_factory(self, authzr, path): """Construct Namedtuple Challenges @@ -530,22 +405,19 @@ _ERROR_HELP = { } -def _report_failed_challs(failed_achalls): - """Notifies the user about failed challenges. +def _report_failed_authzrs(failed_authzrs, account_key): + """Notifies the user about failed authorizations.""" + problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) + for authzr in failed_authzrs for challb in authzr.body.challenges + if challb.error] - :param set failed_achalls: A set of failed - :class:`certbot.achallenges.AnnotatedChallenge`. - - """ - problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: - if achall.error: - problems[achall.error.typ].append(achall) + problems.setdefault(achall.error.typ, []).append(achall) + reporter = zope.component.getUtility(interfaces.IReporter) - for achalls in six.itervalues(problems): - reporter.add_message( - _generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) + for achalls in problems.values(): + reporter.add_message(_generate_failed_chall_msg(achalls), reporter.MEDIUM_PRIORITY) def _generate_failed_chall_msg(failed_achalls): diff --git a/certbot/cli.py b/certbot/cli.py index 31f55711f..398124510 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -108,7 +108,7 @@ manage your account with Let's Encrypt: # This is the short help for certbot --help, where we disable argparse # altogether -HELP_USAGE = """ +HELP_AND_VERSION_USAGE = """ More detailed help: -h, --help [TOPIC] print this message, or detailed help on a topic; @@ -117,6 +117,8 @@ More detailed help: all, automation, commands, paths, security, testing, or any of the subcommands or plugins (certonly, renew, install, register, nginx, apache, standalone, webroot, etc.) + + --version print the version number """ @@ -566,7 +568,7 @@ class HelpfulArgumentParser(object): usage = SHORT_USAGE if help_arg == True: - self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_USAGE) + self.notify(usage + COMMAND_OVERVIEW % (apache_doc, nginx_doc) + HELP_AND_VERSION_USAGE) sys.exit(0) elif help_arg in self.COMMANDS_TOPICS: self.notify(usage + self._list_subcommands()) diff --git a/certbot/compat.py b/certbot/compat.py index 3b5d068a6..f533f5954 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -13,13 +13,6 @@ import stat from certbot import errors -try: - # Linux specific - import fcntl # pylint: disable=import-error -except ImportError: - # Windows specific - import msvcrt # pylint: disable=import-error - UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [ 'certificates', 'enhance', 'revoke', 'delete', 'register', 'unregister', 'config_changes', 'plugins'] @@ -118,55 +111,6 @@ def readline_with_timeout(timeout, prompt): return sys.stdin.readline() -def lock_file(fd): - """ - Lock the file linked to the specified file descriptor. - - :param int fd: The file descriptor of the file to lock. - - """ - if 'fcntl' in sys.modules: - # Linux specific - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - else: - # Windows specific - msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - - -def release_locked_file(fd, path): - """ - Remove, close, and release a lock file specified by its file descriptor and its path. - - :param int fd: The file descriptor of the lock file. - :param str path: The path of the lock file. - - """ - # Linux specific - # - # It is important the lock file is removed before it's released, - # otherwise: - # - # process A: open lock file - # process B: release lock file - # process A: lock file - # process A: check device and inode - # process B: delete file - # process C: open and lock a different file at the same path - try: - os.remove(path) - except OSError as err: - if err.errno == errno.EACCES: - # Windows specific - # We will not be able to remove a file before closing it. - # To avoid race conditions described for Linux, we will not delete the lockfile, - # just close it to be reused on the next Certbot call. - pass - else: - raise - finally: - os.close(fd) - - def compare_file_modes(mode1, mode2): """Return true if the two modes can be considered as equals for this platform""" if os.name != 'nt': diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index c4a389cd5..66c68eb38 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -19,7 +19,7 @@ from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey # https://github.com/python/typeshed/tree/master/third_party/2/cryptography -from cryptography import x509 # type: ignore +from cryptography import x509 # type: ignore from OpenSSL import crypto from OpenSSL import SSL # type: ignore @@ -226,7 +226,7 @@ def verify_renewable_cert(renewable_cert): def verify_renewable_cert_sig(renewable_cert): - """ Verifies the signature of a `.storage.RenewableCert` object. + """Verifies the signature of a `.storage.RenewableCert` object. :param `.storage.RenewableCert` renewable_cert: cert to verify @@ -239,22 +239,8 @@ def verify_renewable_cert_sig(renewable_cert): cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): - warnings.simplefilter("ignore") - if isinstance(pk, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = pk.verifier( # type: ignore - cert.signature, PKCS1v15(), cert.signature_hash_algorithm - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - elif isinstance(pk, EllipticCurvePublicKey): - verifier = pk.verifier( - cert.signature, ECDSA(cert.signature_hash_algorithm) - ) - verifier.update(cert.tbs_certificate_bytes) - verifier.verify() - else: - raise errors.Error("Unsupported public key type") + verify_signed_payload(pk, cert.signature, cert.tbs_certificate_bytes, + cert.signature_hash_algorithm) except (IOError, ValueError, InvalidSignature) as e: error_str = "verifying the signature of the cert located at {0} has failed. \ Details: {1}".format(renewable_cert.cert, e) @@ -262,6 +248,37 @@ def verify_renewable_cert_sig(renewable_cert): raise errors.Error(error_str) +def verify_signed_payload(public_key, signature, payload, signature_hash_algorithm): + """Check the signature of a payload. + + :param RSAPublicKey/EllipticCurvePublicKey public_key: the public_key to check signature + :param bytes signature: the signature bytes + :param bytes payload: the payload bytes + :param cryptography.hazmat.primitives.hashes.HashAlgorithm + signature_hash_algorithm: algorithm used to hash the payload + + :raises InvalidSignature: If signature verification fails. + :raises errors.Error: If public key type is not supported + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if isinstance(public_key, RSAPublicKey): + # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi + verifier = public_key.verifier( # type: ignore + signature, PKCS1v15(), signature_hash_algorithm + ) + verifier.update(payload) + verifier.verify() + elif isinstance(public_key, EllipticCurvePublicKey): + verifier = public_key.verifier( + signature, ECDSA(signature_hash_algorithm) + ) + verifier.update(payload) + verifier.verify() + else: + raise errors.Error("Unsupported public key type") + + def verify_cert_matches_priv_key(cert_path, key_path): """ Verifies that the private key and cert match. diff --git a/certbot/hooks.py b/certbot/hooks.py index d5239a437..7d2e42fcd 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -93,8 +93,7 @@ def _run_pre_hook_if_necessary(command): if command in executed_pre_hooks: logger.info("Pre-hook command already run, skipping: %s", command) else: - logger.info("Running pre-hook command: %s", command) - _run_hook(command) + _run_hook("pre-hook", command) executed_pre_hooks.add(command) @@ -126,8 +125,7 @@ def post_hook(config): _run_eventually(cmd) # certonly / run elif cmd: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) post_hooks = [] # type: List[str] @@ -149,8 +147,7 @@ def _run_eventually(command): def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" for cmd in post_hooks: - logger.info("Running post-hook command: %s", cmd) - _run_hook(cmd) + _run_hook("post-hook", cmd) def deploy_hook(config, domains, lineage_path): @@ -220,23 +217,30 @@ def _run_deploy_hook(command, domains, lineage_path, dry_run): os.environ["RENEWED_DOMAINS"] = " ".join(domains) os.environ["RENEWED_LINEAGE"] = lineage_path - logger.info("Running deploy-hook command: %s", command) - _run_hook(command) + _run_hook("deploy-hook", command) -def _run_hook(shell_cmd): +def _run_hook(cmd_name, shell_cmd): """Run a hook command. - :returns: stderr if there was any""" + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` - err, _ = execute(shell_cmd) + :returns: stderr if there was any""" + err, _ = execute(cmd_name, shell_cmd) return err -def execute(shell_cmd): +def execute(cmd_name, shell_cmd): """Run a command. + :param str cmd_name: the user facing name of the hook being run + :param shell_cmd: shell command to execute + :type shell_cmd: `list` of `str` or `str` + :returns: `tuple` (`str` stderr, `str` stdout)""" + logger.info("Running %s command: %s", cmd_name, shell_cmd) # universal_newlines causes Popen.communicate() # to return str objects instead of bytes in Python 3 @@ -245,12 +249,12 @@ def execute(shell_cmd): out, err = cmd.communicate() base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) if out: - logger.info('Output from %s:\n%s', base_cmd, out) + logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out) if cmd.returncode != 0: - logger.error('Hook command "%s" returned error code %d', - shell_cmd, cmd.returncode) + logger.error('%s command "%s" returned error code %d', + cmd_name, shell_cmd, cmd.returncode) if err: - logger.error('Error output from %s:\n%s', base_cmd, err) + logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err) return (err, out) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index bb9a91b0f..bd91d2272 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -159,21 +159,14 @@ class IAuthenticator(IPlugin): :func:`get_chall_pref` only. :returns: `collections.Iterable` of ACME - :class:`~acme.challenges.ChallengeResponse` instances - or if the :class:`~acme.challenges.Challenge` cannot - be fulfilled then: - - ``None`` - Authenticator can perform challenge, but not at this time. - ``False`` - Authenticator will never be able to perform (error). - + :class:`~acme.challenges.ChallengeResponse` instances corresponding to each provided + :class:`~acme.challenges.Challenge`. :rtype: :class:`collections.Iterable` of :class:`acme.challenges.ChallengeResponse`, where responses are required to be returned in the same order as corresponding input challenges - :raises .PluginError: If challenges cannot be performed + :raises .PluginError: If some or all challenges cannot be performed """ diff --git a/certbot/lock.py b/certbot/lock.py index 3ff46518d..760a12b8f 100644 --- a/certbot/lock.py +++ b/certbot/lock.py @@ -1,15 +1,23 @@ -"""Implements file locks for locking files and directories in UNIX.""" +"""Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging import os +try: + import fcntl # pylint: disable=import-error +except ImportError: + import msvcrt # pylint: disable=import-error + POSIX_MODE = False +else: + POSIX_MODE = True -from certbot import compat from certbot import errors +from acme.magic_typing import Optional, Callable # pylint: disable=unused-import, no-name-in-module logger = logging.getLogger(__name__) def lock_dir(dir_path): + # type: (str) -> LockFile """Place a lock file on the directory at dir_path. The lock file is placed in the root of dir_path with the name @@ -27,34 +35,99 @@ def lock_dir(dir_path): class LockFile(object): - """A UNIX lock file. - - This lock file is released when the locked file is closed or the - process exits. It cannot be used to provide synchronization between - threads. It is based on the lock_file package by Martin Horcicka. - + """ + Platform independent file lock system. + LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, + instance is created, the associated file is 'locked from the point of view of the OS, + meaning that if another instance of Certbot try at the same time to acquire the same lock, + it will raise an Exception. Calling release method will release the lock, and make it + available to every other instance. + Upon exit, Certbot will also release all the locks. + This allows us to protect a file or directory from being concurrently accessed + or modified by two Certbot instances. + LockFile is platform independent: it will proceed to the appropriate OS lock mechanism + depending on Linux or Windows. """ def __init__(self, path): - """Initialize and acquire the lock file. - - :param str path: path to the file to lock - - :raises errors.LockError: if unable to acquire the lock - + # type: (str) -> None + """ + Create a LockFile instance on the given file path, and acquire lock. + :param str path: the path to the file that will hold a lock """ - super(LockFile, self).__init__() self._path = path - self._fd = None + mechanism = _UnixLockMechanism if POSIX_MODE else _WindowsLockMechanism + self._lock_mechanism = mechanism(path) self.acquire() + def __repr__(self): + # type: () -> str + repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) + if self.is_locked(): + repr_str += 'acquired>' + else: + repr_str += 'released>' + return repr_str + def acquire(self): - """Acquire the lock file. - - :raises errors.LockError: if lock is already held - :raises OSError: if unable to open or stat the lock file - + # type: () -> None """ + Acquire the lock on the file, forbidding any other Certbot instance to acquire it. + :raises errors.LockError: if unable to acquire the lock + """ + self._lock_mechanism.acquire() + + def release(self): + # type: () -> None + """ + Release the lock on the file, allowing any other Certbot instance to acquire it. + """ + self._lock_mechanism.release() + + def is_locked(self): + # type: () -> bool + """ + Check if the file is currently locked. + :return: True if the file is locked, False otherwise + """ + return self._lock_mechanism.is_locked() + + +class _BaseLockMechanism(object): + def __init__(self, path): + # type: (str) -> None + """ + Create a lock file mechanism for Unix. + :param str path: the path to the lock file + """ + self._path = path + self._fd = None # type: Optional[int] + + def is_locked(self): + # type: () -> bool + """Check if lock file is currently locked. + :return: True if the lock file is locked + :rtype: bool + """ + return self._fd is not None + + def acquire(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + def release(self): # pylint: disable=missing-docstring + pass # pragma: no cover + + +class _UnixLockMechanism(_BaseLockMechanism): + """ + A UNIX lock file mechanism. + This lock file is released when the locked file is closed or the + process exits. It cannot be used to provide synchronization between + threads. It is based on the lock_file package by Martin Horcicka. + """ + def acquire(self): + # type: () -> None + """Acquire the lock.""" while self._fd is None: # Open the file fd = os.open(self._path, os.O_CREAT | os.O_WRONLY, 0o600) @@ -68,33 +141,29 @@ class LockFile(object): os.close(fd) def _try_lock(self, fd): - """Try to acquire the lock file without blocking. - + # type: (int) -> None + """ + Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock - """ try: - compat.lock_file(fd) + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as err: if err.errno in (errno.EACCES, errno.EAGAIN): - logger.debug( - "A lock on %s is held by another process.", self._path) - raise errors.LockError( - "Another instance of Certbot is already running.") + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') raise def _lock_success(self, fd): - """Did we successfully grab the lock? - + # type: (int) -> bool + """ + Did we successfully grab the lock? Because this class deletes the locked file when the lock is released, it is possible another process removed and recreated the file between us opening the file and acquiring the lock. - :param int fd: file descriptor of the opened file to lock - :returns: True if the lock was successfully acquired :rtype: bool - """ try: stat1 = os.stat(self._path) @@ -108,17 +177,75 @@ class LockFile(object): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def __repr__(self): - repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) - if self._fd is None: - repr_str += 'released>' - else: - repr_str += 'acquired>' - return repr_str + def release(self): + # type: () -> None + """Remove, close, and release the lock file.""" + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + try: + os.remove(self._path) + finally: + # Following check is done to make mypy happy: it ensure that self._fd, marked + # as Optional[int] is effectively int to make it compatible with os.close signature. + if self._fd is None: # pragma: no cover + raise TypeError('Error, self._fd is None.') + try: + os.close(self._fd) + finally: + self._fd = None + + +class _WindowsLockMechanism(_BaseLockMechanism): + """ + A Windows lock file mechanism. + By default on Windows, acquiring a file handler gives exclusive access to the process + and results in an effective lock. However, it is possible to explicitly acquire the + file handler in shared access in terms of read and write, and this is done by os.open + and io.open in Python. So an explicit lock needs to be done through the call of + msvcrt.locking, that will lock the first byte of the file. In theory, it is also + possible to access a file in shared delete access, allowing other processes to delete an + opened file. But this needs also to be done explicitly by all processes using the Windows + low level APIs, and Python does not do it. As of Python 3.7 and below, Python developers + state that deleting a file opened by a process from another process is not possible with + os.open and io.open. + Consequently, mscvrt.locking is sufficient to obtain an effective lock, and the race + condition encountered on Linux is not possible on Windows, leading to a simpler workflow. + """ + def acquire(self): + """Acquire the lock""" + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + + fd = os.open(self._path, open_mode, 0o600) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except (IOError, OSError) as err: + os.close(fd) + # Anything except EACCES is unexpected. Raise directly the error in that case. + if err.errno != errno.EACCES: + raise + logger.debug('A lock on %s is held by another process.', self._path) + raise errors.LockError('Another instance of Certbot is already running.') + + self._fd = fd def release(self): - """Remove, close, and release the lock file.""" + """Release the lock.""" try: - compat.release_locked_file(self._fd, self._path) + msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) + os.close(self._fd) + + try: + os.remove(self._path) + except OSError as e: + # If the lock file cannot be removed, it is not a big deal. + # Likely another instance is acquiring the lock we just released. + logger.debug(str(e)) finally: self._fd = None diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 049e14827..bc63c40f9 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -1,53 +1,79 @@ """Tools for checking certificate revocation.""" import logging import re - +from datetime import datetime, timedelta from subprocess import Popen, PIPE +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp # pylint: disable=import-error + getattr(ocsp.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp = None # type: ignore +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +import requests + +from acme.magic_typing import Optional, Tuple # pylint: disable=unused-import, no-name-in-module +from certbot import crypto_util from certbot import errors from certbot import util logger = logging.getLogger(__name__) + class RevocationChecker(object): - "This class figures out OCSP checking on this system, and performs it." + """This class figures out OCSP checking on this system, and performs it.""" - def __init__(self): + def __init__(self, enforce_openssl_binary_usage=False): self.broken = False + self.use_openssl_binary = enforce_openssl_binary_usage or not ocsp - if not util.exe_exists("openssl"): - logger.info("openssl not installed, can't check revocation") - self.broken = True - return - - # New versions of openssl want -header var=val, old ones want -header var val - test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], - stdout=PIPE, stderr=PIPE, universal_newlines=True) - _out, err = test_host_format.communicate() - if "Missing =" in err: - self.host_args = lambda host: ["Host=" + host] - else: - self.host_args = lambda host: ["Host", host] + if self.use_openssl_binary: + if not util.exe_exists("openssl"): + logger.info("openssl not installed, can't check revocation") + self.broken = True + return + # New versions of openssl want -header var=val, old ones want -header var val + test_host_format = Popen(["openssl", "ocsp", "-header", "var", "val"], + stdout=PIPE, stderr=PIPE, universal_newlines=True) + _out, err = test_host_format.communicate() + if "Missing =" in err: + self.host_args = lambda host: ["Host=" + host] + else: + self.host_args = lambda host: ["Host", host] def ocsp_revoked(self, cert_path, chain_path): + # type: (str, str) -> bool """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call :param str cert_path: Path to certificate :param str chain_path: Path to intermediate cert - :rtype bool or None: :returns: True if revoked; False if valid or the check failed + :rtype: bool """ if self.broken: return False - - url, host = self.determine_ocsp_server(cert_path) - if not host: + url, host = _determine_ocsp_server(cert_path) + if not host or not url: return False + + if self.use_openssl_binary: + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) + else: + return _check_ocsp_cryptography(cert_path, chain_path, url) + + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): + # type: (str, str, str, str) -> bool # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", @@ -65,33 +91,131 @@ class RevocationChecker(object): except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False - return _translate_ocsp_query(cert_path, output, err) - def determine_ocsp_server(self, cert_path): - """Extract the OCSP server host from a certificate. +def _determine_ocsp_server(cert_path): + # type: (str) -> Tuple[Optional[str], Optional[str]] + """Extract the OCSP server host from a certificate. - :param str cert_path: Path to the cert we're checking OCSP for - :rtype tuple: - :returns: (OCSP server URL or None, OCSP server host or None) + :param str cert_path: Path to the cert we're checking OCSP for + :rtype tuple: + :returns: (OCSP server URL or None, OCSP server host or None) - """ - try: - url, _err = util.run_script( - ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logger.debug) - except errors.SubprocessError: - logger.info("Cannot extract OCSP URI from %s", cert_path) - return None, None + """ + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + try: + extension = cert.extensions.get_extension_for_class(x509.AuthorityInformationAccess) + ocsp_oid = x509.AuthorityInformationAccessOID.OCSP + descriptions = [description for description in extension.value + if description.access_method == ocsp_oid] + + url = descriptions[0].access_location.value + except (x509.ExtensionNotFound, IndexError): + logger.info("Cannot extract OCSP URI from %s", cert_path) + return None, None + + url = url.rstrip() + host = url.partition("://")[2].rstrip("/") + + if host: + return url, host + else: + logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) + return None, None + + +def _check_ocsp_cryptography(cert_path, chain_path, url): + # type: (str, str, str) -> bool + # Retrieve OCSP response + with open(chain_path, 'rb') as file_handler: + issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + with open(cert_path, 'rb') as file_handler: + cert = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) + builder = ocsp.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + request_binary = request.public_bytes(serialization.Encoding.DER) + try: + response = requests.post(url, data=request_binary, + headers={'Content-Type': 'application/ocsp-request'}) + except requests.exceptions.RequestException: + logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) + return False + if response.status_code != 200: + logger.info("OCSP check failed for %s (HTTP status: %d)", cert_path, response.status_code) + return False + + response_ocsp = ocsp.load_der_ocsp_response(response.content) + + # Check OCSP response validity + if response_ocsp.response_status != ocsp.OCSPResponseStatus.SUCCESSFUL: + logger.error("Invalid OCSP response status for %s: %s", + cert_path, response_ocsp.response_status) + return False + + # Check OCSP signature + try: + _check_ocsp_response(response_ocsp, request, issuer) + except UnsupportedAlgorithm as e: + logger.error(str(e)) + except errors.Error as e: + logger.error(str(e)) + except InvalidSignature: + logger.error('Invalid signature on OCSP response for %s', cert_path) + except AssertionError as error: + logger.error('Invalid OCSP response for %s: %s.', cert_path, str(error)) + else: + # Check OCSP certificate status + logger.debug("OCSP certificate status for %s is: %s", + cert_path, response_ocsp.certificate_status) + return response_ocsp.certificate_status == ocsp.OCSPCertStatus.REVOKED + + return False + + +def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert): + """Verify that the OCSP is valid for serveral criterias""" + # Assert OCSP response corresponds to the certificate we are talking about + if response_ocsp.serial_number != request_ocsp.serial_number: + raise AssertionError('the certificate in response does not correspond ' + 'to the certificate in request') + + # Assert signature is valid + _check_ocsp_response_signature(response_ocsp, issuer_cert) + + # Assert issuer in response is the expected one + if (not isinstance(response_ocsp.hash_algorithm, type(request_ocsp.hash_algorithm)) + or response_ocsp.issuer_key_hash != request_ocsp.issuer_key_hash + or response_ocsp.issuer_name_hash != request_ocsp.issuer_name_hash): + raise AssertionError('the issuer does not correspond to issuer of the certificate.') + + # In following checks, two situations can occur: + # * nextUpdate is set, and requirement is thisUpdate < now < nextUpdate + # * nextUpdate is not set, and requirement is thisUpdate < now + # NB1: We add a validity period tolerance to handle clock time inconsistencies, + # value is 5 min like for OpenSSL. + # NB2: Another check is to verify that thisUpdate is not too old, it is optional + # for OpenSSL, so we do not do it here. + # See OpenSSL implementation as a reference: + # https://github.com/openssl/openssl/blob/ef45aa14c5af024fcb8bef1c9007f3d1c115bd85/crypto/ocsp/ocsp_cl.c#L338-L391 + now = datetime.utcnow() # thisUpdate/nextUpdate are expressed in UTC/GMT time zone + if not response_ocsp.this_update: + raise AssertionError('param thisUpdate is not set.') + if response_ocsp.this_update > now + timedelta(minutes=5): + raise AssertionError('param thisUpdate is in the future.') + if response_ocsp.next_update and response_ocsp.next_update < now - timedelta(minutes=5): + raise AssertionError('param nextUpdate is in the past.') + + +def _check_ocsp_response_signature(response_ocsp, issuer_cert): + """Verify an OCSP response signature against certificate issuer""" + # Following line may raise UnsupportedAlgorithm + chosen_hash = response_ocsp.signature_hash_algorithm + crypto_util.verify_signed_payload(issuer_cert.public_key(), response_ocsp.signature, + response_ocsp.tbs_response_bytes, chosen_hash) - url = url.rstrip() - host = url.partition("://")[2].rstrip("/") - if host: - return url, host - else: - logger.info("Cannot process OCSP host from URL (%s) in cert at %s", url, cert_path) - return None, None def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): """Parse openssl's weird output to work out what it means.""" @@ -102,7 +226,7 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): warning = good.group(1) if good else None - if (not "Response verify OK" in ocsp_errors) or (good and warning) or unknown: + if ("Response verify OK" not in ocsp_errors) or (good and warning) or unknown: logger.info("Revocation status for %s is unknown", cert_path) logger.debug("Uncertain output:\n%s\nstderr:\n%s", ocsp_output, ocsp_errors) return False @@ -115,6 +239,5 @@ def _translate_ocsp_query(cert_path, ocsp_output, ocsp_errors): return True else: logger.warning("Unable to properly parse OCSP output: %s\nstderr:%s", - ocsp_output, ocsp_errors) + ocsp_output, ocsp_errors) return False - diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index ee1af4978..c14129d87 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -351,7 +351,7 @@ class ChallengePerformer(object): def perform(self): """Perform all added challenges. - :returns: challenge respones + :returns: challenge responses :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse` diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 8723a1c62..123a5bfea 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -202,7 +202,7 @@ permitted by DNS standards.) os.environ.pop('CERTBOT_KEY_PATH', None) os.environ.pop('CERTBOT_SNI_DOMAIN', None) os.environ.update(env) - _, out = hooks.execute(self.conf('auth-hook')) + _, out = self._execute_hook('auth-hook') env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env @@ -243,5 +243,8 @@ permitted by DNS standards.) if 'CERTBOT_TOKEN' not in env: os.environ.pop('CERTBOT_TOKEN', None) os.environ.update(env) - hooks.execute(self.conf('cleanup-hook')) + self._execute_hook('cleanup-hook') self.reverter.recovery_routine() + + def _execute_hook(self, hook_name): + return hooks.execute(self.option_name(hook_name), self.conf(hook_name)) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index fe0ece12e..353c34da2 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -4,13 +4,11 @@ import logging import unittest import mock -import six import zope.component from acme import challenges from acme import client as acme_client from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors @@ -82,6 +80,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) self.mock_net.acme_version = 1 + self.mock_net.retry_after.side_effect = acme_client.Client.retry_after self.handler = AuthHandler( self.mock_auth, self.mock_net, self.mock_account, []) @@ -95,23 +94,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p authzr = gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES, combos=combos) mock_order = mock.MagicMock(authorizations=[authzr]) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=1, wait_value=30) + with mock.patch('certbot.auth_handler.time') as mock_time: authzr = self.handler.handle_authorizations(mock_order) - self.assertEqual(self.mock_net.answer_challenge.call_count, 1) + self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 2) # Because there is one retry + self.assertEqual(mock_time.sleep.call_count, 2) + # Retry-After header is 30 seconds, but at the time sleep is invoked, several + # instructions are executed, and next pool is in less than 30 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + # However, assert that we did not took the default value of 3 seconds. + self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) - self.assertEqual(self.mock_auth.cleanup.call_count, 1) - # Test if list first element is TLSSNI01, use typ because it is an achall - self.assertEqual( - self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + self.assertEqual(self.mock_auth.cleanup.call_count, 1) + # Test if list first element is TLSSNI01, use typ because it is an achall + self.assertEqual( + self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - self.assertEqual(len(authzr), 1) + self.assertEqual(len(authzr), 1) def test_name1_tls_sni_01_1_acme_1(self): self._test_name1_tls_sni_01_1_common(combos=True) @@ -120,9 +122,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name1_tls_sni_01_1_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self, mock_poll): - mock_poll.side_effect = self._validate_all + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_1(self): + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -132,10 +133,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 3) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is TLSSNI01, use typ because it is an achall @@ -145,10 +143,9 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p # Length of authorizations list self.assertEqual(len(authzr), 1) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self, mock_poll): + def test_name1_tls_sni_01_1_http_01_1_dns_1_acme_2(self): self.mock_net.acme_version = 2 - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.mock_auth.get_chall_pref.return_value.append(challenges.HTTP01) self.mock_auth.get_chall_pref.return_value.append(challenges.DNS01) @@ -158,10 +155,7 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual(self.mock_net.answer_challenge.call_count, 1) - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(list(six.iterkeys(chall_update)), [0]) - self.assertEqual(len(chall_update.values()), 1) + self.assertEqual(self.mock_net.poll.call_count, 1) self.assertEqual(self.mock_auth.cleanup.call_count, 1) cleaned_up_achalls = self.mock_auth.cleanup.call_args[0][0] @@ -175,27 +169,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.request_domain_challenges.side_effect = functools.partial( gen_dom_authzr, challs=acme_util.CHALLENGES, combos=combos) - authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="1", challs=acme_util.CHALLENGES), gen_dom_authzr(domain="2", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - authzr = self.handler.handle_authorizations(mock_order) + + self.mock_net.poll.side_effect = _gen_mock_on_poll() + authzr = self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_net.answer_challenge.call_count, 3) # Check poll call - self.assertEqual(mock_poll.call_count, 1) - chall_update = mock_poll.call_args[0][1] - self.assertEqual(len(list(six.iterkeys(chall_update))), 3) - self.assertTrue(0 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[0]), 1) - self.assertTrue(1 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[1]), 1) - self.assertTrue(2 in list(six.iterkeys(chall_update))) - self.assertEqual(len(chall_update[2]), 1) + self.assertEqual(self.mock_net.poll.call_count, 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) @@ -208,14 +193,13 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.mock_net.acme_version = 2 self._test_name3_tls_sni_01_3_common(combos=False) - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - def test_debug_challenges(self, mock_poll): + def test_debug_challenges(self): zope.component.provideUtility( mock.Mock(debug_challenges=True), interfaces.IConfig) authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll() self.handler.handle_authorizations(mock_order) @@ -231,6 +215,18 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + def test_max_retries_exceeded(self): + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] + mock_order = mock.MagicMock(authorizations=authzrs) + + # We will return STATUS_PENDING twice before returning STATUS_VALID. + self.mock_net.poll.side_effect = _gen_mock_on_poll(retry=2) + + with self.assertRaises(errors.AuthorizationError) as error: + # We retry only once, so retries will be exhausted before STATUS_VALID is returned. + self.handler.handle_authorizations(mock_order, False, 1) + self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) self.assertRaises(errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -244,9 +240,8 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.handler.pref_challs.extend((challenges.HTTP01.typ, challenges.DNS01.typ,)) - with mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") as mock_poll: - mock_poll.side_effect = self._validate_all - self.handler.handle_authorizations(mock_order) + self.mock_net.poll.side_effect = _gen_mock_on_poll() + self.handler.handle_authorizations(mock_order) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( @@ -290,11 +285,11 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._respond") - def test_respond_error(self, mock_respond): + def test_answer_error(self): + self.mock_net.answer_challenge.side_effect = errors.AuthorizationError + authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_respond.side_effect = errors.AuthorizationError self.assertRaises( errors.AuthorizationError, self.handler.handle_authorizations, mock_order) @@ -302,20 +297,52 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") - @mock.patch("certbot.auth_handler.AuthHandler._poll_challenges") - @mock.patch("certbot.auth_handler.AuthHandler.verify_authzr_complete") - def test_incomplete_authzr_error(self, mock_verify, mock_poll): + def test_incomplete_authzr_error(self): authzrs = [gen_dom_authzr(domain="0", challs=acme_util.CHALLENGES)] mock_order = mock.MagicMock(authorizations=authzrs) - mock_verify.side_effect = errors.AuthorizationError - mock_poll.side_effect = self._validate_all + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) - self.assertRaises( - errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, False) + self.assertTrue('Some challenges have failed.' in str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_best_effort(self): + def _conditional_mock_on_poll(authzr): + """This mock will invalidate one authzr, and invalidate the other one""" + valid_mock = _gen_mock_on_poll(messages.STATUS_VALID) + invalid_mock = _gen_mock_on_poll(messages.STATUS_INVALID) + + if authzr.body.identifier.value == 'will-be-invalid': + return invalid_mock(authzr) + return valid_mock(authzr) + + # Two authzrs. Only one will be valid. + authzrs = [gen_dom_authzr(domain="will-be-valid", challs=acme_util.CHALLENGES), + gen_dom_authzr(domain="will-be-invalid", challs=acme_util.CHALLENGES)] + self.mock_net.poll.side_effect = _conditional_mock_on_poll + + mock_order = mock.MagicMock(authorizations=authzrs) + + with mock.patch('certbot.auth_handler._report_failed_authzrs') as mock_report: + valid_authzr = self.handler.handle_authorizations(mock_order, True) + + # Because best_effort=True, we did not blow up. Instead ... + self.assertEqual(len(valid_authzr), 1) # ... the valid authzr has been processed + self.assertEqual(mock_report.call_count, 1) # ... the invalid authzr has been reported + + self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID) + + with test_util.patch_get_utility(): + with self.assertRaises(errors.AuthorizationError) as error: + self.handler.handle_authorizations(mock_order, True) + + # Despite best_effort=True, process will fail because no authzr is valid. + self.assertTrue('All challenges have failed.' in str(error.exception)) + def test_validated_challenge_not_rerun(self): # With pending challenge, we expect the challenge to be tried, and fail. authzr = acme_util.gen_authzr( @@ -334,138 +361,26 @@ class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-p mock_order = mock.MagicMock(authorizations=[authzr]) self.handler.handle_authorizations(mock_order) - def _validate_all(self, aauthzrs, unused_1, unused_2): - for i, aauthzr in enumerate(aauthzrs): - azr = aauthzr.authzr - updated_azr = acme_util.gen_authzr( - messages.STATUS_VALID, - azr.body.identifier.value, - [challb.chall for challb in azr.body.challenges], - [messages.STATUS_VALID] * len(azr.body.challenges), - azr.body.combinations) - aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls) - @mock.patch("certbot.auth_handler.logger") def test_tls_sni_logs(self, logger): self._test_name1_tls_sni_01_1_common(combos=True) self.assertTrue("deprecated" in logger.warning.call_args[0][0]) -class PollChallengesTest(unittest.TestCase): - # pylint: disable=protected-access - """Test poll challenges.""" +def _gen_mock_on_poll(status=messages.STATUS_VALID, retry=0, wait_value=1): + state = {'count': retry} - def setUp(self): - from certbot.auth_handler import challb_to_achall - from certbot.auth_handler import AuthHandler, AnnotatedAuthzr - - # Account and network are mocked... - self.mock_net = mock.MagicMock() - self.handler = AuthHandler( - None, self.mock_net, mock.Mock(key="mock_key"), []) - - self.doms = ["0", "1", "2"] - self.aauthzrs = [ - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[0], - [acme_util.HTTP01, acme_util.TLSSNI01], - [messages.STATUS_PENDING] * 2, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[1], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []), - AnnotatedAuthzr(acme_util.gen_authzr( - messages.STATUS_PENDING, self.doms[2], - acme_util.CHALLENGES, [messages.STATUS_PENDING] * 3, False), []) - ] - - self.chall_update = {} # type: Dict[int, achallenges.KeyAuthorizationAnnotatedChallenge] - for i, aauthzr in enumerate(self.aauthzrs): - self.chall_update[i] = [ - challb_to_achall(challb, mock.Mock(key="dummy_key"), self.doms[i]) - for challb in aauthzr.authzr.body.challenges] - - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, False) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_VALID) - - @mock.patch("certbot.auth_handler.time") - def test_poll_challenges_failure_best_effort(self, unused_mock_time): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.handler._poll_challenges(self.aauthzrs, self.chall_update, True) - - for aauthzr in self.aauthzrs: - self.assertEqual(aauthzr.authzr.body.status, messages.STATUS_PENDING) - - @mock.patch("certbot.auth_handler.time") - @test_util.patch_get_utility() - def test_poll_challenges_failure(self, unused_mock_time, unused_mock_zope): - self.mock_net.poll.side_effect = self._mock_poll_solve_one_invalid - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - @mock.patch("certbot.auth_handler.time") - def test_unable_to_find_challenge_status(self, unused_mock_time): - from certbot.auth_handler import challb_to_achall - self.mock_net.poll.side_effect = self._mock_poll_solve_one_valid - self.chall_update[0].append( - challb_to_achall(acme_util.DNS01_P, "key", self.doms[0])) - self.assertRaises( - errors.AuthorizationError, self.handler._poll_challenges, - self.aauthzrs, self.chall_update, False) - - def test_verify_authzr_failure(self): - self.assertRaises(errors.AuthorizationError, - self.handler.verify_authzr_complete, self.aauthzrs) - - def _mock_poll_solve_one_valid(self, authzr): - # Pending here because my dummy script won't change the full status. - # Basically it didn't raise an error and it stopped earlier than - # Making all challenges invalid which would make mock_poll_solve_one - # change authzr to invalid - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_VALID) - - def _mock_poll_solve_one_invalid(self, authzr): - return self._mock_poll_solve_one_chall(authzr, messages.STATUS_INVALID) - - def _mock_poll_solve_one_chall(self, authzr, desired_status): - # pylint: disable=no-self-use - """Dummy method that solves one chall at a time to desired_status. - - When all are solved.. it changes authzr.status to desired_status - - """ - new_challbs = authzr.body.challenges - for challb in authzr.body.challenges: - if challb.status != desired_status: - new_challbs = tuple( - challb_temp if challb_temp != challb - else acme_util.chall_to_challb(challb.chall, desired_status) - for challb_temp in authzr.body.challenges - ) - break - - if all(test_challb.status == desired_status - for test_challb in new_challbs): - status_ = desired_status - else: - status_ = authzr.body.status - - new_authzr = messages.AuthorizationResource( - uri=authzr.uri, - body=messages.Authorization( - identifier=authzr.body.identifier, - challenges=new_challbs, - combinations=authzr.body.combinations, - status=status_, - ), - ) - return (new_authzr, "response") + def _mock(authzr): + state['count'] = state['count'] - 1 + effective_status = status if state['count'] < 0 else messages.STATUS_PENDING + updated_azr = acme_util.gen_authzr( + effective_status, + authzr.body.identifier.value, + [challb.chall for challb in authzr.body.challenges], + [effective_status] * len(authzr.body.challenges), + authzr.body.combinations) + return updated_azr, mock.MagicMock(headers={'Retry-After': str(wait_value)}) + return _mock class ChallbToAchallTest(unittest.TestCase): @@ -527,8 +442,8 @@ class GenChallengePathTest(unittest.TestCase): errors.AuthorizationError, self._call, challbs, prefs, None) -class ReportFailedChallsTest(unittest.TestCase): - """Tests for certbot.auth_handler._report_failed_challs.""" +class ReportFailedAuthzrsTest(unittest.TestCase): + """Tests for certbot.auth_handler._report_failed_authzrs.""" # pylint: disable=protected-access def setUp(self): @@ -542,31 +457,27 @@ class ReportFailedChallsTest(unittest.TestCase): # Prevent future regressions if the error type changes self.assertTrue(kwargs["error"].description is not None) - self.http01 = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + http_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args kwargs["chall"] = acme_util.TLSSNI01 - self.tls_sni_same = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="example.com", - account_key="key") + tls_sni_01 = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr1 = mock.MagicMock() + self.authzr1.body.identifier.value = 'example.com' + self.authzr1.body.challenges = [http_01, tls_sni_01] kwargs["error"] = messages.Error(typ="dnssec", detail="detail") - self.tls_sni_diff = achallenges.KeyAuthorizationAnnotatedChallenge( - # pylint: disable=star-args - challb=messages.ChallengeBody(**kwargs), - domain="foo.bar", - account_key="key") + tls_sni_01_diff = messages.ChallengeBody(**kwargs) # pylint: disable=star-args + + self.authzr2 = mock.MagicMock() + self.authzr2.body.identifier.value = 'foo.bar' + self.authzr2.body.challenges = [tls_sni_01_diff] @test_util.patch_get_utility() def test_same_error_and_domain(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_same]) + auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertTrue(len(call_list) == 1) self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) @@ -575,7 +486,7 @@ class ReportFailedChallsTest(unittest.TestCase): def test_different_errors_and_domains(self, mock_zope): from certbot import auth_handler - auth_handler._report_failed_challs([self.http01, self.tls_sni_diff]) + auth_handler._report_failed_authzrs([self.authzr1, self.authzr2], 'key') self.assertTrue(mock_zope().add_message.call_count == 2) diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index e16a1bdcf..f6669e7be 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -340,6 +340,8 @@ class ParseTest(unittest.TestCase): # pylint: disable=too-many-public-methods config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( config_dir_option, cli.flag_default(config_dir_option))) + self.assertFalse(cli.option_was_set( + 'authenticator', cli.flag_default('authenticator'))) def test_encode_revocation_reason(self): for reason, code in constants.REVOCATION_REASONS.items(): diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index f5bb0c8b5..90f639958 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -121,7 +121,7 @@ class PreHookTest(HookTest): def _test_nonrenew_common(self): mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_no_hooks(self): @@ -137,21 +137,21 @@ class PreHookTest(HookTest): def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.pre_hook) + mock_execute.assert_called_once_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_no_overlap(self): self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.pre_hook) + mock_execute.assert_any_call("pre-hook", self.dir_hook) + mock_execute.assert_called_with("pre-hook", self.config.pre_hook) self._test_no_executions_common() def test_renew_with_overlap(self): self.config.pre_hook = self.dir_hook self.config.verb = "renew" mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("pre-hook", self.dir_hook) self._test_no_executions_common() def _test_no_executions_common(self): @@ -193,7 +193,7 @@ class PostHookTest(HookTest): for verb in ("certonly", "run",): self.config.verb = verb mock_execute = self._call_with_mock_execute(self.config) - mock_execute.assert_called_once_with(self.config.post_hook) + mock_execute.assert_called_once_with("post-hook", self.config.post_hook) self.assertFalse(self._get_eventually()) def test_cert_only_and_run_without_hook(self): @@ -277,12 +277,12 @@ class RunSavedPostHooksTest(HookTest): calls = mock_execute.call_args_list for actual_call, expected_arg in zip(calls, self.eventually): - self.assertEqual(actual_call[0][0], expected_arg) + self.assertEqual(actual_call[0][1], expected_arg) def test_single(self): self.eventually = ["foo"] mock_execute = self._call_with_mock_execute_and_eventually() - mock_execute.assert_called_once_with(self.eventually[0]) + mock_execute.assert_called_once_with("post-hook", self.eventually[0]) class RenewalHookTest(HookTest): @@ -360,7 +360,7 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = "foo" mock_execute = self._call_with_mock_execute( self.config, domains, lineage) - mock_execute.assert_called_once_with(self.config.deploy_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.deploy_hook) class RenewHookTest(RenewalHookTest): @@ -384,7 +384,7 @@ class RenewHookTest(RenewalHookTest): self.config.directory_hooks = False mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.config.renew_hook) + mock_execute.assert_called_once_with("deploy-hook", self.config.renew_hook) @mock.patch("certbot.hooks.logger") def test_dry_run(self, mock_logger): @@ -408,13 +408,13 @@ class RenewHookTest(RenewalHookTest): self.config.renew_hook = self.dir_hook mock_execute = self._call_with_mock_execute( self.config, ["example.net", "example.org"], "/foo/bar") - mock_execute.assert_called_once_with(self.dir_hook) + mock_execute.assert_called_once_with("deploy-hook", self.dir_hook) def test_no_overlap(self): mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - mock_execute.assert_any_call(self.dir_hook) - mock_execute.assert_called_with(self.config.renew_hook) + mock_execute.assert_any_call("deploy-hook", self.dir_hook) + mock_execute.assert_called_with("deploy-hook", self.config.renew_hook) class ExecuteTest(unittest.TestCase): @@ -433,18 +433,22 @@ class ExecuteTest(unittest.TestCase): def _test_common(self, returncode, stdout, stderr): given_command = "foo" + given_name = "foo-hook" with mock.patch("certbot.hooks.Popen") as mock_popen: mock_popen.return_value.communicate.return_value = (stdout, stderr) mock_popen.return_value.returncode = returncode with mock.patch("certbot.hooks.logger") as mock_logger: - self.assertEqual(self._call(given_command), (stderr, stdout)) + self.assertEqual(self._call(given_name, given_command), (stderr, stdout)) executed_command = mock_popen.call_args[1].get( "args", mock_popen.call_args[0][0]) self.assertEqual(executed_command, given_command) + mock_logger.info.assert_any_call("Running %s command: %s", + given_name, given_command) if stdout: - self.assertTrue(mock_logger.info.called) + mock_logger.info.assert_any_call(mock.ANY, mock.ANY, + mock.ANY, stdout) if stderr or returncode: self.assertTrue(mock_logger.error.called) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index aa82701f3..6379693ae 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,6 +3,12 @@ import functools import multiprocessing import os import unittest +try: + import fcntl # pylint: disable=import-error,unused-import +except ImportError: + POSIX_MODE = False +else: + POSIX_MODE = True import mock @@ -10,7 +16,6 @@ from certbot import errors from certbot.tests import util as test_util -@test_util.broken_on_windows class LockDirTest(test_util.TempDirTestCase): """Tests for certbot.lock.lock_dir.""" @classmethod @@ -25,7 +30,6 @@ class LockDirTest(test_util.TempDirTestCase): test_util.lock_and_call(assert_raises, lock_path) -@test_util.broken_on_windows class LockFileTest(test_util.TempDirTestCase): """Tests for certbot.lock.LockFile.""" @classmethod @@ -37,6 +41,7 @@ class LockFileTest(test_util.TempDirTestCase): super(LockFileTest, self).setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') + @test_util.broken_on_windows def test_acquire_without_deletion(self): # acquire the lock in another process but don't delete the file child = multiprocessing.Process(target=self._call, @@ -54,6 +59,7 @@ class LockFileTest(test_util.TempDirTestCase): self.assertRaises, errors.LockError, self._call, self.lock_path) test_util.lock_and_call(assert_raises, self.lock_path) + @test_util.broken_on_windows def test_locked_repr(self): lock_file = self._call(self.lock_path) locked_repr = repr(lock_file) @@ -71,6 +77,8 @@ class LockFileTest(test_util.TempDirTestCase): self.assertTrue(lock_file.__class__.__name__ in lock_repr) self.assertTrue(self.lock_path in lock_repr) + @test_util.skip_on_windows( + 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') def test_race(self): should_delete = [True, False] stat = os.stat @@ -86,32 +94,42 @@ class LockFileTest(test_util.TempDirTestCase): self._call(self.lock_path) self.assertFalse(should_delete) + @test_util.broken_on_windows def test_removed(self): lock_file = self._call(self.lock_path) lock_file.release() self.assertFalse(os.path.exists(self.lock_path)) - @mock.patch('certbot.compat.fcntl.lockf') - def test_unexpected_lockf_err(self, mock_lockf): + def test_unexpected_lockf_or_locking_err(self): + if POSIX_MODE: + mocked_function = 'certbot.lock.fcntl.lockf' + else: + mocked_function = 'certbot.lock.msvcrt.locking' msg = 'hi there' - mock_lockf.side_effect = IOError(msg) - try: - self._call(self.lock_path) - except IOError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('IOError not raised') + with mock.patch(mocked_function) as mock_lock: + mock_lock.side_effect = IOError(msg) + try: + self._call(self.lock_path) + except IOError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('IOError not raised') - @mock.patch('certbot.lock.os.stat') - def test_unexpected_stat_err(self, mock_stat): + def test_unexpected_os_err(self): + if POSIX_MODE: + mock_function = 'certbot.lock.os.stat' + else: + mock_function = 'certbot.lock.msvcrt.locking' + # The only expected errno are ENOENT and EACCES in lock module. msg = 'hi there' - mock_stat.side_effect = OSError(msg) - try: - self._call(self.lock_path) - except OSError as err: - self.assertTrue(msg in str(err)) - else: # pragma: no cover - self.fail('OSError not raised') + with mock.patch(mock_function) as mock_os: + mock_os.side_effect = OSError(msg) + try: + self._call(self.lock_path) + except OSError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('OSError not raised') if __name__ == "__main__": diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 55cd24adb..ad3467e5a 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -1,18 +1,33 @@ """Tests for ocsp.py""" # pylint: disable=protected-access - import unittest +from datetime import datetime, timedelta +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature +from cryptography import x509 +try: + # Only cryptography>=2.5 has ocsp module + # and signature_hash_algorithm attribute in OCSPResponse class + from cryptography.x509 import ocsp as ocsp_lib # pylint: disable=import-error + getattr(ocsp_lib.OCSPResponse, 'signature_hash_algorithm') +except (ImportError, AttributeError): # pragma: no cover + ocsp_lib = None # type: ignore import mock from certbot import errors +from certbot.tests import util as test_util out = """Missing = in header key=value ocsp: Use -help for summary. """ -class OCSPTest(unittest.TestCase): +class OCSPTestOpenSSL(unittest.TestCase): + """ + OCSP revokation tests using OpenSSL binary. + """ def setUp(self): from certbot import ocsp @@ -22,7 +37,7 @@ class OCSPTest(unittest.TestCase): mock_communicate.communicate.return_value = (None, out) mock_popen.return_value = mock_communicate mock_exists.return_value = True - self.checker = ocsp.RevocationChecker() + self.checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) def tearDown(self): pass @@ -37,23 +52,23 @@ class OCSPTest(unittest.TestCase): mock_exists.return_value = True from certbot import ocsp - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 1) self.assertEqual(checker.host_args("x"), ["Host=x"]) mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) self.assertEqual(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 - checker = ocsp.RevocationChecker() + checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) self.assertEqual(checker.broken, True) - @mock.patch('certbot.ocsp.RevocationChecker.determine_ocsp_server') + @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.util.run_script') def test_ocsp_revoked(self, mock_run, mock_determine): self.checker.broken = True @@ -71,21 +86,12 @@ class OCSPTest(unittest.TestCase): self.assertEqual(self.checker.ocsp_revoked("x", "y"), False) self.assertEqual(mock_run.call_count, 2) + def test_determine_ocsp_server(self): + cert_path = test_util.vector_path('google_certificate.pem') - @mock.patch('certbot.ocsp.logger.info') - @mock.patch('certbot.util.run_script') - def test_determine_ocsp_server(self, mock_run, mock_info): - uri = "http://ocsp.stg-int-x1.letsencrypt.org/" - host = "ocsp.stg-int-x1.letsencrypt.org" - mock_run.return_value = uri, "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (uri, host)) - mock_run.return_value = "ftp:/" + host + "/", "" - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) - self.assertEqual(mock_info.call_count, 1) - - c = "confusion" - mock_run.side_effect = errors.SubprocessError(c) - self.assertEqual(self.checker.determine_ocsp_server("beep"), (None, None)) + from certbot import ocsp + result = ocsp._determine_ocsp_server(cert_path) + self.assertEqual(('http://ocsp.digicert.com', 'ocsp.digicert.com'), result) @mock.patch('certbot.ocsp.logger') @mock.patch('certbot.util.run_script') @@ -112,6 +118,129 @@ class OCSPTest(unittest.TestCase): self.assertEqual(mock_log.info.call_count, 1) +@unittest.skipIf(not ocsp_lib, + reason='This class tests functionalities available only on cryptography>=2.5.0') +class OSCPTestCryptography(unittest.TestCase): + """ + OCSP revokation tests using Cryptography >= 2.4.0 + """ + + def setUp(self): + from certbot import ocsp + self.checker = ocsp.RevocationChecker() + self.cert_path = test_util.vector_path('google_certificate.pem') + self.chain_path = test_util.vector_path('google_issuer_certificate.pem') + + @mock.patch('certbot.ocsp._determine_ocsp_server') + @mock.patch('certbot.ocsp._check_ocsp_cryptography') + def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + mock_determine.return_value = ('http://example.com', 'example.com') + self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke(self, mock_ocsp_response, mock_post): + with mock.patch('certbot.ocsp.crypto_util.verify_signed_payload'): + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertTrue(revoked) + + @mock.patch('certbot.ocsp.crypto_util.verify_signed_payload') + @mock.patch('certbot.ocsp.requests.post') + @mock.patch('certbot.ocsp.ocsp.load_der_ocsp_response') + def test_revoke_resiliency(self, mock_ocsp_response, mock_post, mock_check): + # Server return an invalid HTTP response + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=400) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response in invalid + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # OCSP response is valid, but certificate status is unknown + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # The OCSP response says that the certificate is revoked, but certificate + # does not contain the OCSP extension. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + with mock.patch('cryptography.x509.Extensions.get_extension_for_class', + side_effect=x509.ExtensionNotFound( + 'Not found', x509.AuthorityInformationAccessOID.OCSP)): + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Valid response, OCSP extension is present, + # but OCSP response uses an unsupported signature. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = UnsupportedAlgorithm('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # And now, the signature itself is invalid. + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = InvalidSignature('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + # Finally, assertion error on OCSP response validity + mock_ocsp_response.return_value = _construct_mock_ocsp_response( + ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) + mock_post.return_value = mock.Mock(status_code=200) + mock_check.side_effect = AssertionError('foo') + revoked = self.checker.ocsp_revoked(self.cert_path, self.chain_path) + + self.assertFalse(revoked) + + +def _construct_mock_ocsp_response(certificate_status, response_status): + cert = x509.load_pem_x509_certificate( + test_util.load_vector('google_certificate.pem'), default_backend()) + issuer = x509.load_pem_x509_certificate( + test_util.load_vector('google_issuer_certificate.pem'), default_backend()) + builder = ocsp_lib.OCSPRequestBuilder() + builder = builder.add_certificate(cert, issuer, hashes.SHA1()) + request = builder.build() + + return mock.Mock( + response_status=response_status, + certificate_status=certificate_status, + serial_number=request.serial_number, + issuer_key_hash=request.issuer_key_hash, + issuer_name_hash=request.issuer_name_hash, + hash_algorithm=hashes.SHA1(), + next_update=datetime.now() + timedelta(days=1), + this_update=datetime.now() - timedelta(days=1), + signature_algorithm_oid=x509.oid.SignatureAlgorithmOID.RSA_WITH_SHA1, + ) + + # pylint: disable=line-too-long openssl_confused = ("", """ /etc/letsencrypt/live/example.org/cert.pem: good @@ -165,5 +294,6 @@ revoked """, """Response verify OK""") + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index e61ed2aca..872fb4302 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -12,7 +12,6 @@ import pytz import six import certbot -from certbot import cli from certbot import compat from certbot import errors from certbot.storage import ALL_FOUR @@ -37,6 +36,48 @@ def fill_with_sample_data(rc_object): f.write(kind) +class RelevantValuesTest(unittest.TestCase): + """Tests for certbot.storage.relevant_values.""" + + def setUp(self): + self.values = {"server": "example.org"} + + def _call(self, *args, **kwargs): + from certbot.storage import relevant_values + return relevant_values(*args, **kwargs) + + @mock.patch("certbot.cli.option_was_set") + @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") + def test_namespace(self, mock_find_all, mock_option_was_set): + mock_find_all.return_value = ["certbot-foo:bar"] + mock_option_was_set.return_value = True + + self.values["certbot_foo:bar_baz"] = 42 + self.assertEqual( + self._call(self.values.copy()), self.values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_set(self, mock_option_was_set): + mock_option_was_set.return_value = True + + self.values["allow_subset_of_names"] = True + self.values["authenticator"] = "apache" + self.values["rsa_key_size"] = 1337 + expected_relevant_values = self.values.copy() + self.values["hello"] = "there" + + self.assertEqual(self._call(self.values), expected_relevant_values) + + @mock.patch("certbot.cli.option_was_set") + def test_option_unset(self, mock_option_was_set): + mock_option_was_set.return_value = False + + expected_relevant_values = self.values.copy() + self.values["rsa_key_size"] = 2048 + + self.assertEqual(self._call(self.values), expected_relevant_values) + + class BaseRenewableCertTest(test_util.ConfigTestCase): """Base class for setting up Renewable Cert tests. @@ -563,72 +604,6 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_chown.called) - def _test_relevant_values_common(self, values): - defaults = dict((option, cli.flag_default(option)) - for option in ("authenticator", "installer", - "rsa_key_size", "server",)) - mock_parser = mock.Mock(args=[], verb="plugins", - defaults=defaults) - - # make a copy to ensure values isn't modified - values = values.copy() - values.setdefault("server", defaults["server"]) - expected_server = values["server"] - - from certbot.storage import relevant_values - with mock.patch("certbot.cli.helpful_parser", mock_parser): - rv = relevant_values(values) - self.assertIn("server", rv) - self.assertEqual(rv.pop("server"), expected_server) - return rv - - def test_relevant_values(self): - """Test that relevant_values() can reject an irrelevant value.""" - self.assertEqual( - self._test_relevant_values_common({"hello": "there"}), {}) - - def test_relevant_values_default(self): - """Test that relevant_values() can reject a default value.""" - option = "rsa_key_size" - values = {option: cli.flag_default(option)} - self.assertEqual(self._test_relevant_values_common(values), {}) - - def test_relevant_values_nondefault(self): - """Test that relevant_values() can retain a non-default value.""" - values = {"rsa_key_size": 12} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_bool(self): - values = {"allow_subset_of_names": True} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_str(self): - values = {"authenticator": "apache"} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_plugins_none(self): - self.assertEqual( - self._test_relevant_values_common( - {"authenticator": None, "installer": None}), {}) - - @mock.patch("certbot.cli.set_by_cli") - @mock.patch("certbot.plugins.disco.PluginsRegistry.find_all") - def test_relevant_values_namespace(self, mock_find_all, mock_set_by_cli): - mock_set_by_cli.return_value = True - mock_find_all.return_value = ["certbot-foo:bar"] - values = {"certbot_foo:bar_baz": 42} - self.assertEqual( - self._test_relevant_values_common(values), values) - - def test_relevant_values_server(self): - self.assertEqual( - # _test_relevant_values_common handles testing the server - # value and removes it - self._test_relevant_values_common({"server": "example.org"}), {}) - @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): """Test for new_lineage() class method.""" diff --git a/certbot/tests/testdata/google_certificate.pem b/certbot/tests/testdata/google_certificate.pem new file mode 100644 index 000000000..c26fea0b1 --- /dev/null +++ b/certbot/tests/testdata/google_certificate.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHQjCCBiqgAwIBAgIQCgYwQn9bvO1pVzllk7ZFHzANBgkqhkiG9w0BAQsFADB1 +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMTQwMgYDVQQDEytEaWdpQ2VydCBTSEEyIEV4dGVuZGVk +IFZhbGlkYXRpb24gU2VydmVyIENBMB4XDTE4MDUwODAwMDAwMFoXDTIwMDYwMzEy +MDAwMFowgccxHTAbBgNVBA8MFFByaXZhdGUgT3JnYW5pemF0aW9uMRMwEQYLKwYB +BAGCNzwCAQMTAlVTMRkwFwYLKwYBBAGCNzwCAQITCERlbGF3YXJlMRAwDgYDVQQF +Ewc1MTU3NTUwMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQG +A1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYD +VQQDEwpnaXRodWIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +xjyq8jyXDDrBTyitcnB90865tWBzpHSbindG/XqYQkzFMBlXmqkzC+FdTRBYyneZ +w5Pz+XWQvL+74JW6LsWNc2EF0xCEqLOJuC9zjPAqbr7uroNLghGxYf13YdqbG5oj +/4x+ogEG3dF/U5YIwVr658DKyESMV6eoYV9mDVfTuJastkqcwero+5ZAKfYVMLUE +sMwFtoTDJFmVf6JlkOWwsxp1WcQ/MRQK1cyqOoUFUgYylgdh3yeCDPeF22Ax8AlQ +xbcaI+GwfQL1FB7Jy+h+KjME9lE/UpgV6Qt2R1xNSmvFCBWu+NFX6epwFP/JRbkM +fLz0beYFUvmMgLtwVpEPSwIDAQABo4IDeTCCA3UwHwYDVR0jBBgwFoAUPdNQpdag +re7zSmAKZdMh1Pj41g8wHQYDVR0OBBYEFMnCU2FmnV+rJfQmzQ84mqhJ6kipMCUG +A1UdEQQeMByCCmdpdGh1Yi5jb22CDnd3dy5naXRodWIuY29tMA4GA1UdDwEB/wQE +AwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0fBG4wbDA0 +oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItZXYtc2VydmVyLWcy +LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTItZXYtc2Vy +dmVyLWcyLmNybDBLBgNVHSAERDBCMDcGCWCGSAGG/WwCATAqMCgGCCsGAQUFBwIB +FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAcGBWeBDAEBMIGIBggrBgEF +BQcBAQR8MHowJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBS +BggrBgEFBQcwAoZGaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0 +U0hBMkV4dGVuZGVkVmFsaWRhdGlvblNlcnZlckNBLmNydDAMBgNVHRMBAf8EAjAA +MIIBfgYKKwYBBAHWeQIEAgSCAW4EggFqAWgAdgCkuQmQtBhYFIe7E6LMZ3AKPDWY +BPkb37jjd80OyA3cEAAAAWNBYm0KAAAEAwBHMEUCIQDRZp38cTWsWH2GdBpe/uPT +Wnsu/m4BEC2+dIcvSykZYgIgCP5gGv6yzaazxBK2NwGdmmyuEFNSg2pARbMJlUFg +U5UAdgBWFAaaL9fC7NP14b1Esj7HRna5vJkRXMDvlJhV1onQ3QAAAWNBYm0tAAAE +AwBHMEUCIQCi7omUvYLm0b2LobtEeRAYnlIo7n6JxbYdrtYdmPUWJQIgVgw1AZ51 +vK9ENinBg22FPxb82TvNDO05T17hxXRC2IYAdgC72d+8H4pxtZOUI5eqkntHOFeV +CqtS6BqQlmQ2jh7RhQAAAWNBYm3fAAAEAwBHMEUCIQChzdTKUU2N+XcqcK0OJYrN +8EYynloVxho4yPk6Dq3EPgIgdNH5u8rC3UcslQV4B9o0a0w204omDREGKTVuEpxG +eOQwDQYJKoZIhvcNAQELBQADggEBAHAPWpanWOW/ip2oJ5grAH8mqQfaunuCVE+v +ac+88lkDK/LVdFgl2B6kIHZiYClzKtfczG93hWvKbST4NRNHP9LiaQqdNC17e5vN +HnXVUGw+yxyjMLGqkgepOnZ2Rb14kcTOGp4i5AuJuuaMwXmCo7jUwPwfLe1NUlVB +Kqg6LK0Hcq4K0sZnxE8HFxiZ92WpV2AVWjRMEc/2z2shNoDvxvFUYyY1Oe67xINk +myQKc+ygSBZzyLnXSFVWmHr3u5dcaaQGGAR42v6Ydr4iL38Hd4dOiBma+FXsXBIq +WUjbST4VXmdaol7uzFMojA4zkxQDZAvF5XgJlAFadfySna/teik= +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/google_issuer_certificate.pem b/certbot/tests/testdata/google_issuer_certificate.pem new file mode 100644 index 000000000..50db47bc4 --- /dev/null +++ b/certbot/tests/testdata/google_issuer_certificate.pem @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEXDCCA0SgAwIBAgINAeOpMBz8cgY4P5pTHTANBgkqhkiG9w0BAQsFADBMMSAw +HgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFs +U2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEy +MTUwMDAwNDJaMFQxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3Qg +U2VydmljZXMxJTAjBgNVBAMTHEdvb2dsZSBJbnRlcm5ldCBBdXRob3JpdHkgRzMw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKUkvqHv/OJGuo2nIYaNVW +XQ5IWi01CXZaz6TIHLGp/lOJ+600/4hbn7vn6AAB3DVzdQOts7G5pH0rJnnOFUAK +71G4nzKMfHCGUksW/mona+Y2emJQ2N+aicwJKetPKRSIgAuPOB6Aahh8Hb2XO3h9 +RUk2T0HNouB2VzxoMXlkyW7XUR5mw6JkLHnA52XDVoRTWkNty5oCINLvGmnRsJ1z +ouAqYGVQMc/7sy+/EYhALrVJEA8KbtyX+r8snwU5C1hUrwaW6MWOARa8qBpNQcWT +kaIeoYvy/sGIJEmjR0vFEwHdp1cSaWIr6/4g72n7OqXwfinu7ZYW97EfoOSQJeAz +AgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH +AwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFHfCuFCa +Z3Z2sS3ChtCDoH6mfrpLMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYu +MDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdv +b2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dz +cjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYc +aHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEA +HLeJluRT7bvs26gyAZ8so81trUISd7O45skDUmAge1cnxhG1P2cNmSxbWsoiCt2e +ux9LSD+PAj2LIYRFHW31/6xoic1k4tbWXkDCjir37xTTNqRAMPUyFRWSdvt+nlPq +wnb8Oa2I/maSJukcxDjNSfpDh/Bd1lZNgdd/8cLdsE3+wypufJ9uXO1iQpnh9zbu +FIwsIONGl1p3A8CgxkqI/UAih3JaGOqcpcdaCIzkBaR9uYQ1X4k2Vg5APRLouzVy +7a8IVk6wuy6pm+T7HT4LY8ibS5FEZlfAFLSW8NwsVz9SBK2Vqn1N0PIMn5xA6NZV +c7o835DLAFshEWfC7TIe3g== +-----END CERTIFICATE----- diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 8c5db2c2f..4c0c66a42 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,14 +3,15 @@ .. warning:: This module is not part of the public API. """ -import multiprocessing +import logging import os import pkg_resources import shutil +import stat import tempfile import unittest import sys -import warnings +from multiprocessing import Process, Event from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -23,8 +24,9 @@ from six.moves import reload_module # pylint: disable=import-error from certbot import constants from certbot import interfaces from certbot import storage -from certbot import util from certbot import configuration +from certbot import lock +from certbot import util from certbot.display import util as display_util @@ -211,7 +213,7 @@ class FreezableMock(object): """ def __init__(self, frozen=False, func=None, return_value=mock.sentinel.DEFAULT): - self._frozen_set = set() if frozen else set(('freeze',)) + self._frozen_set = set() if frozen else {'freeze', } self._func = func self._mock = mock.MagicMock() if return_value != mock.sentinel.DEFAULT: @@ -328,22 +330,25 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # On Windows we have various files which are not correctly closed at the time of tearDown. - # For know, we log them until a proper file close handling is written. - # Useful for development only, so no warning when we are on a CI process. - def onerror_handler(_, path, excinfo): - """On error handler""" - if not os.environ.get('APPVEYOR'): # pragma: no cover - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) - shutil.rmtree(self.tempdir, onerror=onerror_handler) + # Cleanup opened resources after a test. This is usually done through atexit handlers in + # Certbot, but during tests, atexit will not run registered functions before tearDown is + # called and instead will run them right before the entire test process exits. + # It is a problem on Windows, that does not accept to clean resources before closing them. + logging.shutdown() + # Remove logging handlers that have been closed so they won't be + # accidentally used in future tests. + logging.getLogger().handlers = [] + util._release_locks() # pylint: disable=protected-access + + def handle_rw_files(_, path, __): + """Handle read-only files, that will fail to be removed on Windows.""" + os.chmod(path, stat.S_IWRITE) + os.remove(path) + shutil.rmtree(self.tempdir, onerror=handle_rw_files) + class ConfigTestCase(TempDirTestCase): - """Test class which sets up a NamespaceConfig object. - - """ + """Test class which sets up a NamespaceConfig object.""" def setUp(self): super(ConfigTestCase, self).setUp() self.config = configuration.NamespaceConfig( @@ -358,47 +363,51 @@ class ConfigTestCase(TempDirTestCase): self.config.chain_path = constants.CLI_DEFAULTS['auth_chain_path'] self.config.server = "https://example.com" -def lock_and_call(func, lock_path): - """Grab a lock for lock_path and call func. - - :param callable func: object to call after acquiring the lock - :param str lock_path: path to file or directory to lock +def _handle_lock(event_in, event_out, path): """ - # Reload module to reset internal _LOCKS dictionary + Acquire a file lock on given path, then wait to release it. This worker is coordinated + using events to signal when the lock should be acquired and released. + :param multiprocessing.Event event_in: event object to signal when to release the lock + :param multiprocessing.Event event_out: event object to signal when the lock is acquired + :param path: the path to lock + """ + if os.path.isdir(path): + my_lock = lock.lock_dir(path) + else: + my_lock = lock.LockFile(path) + try: + event_out.set() + assert event_in.wait(timeout=20), 'Timeout while waiting to release the lock.' + finally: + my_lock.release() + + +def lock_and_call(callback, path_to_lock): + """ + Grab a lock on path_to_lock from a foreign process then execute the callback. + :param callable callback: object to call after acquiring the lock + :param str path_to_lock: path to file or directory to lock + """ + # Reload certbot.util module to reset internal _LOCKS dictionary. reload_module(util) - # start child and wait for it to grab the lock - cv = multiprocessing.Condition() - cv.acquire() - child_args = (cv, lock_path,) - child = multiprocessing.Process(target=hold_lock, args=child_args) - child.start() - cv.wait() + emit_event = Event() + receive_event = Event() + process = Process(target=_handle_lock, args=(emit_event, receive_event, path_to_lock)) + process.start() - # call func and terminate the child - func() - cv.notify() - cv.release() - child.join() - assert child.exitcode == 0 + # Wait confirmation that lock is acquired + assert receive_event.wait(timeout=10), 'Timeout while waiting to acquire the lock.' + # Execute the callback + callback() + # Trigger unlock from foreign process + emit_event.set() -def hold_lock(cv, lock_path): # pragma: no cover - """Acquire a file lock at lock_path and wait to release it. + # Wait for process termination + process.join(timeout=10) + assert process.exitcode == 0 - :param multiprocessing.Condition cv: condition for synchronization - :param str lock_path: path to the file lock - - """ - from certbot import lock - if os.path.isdir(lock_path): - my_lock = lock.lock_dir(lock_path) - else: - my_lock = lock.LockFile(lock_path) - cv.acquire() - cv.notify() - cv.wait() - my_lock.release() def skip_on_windows(reason): """Decorator to skip permanently a test on Windows. A reason is required.""" @@ -407,6 +416,7 @@ def skip_on_windows(reason): return unittest.skipIf(sys.platform == 'win32', reason)(function) return wrapper + def broken_on_windows(function): """Decorator to skip temporarily a broken test on Windows.""" reason = 'Test is broken and ignored on windows but should be fixed.' @@ -415,9 +425,10 @@ def broken_on_windows(function): and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true', reason)(function) + def temp_join(path): """ Return the given path joined to the tempdir path for the current platform Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows) """ - return os.path.join(tempfile.gettempdir(), path) + return os.path.join(tempfile.gettempdir(), path) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 6685b88c6..cac587995 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -2,7 +2,6 @@ import argparse import errno import os -import shutil import unittest import mock @@ -88,7 +87,6 @@ class LockDirUntilExit(test_util.TempDirTestCase): import certbot.util reload_module(certbot.util) - @test_util.broken_on_windows @mock.patch('certbot.util.logger') @mock.patch('certbot.util.atexit_register') def test_it(self, mock_register, mock_logger): @@ -100,11 +98,15 @@ class LockDirUntilExit(test_util.TempDirTestCase): self.assertEqual(mock_register.call_count, 1) registered_func = mock_register.call_args[0][0] - shutil.rmtree(subdir) - registered_func() # exception not raised - # logger.debug is only called once because the second call - # to lock subdir was ignored because it was already locked - self.assertEqual(mock_logger.debug.call_count, 1) + + from certbot import util + # Despite lock_dir_until_exit has been called twice to subdir, its lock should have been + # added only once. So we expect to have two lock references: for self.tempdir and subdir + self.assertTrue(len(util._LOCKS) == 2) # pylint: disable=protected-access + registered_func() # Exception should not be raised + # Logically, logger.debug, that would be invoked in case of unlock failure, + # should never been called. + self.assertEqual(mock_logger.debug.call_count, 0) class SetUpCoreDirTest(test_util.TempDirTestCase): @@ -191,7 +193,12 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_wrong_mode(self): os.chmod(self.tempdir, 0o400) - self.assertFalse(self._call(0o600)) + try: + self.assertFalse(self._call(0o600)) + finally: + # Without proper write permissions, Windows is unable to delete a folder, + # even with admin permissions. Write access must be explicitly set first. + os.chmod(self.tempdir, 0o700) class UniqueFileTest(test_util.TempDirTestCase): @@ -277,20 +284,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for f, _ in items: f.close() - @mock.patch("certbot.util.os.fdopen") - def test_failure(self, mock_fdopen): - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") - - @mock.patch("certbot.util.os.fdopen") - def test_subsequent_failure(self, mock_fdopen): - self._call("wow") - err = OSError("whoops") - err.errno = errno.EIO - mock_fdopen.side_effect = err - self.assertRaises(OSError, self._call, "wow") + def test_failure(self): + with mock.patch("certbot.util.os.open", side_effect=OSError(errno.EIO)): + self.assertRaises(OSError, self._call, "wow") class SafelyRemoveTest(test_util.TempDirTestCase): diff --git a/certbot/util.py b/certbot/util.py index d7c542465..097593e9d 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -62,7 +62,7 @@ def run_script(params, log=logger.error): """Run the script with the given params. :param list params: List of parameters to pass to Popen - :param logging.Logger log: Logger to use for errors + :param callable log: Logger method to use for errors """ try: @@ -142,6 +142,7 @@ def _release_locks(): except: # pylint: disable=bare-except msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) logger.debug(msg, exc_info=True) + _LOCKS.clear() def set_up_core_dir(directory, mode, uid, strict): @@ -225,9 +226,8 @@ def safe_open(path, mode="w", chmod=None, buffering=None): fdopen_args = () # type: Union[Tuple[()], Tuple[int]] if buffering is not None: fdopen_args = (buffering,) - return os.fdopen( - os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args), - mode, *fdopen_args) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) + return os.fdopen(fd, mode, *fdopen_args) def _unique_file(path, filename_pat, count, chmod, mode): diff --git a/docs/ciphers.rst b/docs/ciphers.rst index 1b320cdf9..b748dd87a 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -227,7 +227,7 @@ BetterCrypto.org BetterCrypto.org, a collaboration of mostly European IT security experts, has published a draft paper, "Applied Crypto Hardening" -https://bettercrypto.org/static/applied-crypto-hardening.pdf +https://bettercrypto.org/ FF-DHE Internet-Draft ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/cli-help.txt b/docs/cli-help.txt index e95b2fcd5..ed7794c3b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.31.0 + "". (default: CertbotACMEClient/0.32.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -477,7 +477,9 @@ plugins: using Sakura Cloud for DNS). (default: False) apache: - Apache Web Server plugin + Apache Web Server plugin (Please note that the default values of the + Apache plugin options change depending on the operating system Certbot is + run on.) --apache-enmod APACHE_ENMOD Path to the Apache 'a2enmod' binary (default: None) @@ -496,7 +498,7 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2/other) + /etc/apache2) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you (Only Ubuntu/Debian currently) (default: False) @@ -505,7 +507,7 @@ apache: Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apachectl) + apache2ctl) dns-cloudflare: Obtain certificates using a DNS TXT record (if you are using Cloudflare diff --git a/docs/install.rst b/docs/install.rst index 35b262482..eae40c1f0 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,6 +11,8 @@ About Certbot *Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server. +System administrators can use Certbot directly to request certificates; they should *not* allow unprivileged users to run arbitrary Certbot commands as ``root``, because Certbot allows its user to specify arbitrary file locations and run arbitrary scripts. + Certbot is packaged for many common operating systems and web servers. Check whether ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting certbot.eff.org_, where you will also find the correct installation instructions for diff --git a/docs/using.rst b/docs/using.rst index e17e56b64..de17742a1 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -277,7 +277,7 @@ Plugin Auth Inst Notes plesk_ Y Y Integration with the Plesk web hosting tool haproxy_ Y Y Integration with the HAProxy load balancer s3front_ Y Y Integration with Amazon CloudFront distribution of S3 buckets -gandi_ Y Y Integration with Gandi's hosting products and API +gandi_ Y Y Integration with Gandi LiveDNS API varnish_ Y N Obtain certificates via a Varnish server external_ Y N A plugin for convenient scripting (See also ticket 2782_) icecast_ N Y Deploy certificates to Icecast 2 streaming media servers @@ -290,7 +290,7 @@ heroku_ Y Y Integration with Heroku SSL .. _plesk: https://github.com/plesk/letsencrypt-plesk .. _haproxy: https://github.com/greenhost/certbot-haproxy .. _s3front: https://github.com/dlapiduz/letsencrypt-s3front -.. _gandi: https://github.com/Gandi/letsencrypt-gandi +.. _gandi: https://github.com/obynio/certbot-plugin-gandi .. _icecast: https://github.com/e00E/lets-encrypt-icecast .. _varnish: http://git.sesse.net/?p=letsencrypt-varnish-plugin .. _2782: https://github.com/certbot/certbot/issues/2782 diff --git a/letsencrypt-auto b/letsencrypt-auto index 03c2994e3..0c82a7437 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.31.0" +LE_AUTO_VERSION="0.32.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -521,10 +521,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -1034,26 +1044,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1190,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1231,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1250,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1272,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1353,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1364,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1382,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index d95f9abcb..eeb8d7d12 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlxcop8ACgkQTRfJlc2X -dfIbZwf/faKu7IjLi0qFQ+kw8zaAnV47JDgfWqbR5GSdwWPqld+QyHlcRfPgwYma -fKj9+g/FvPNPSfjHRRCoFrYvpZ4lZ+f4HPN9+OjydfM77rdDhVDwzs8dbKIk02yU -0IEJhXj5Q9hF3TSDZcyXAJdBU1lz51ohtVIXelMBPmzhYPCZF47iE9/k9pApQi86 -RTji7hxPcF/n7mzXrbyTvk+kDxSdDlE0Eg9syK7XaFDBTa2lqgG8wTnMPVqhc/hm -WM/uwkzbYarjy05ffV1kM683nP0rECnHlYT38pYcT2puw2kn/QthwR5j/jB/DWSc -94Kw7BeMH651V8EaNwYIiouylnVH3A== -=U+Qh +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlyAMZ0ACgkQTRfJlc2X +dfLItQf/SNv+at1Pw5oiEbWleNPpmz9srlkf9AHU92Hh3p7+OljcaWQindtCYtlO +UvWV/CjGzObmJvO+Pgy2epNhD8cTjPamI46l5UG2nwvy8V+JemS937Ae6paivt8T +/RaFKfyNDfxBjQhHS1ypVuRrFgAQ5CG0iGuJSMgwLpcKCZyKAim3+Vb57+Esq+zG +Cp7GmJk9h1z5FbNukbaFHBlJQIefJoQclh1yUw11pLab0uxIOdc9WiEWLLAVE512 +SMQM2sNv49uh7mRnxW+6WU6dor6JI9Ff1L4D5hfglzBJRM4qLU/hGv54oVycWq4i +eFjtqDfo5XMwnbnUnVkEB73pY6lzDg== +=YaE3 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 02656521f..2a44a58dc 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.32.0.dev0" +LE_AUTO_VERSION="0.33.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -488,11 +488,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools @@ -521,10 +528,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ @@ -731,7 +748,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 @@ -1034,26 +1054,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -1180,18 +1200,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1221,7 +1241,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -1241,7 +1260,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1263,7 +1282,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -1344,7 +1363,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -1354,12 +1374,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -1372,7 +1392,7 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 3ecca1510..17eea3eba 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2cd3f4336..29c54bf2d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -323,7 +323,10 @@ elif [ -f /etc/redhat-release ]; then prev_le_python="$LE_PYTHON" unset LE_PYTHON DeterminePythonVersion "NOCRASH" - if [ "$PYVER" -eq 26 ]; then + # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. + RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` + RPM_DIST_VERSION=`(. /etc/os-release 2> /dev/null && echo $VERSION_ID) || echo "0"` + if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 -o "$PYVER" -eq 26 ]; then Bootstrap() { BootstrapMessage "RedHat-based OSes that will use Python3" BootstrapRpmPython3 diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh index b011a7235..f33b07ca9 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh @@ -5,11 +5,18 @@ BOOTSTRAP_RPM_PYTHON3_VERSION=1 BootstrapRpmPython3() { # Tested with: # - CentOS 6 + # - Fedora 29 InitializeRPMCommonBase + # Fedora 29 must use python3-virtualenv + if $TOOL list python3-virtualenv >/dev/null 2>&1; then + python_pkgs="python3 + python3-virtualenv + python3-devel + " # EPEL uses python34 - if $TOOL list python34 >/dev/null 2>&1; then + elif $TOOL list python34 >/dev/null 2>&1; then python_pkgs="python34 python34-devel python34-tools diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh index c531cbe99..ac66119c3 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh @@ -14,10 +14,20 @@ BootstrapSuseCommon() { QUIET_FLAG='-qq' fi + if zypper search -x python-virtualenv >/dev/null 2>&1; then + OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" + else + # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv + # is a source package, and python2-virtualenv must be used instead. + # Also currently python2-setuptools is not a dependency of python2-virtualenv, + # while it should be. Installing it explicitly until upstreqm fix. + OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" + fi + zypper $QUIET_FLAG $zypper_flags in $install_flags \ python \ python-devel \ - python-virtualenv \ + $OPENSUSE_VIRTUALENV_PACKAGES \ gcc \ augeas-lenses \ libopenssl-devel \ diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index 514d3271b..3e0dbde7e 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.31.0 \ - --hash=sha256:1a1b4b2675daf5266cc2cf2a44ded44de1d83e9541ffa078913c0e4c3231a1c4 \ - --hash=sha256:0c3196f80a102c0f9d82d566ba859efe3b70e9ed4670520224c844fafd930473 -acme==0.31.0 \ - --hash=sha256:a0c851f6b7845a0faa3a47a3e871440eed9ec11b4ab949de0dc4a0fb1201cd24 \ - --hash=sha256:7e5c2d01986e0f34ca08fee58981892704c82c48435dcd3592b424c312d8b2bf -certbot-apache==0.31.0 \ - --hash=sha256:740bb55dd71723a21eebabb16e6ee5d8883f8b8f8cf6956dd1d4873e0cccae21 \ - --hash=sha256:cc4b840b2a439a63e2dce809272c3c3cd4b1aeefc4053cd188935135be137edd -certbot-nginx==0.31.0 \ - --hash=sha256:7a1ffda9d93dc7c2aaf89452ce190250de8932e624d31ebba8e4fa7d950025c5 \ - --hash=sha256:d450d75650384f74baccb7673c89e2f52468afa478ed354eb6d4b99aa33bf865 +certbot==0.32.0 \ + --hash=sha256:75fd986ae42cd90bde6400c5f5a0dd936a7f4a42a416146b1e8bb0f92028b443 \ + --hash=sha256:c0b94e25a07d83809d98029f09e9b501f86ec97624f45ce86800a7002488c3c8 +acme==0.32.0 \ + --hash=sha256:88b2d2741e5ea028c590a33b16fb647cb74af6b2db6c7909c738a48f879efdec \ + --hash=sha256:0eefce8b7880eb7eccc049a6b8ba262fc624bc34b3a8581d05b82f2bb39f1aec +certbot-apache==0.32.0 \ + --hash=sha256:b2c82b7a1c44799ba3a150970513ed4fa9afeee40e326440800b1243f917ddb6 \ + --hash=sha256:68072775f1bb4bc9fc64cabe051a761f6dbf296012512eff7819144ac8b9ec97 +certbot-nginx==0.32.0 \ + --hash=sha256:3fc3664231586565d886ddcb679c95a2fb2494a2ce3e028149f1496dca5b47cf \ + --hash=sha256:82c43cd26aacc2eb0ae890be6a2f74d726b6dcb4ee7b63c0e55ec33e576f3e84 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 1fac78836..dff57dfd5 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -60,26 +60,26 @@ ConfigArgParse==0.12.0 \ configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \ --no-binary configobj -cryptography==2.2.2 \ - --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \ - --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \ - --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \ - --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \ - --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \ - --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \ - --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \ - --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \ - --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \ - --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \ - --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \ - --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \ - --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \ - --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \ - --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \ - --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \ - --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \ - --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \ - --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887 +cryptography==2.5 \ + --hash=sha256:9e29af877c29338f0cab5f049ccc8bd3ead289a557f144376c4fbc7d1b98914f \ + --hash=sha256:b13c80b877e73bcb6f012813c6f4a9334fcf4b0e96681c5a15dac578f2eedfa0 \ + --hash=sha256:8504661ffe324837f5c4607347eeee4cf0fcad689163c6e9c8d3b18cf1f4a4ad \ + --hash=sha256:e091bd424567efa4b9d94287a952597c05d22155a13716bf5f9f746b9dc906d3 \ + --hash=sha256:42fad67d7072216a49e34f923d8cbda9edacbf6633b19a79655e88a1b4857063 \ + --hash=sha256:9a30384cc402eac099210ab9b8801b2ae21e591831253883decdb4513b77a3cd \ + --hash=sha256:08b753df3672b7066e74376f42ce8fc4683e4fd1358d34c80f502e939ee944d2 \ + --hash=sha256:6f841c7272645dd7c65b07b7108adfa8af0aaea57f27b7f59e01d41f75444c85 \ + --hash=sha256:bfe66b577a7118e05b04141f0f1ed0959552d45672aa7ecb3d91e319d846001e \ + --hash=sha256:522fdb2809603ee97a4d0ef2f8d617bc791eb483313ba307cb9c0a773e5e5695 \ + --hash=sha256:05b3ded5e88747d28ee3ef493f2b92cbb947c1e45cf98cfef22e6d38bb67d4af \ + --hash=sha256:fa2b38c8519c5a3aa6e2b4e1cf1a549b54acda6adb25397ff542068e73d1ed00 \ + --hash=sha256:ab50da871bc109b2d9389259aac269dd1b7c7413ee02d06fe4e486ed26882159 \ + --hash=sha256:9260b201ce584d7825d900c88700aa0bd6b40d4ebac7b213857bd2babee9dbca \ + --hash=sha256:06826e7f72d1770e186e9c90e76b4f84d90cdb917b47ff88d8dc59a7b10e2b1e \ + --hash=sha256:2cd29bd1911782baaee890544c653bb03ec7d95ebeb144d714b0f5c33deb55c7 \ + --hash=sha256:7d335e35306af5b9bc0560ca39f740dfc8def72749645e193dd35be11fb323b3 \ + --hash=sha256:31e5637e9036d966824edaa91bf0aa39dc6f525a1c599f39fd5c50340264e079 \ + --hash=sha256:4946b67235b9d2ea7d31307be9d5ad5959d6c4a8f98f900157b47abddf698401 enum34==1.1.2 ; python_version < '3.4' \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index 6a00dd9cb..f04ebe6fe 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -23,7 +23,6 @@ from distutils.version import StrictVersion from hashlib import sha256 from os import environ from os.path import join -from pipes import quote from shutil import rmtree try: from subprocess import check_output @@ -43,7 +42,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +import sys from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -65,7 +64,7 @@ maybe_argparse = ( [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' 'argparse-1.4.0.tar.gz', '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if version_info < (2, 7, 0) else []) + if sys.version_info < (2, 7, 0) else []) PACKAGES = maybe_argparse + [ @@ -146,7 +145,8 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + python = sys.executable or 'python' + pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) has_pip_cache = pip_version >= StrictVersion('6.0') index_base = get_index_base() @@ -156,12 +156,12 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - # Disable cache since we're not using it and it otherwise - # sometimes throws permission warnings: - ('--no-cache-dir ' if has_pip_cache else '') + - ' '.join(quote(d) for d in downloads), - shell=True) + # On Windows, pip self-upgrade is not possible, it must be done through python interpreter. + command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] + # Disable cache since it is not used and it otherwise sometimes throws permission warnings: + command.extend(['--no-cache-dir'] if has_pip_cache else []) + command.extend(downloads) + check_output(command) except HashError as exc: print(exc) except Exception: @@ -174,4 +174,4 @@ def main(): if __name__ == '__main__': - exit(main()) + sys.exit(main()) diff --git a/setup.py b/setup.py index 14fef37f3..26a1c4293 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ import codecs import os import re +import sys from setuptools import find_packages, setup +from setuptools.command.test import test as TestCommand # Workaround for http://bugs.python.org/issue8876, see # http://bugs.python.org/issue8876#msg208792 @@ -77,6 +79,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] + +class PyTest(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def run_tests(self): + import shlex + # import here, cause outside the eggs aren't loaded + import pytest + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + + setup( name='certbot', version=version, @@ -123,6 +141,8 @@ setup( # to test all packages run "python setup.py test -s # {acme,certbot_apache,certbot_nginx}" test_suite='certbot', + tests_require=["pytest"], + cmdclass={"test": PyTest}, entry_points={ 'console_scripts': [ diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 630571148..05b144df4 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -15,9 +15,11 @@ command -v python > /dev/null || (echo "Error, python executable is not in the P . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx +CURRENT_DIR="$(pwd)" cleanup_and_exit() { EXIT_STATUS=$? + cd $CURRENT_DIR if SERVER_STILL_RUNNING=`ps -p $python_server_pid -o pid=` then echo Kill server subprocess, left running by abnormal exit @@ -526,4 +528,57 @@ if [ "${BOULDER_INTEGRATION:-v1}" = "v2" ]; then --manual-cleanup-hook ./tests/manual-dns-cleanup.sh fi +# Test OCSP status + +## OCSP 1: Check stale OCSP status +pushd ./tests/integration + +OUT=`common certificates --config-dir sample-config` +TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` +EXPIRED=`echo "$OUT" | grep EXPIRED | wc -l` + +if [ "$TEST_CERTS" != 2 ] ; then + echo "Did not find two test certs as expected ($TEST_CERTS)" + exit 1 +fi + +if [ "$EXPIRED" != 2 ] ; then + echo "Did not find two test certs as expected ($EXPIRED)" + exit 1 +fi + +popd + +## OSCP 2: Check live certificate OCSP status (VALID) +common --domains le-ocsp-check.wtf +OUT=`common certificates` +VALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep VALID | wc -l` +EXPIRED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep EXPIRED | wc -l` + +if [ "$VALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be VALID" + exit 1 +fi + +if [ "$EXPIRED" != 0 ] ; then + echo "Did not expect le-ocsp-check.wtf to be EXPIRED" + exit 1 +fi + +## OSCP 3: Check live certificate OCSP status (REVOKED) +common revoke --cert-name le-ocsp-check.wtf --no-delete-after-revoke +OUT=`common certificates` +INVALID=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep INVALID | wc -l` +REVOKED=`echo $OUT | grep 'Domains: le-ocsp-check.wtf' -A 1 | grep REVOKED | wc -l` + +if [ "$INVALID" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be INVALID" + exit 1 +fi + +if [ "$REVOKED" != 1 ] ; then + echo "Expected le-ocsp-check.wtf to be REVOKED" + exit 1 +fi + coverage report --fail-under 64 --include 'certbot/*' --show-missing diff --git a/tests/display.py b/tests/display.py deleted file mode 100644 index 1f548e33d..000000000 --- a/tests/display.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Manual test of display functions.""" -import sys - -from certbot.display import util -from certbot.tests.display import util_test - - -def test_visual(displayer, choices): - """Visually test all of the display functions.""" - displayer.notification("Random notification!") - displayer.menu("Question?", choices, - ok_label="O", cancel_label="Can", help_label="??") - displayer.menu("Question?", [choice[1] for choice in choices], - ok_label="O", cancel_label="Can", help_label="??") - displayer.input("Input Message") - displayer.yesno("YesNo Message", yes_label="Yessir", no_label="Nosir") - displayer.checklist("Checklist Message", [choice[0] for choice in choices]) - - -if __name__ == "__main__": - displayer = util.FileDisplay(sys.stdout, False) - test_visual(displayer, util_test.CHOICES) diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/meta.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/private_key.json diff --git a/tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json b/tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json similarity index 100% rename from tests/letstest/testdata/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json rename to tests/integration/sample-config/accounts/acme-staging.api.letsencrypt.org/directory/48d6b9e8d767eccf7e4d877d6ffa81e3/regr.json diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/a.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/a.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/cert1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/cert1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/chain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/chain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/fullchain1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/fullchain1.pem diff --git a/tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem b/tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem similarity index 100% rename from tests/letstest/testdata/sample-config/archive/b.encryption-example.com/privkey1.pem rename to tests/integration/sample-config/archive/b.encryption-example.com/privkey1.pem diff --git a/tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem b/tests/integration/sample-config/csr/0000_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0000_csr-certbot.pem rename to tests/integration/sample-config/csr/0000_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem b/tests/integration/sample-config/csr/0001_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0001_csr-certbot.pem rename to tests/integration/sample-config/csr/0001_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem b/tests/integration/sample-config/csr/0002_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0002_csr-certbot.pem rename to tests/integration/sample-config/csr/0002_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem b/tests/integration/sample-config/csr/0003_csr-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/csr/0003_csr-certbot.pem rename to tests/integration/sample-config/csr/0003_csr-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem b/tests/integration/sample-config/keys/0000_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0000_key-certbot.pem rename to tests/integration/sample-config/keys/0000_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem b/tests/integration/sample-config/keys/0001_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0001_key-certbot.pem rename to tests/integration/sample-config/keys/0001_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem b/tests/integration/sample-config/keys/0002_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0002_key-certbot.pem rename to tests/integration/sample-config/keys/0002_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem b/tests/integration/sample-config/keys/0003_key-certbot.pem similarity index 100% rename from tests/letstest/testdata/sample-config/keys/0003_key-certbot.pem rename to tests/integration/sample-config/keys/0003_key-certbot.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/README b/tests/integration/sample-config/live/a.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/README rename to tests/integration/sample-config/live/a.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem b/tests/integration/sample-config/live/a.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/a.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem b/tests/integration/sample-config/live/a.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/a.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/a.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/a.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/a.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/README b/tests/integration/sample-config/live/b.encryption-example.com/README similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/README rename to tests/integration/sample-config/live/b.encryption-example.com/README diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem b/tests/integration/sample-config/live/b.encryption-example.com/cert.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/cert.pem rename to tests/integration/sample-config/live/b.encryption-example.com/cert.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem b/tests/integration/sample-config/live/b.encryption-example.com/chain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/chain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/chain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem b/tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/fullchain.pem rename to tests/integration/sample-config/live/b.encryption-example.com/fullchain.pem diff --git a/tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem b/tests/integration/sample-config/live/b.encryption-example.com/privkey.pem similarity index 100% rename from tests/letstest/testdata/sample-config/live/b.encryption-example.com/privkey.pem rename to tests/integration/sample-config/live/b.encryption-example.com/privkey.pem diff --git a/tests/letstest/testdata/sample-config/options-ssl-apache.conf b/tests/integration/sample-config/options-ssl-apache.conf similarity index 100% rename from tests/letstest/testdata/sample-config/options-ssl-apache.conf rename to tests/integration/sample-config/options-ssl-apache.conf diff --git a/tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf b/tests/integration/sample-config/renewal/a.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/a.encryption-example.com.conf rename to tests/integration/sample-config/renewal/a.encryption-example.com.conf diff --git a/tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf b/tests/integration/sample-config/renewal/b.encryption-example.com.conf similarity index 100% rename from tests/letstest/testdata/sample-config/renewal/b.encryption-example.com.conf rename to tests/integration/sample-config/renewal/b.encryption-example.com.conf diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 8babc67b3..b8ae937ad 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -564,6 +564,11 @@ try: ii, target, status = outq print('%d %s %s'%(ii, target['name'], status)) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + if len(outputs) != num_processes: + failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ + 'Tests should be rerun.' + print(failure_message) + results_file.write(failure_message + '\n') results_file.close() finally: diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh index 2cbe66a83..081ff3829 100755 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh @@ -17,27 +17,6 @@ letsencrypt-auto certonly --no-self-upgrade -v --standalone --debug \ --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -# we have to jump through some hoops to cope with relative paths in renewal -# conf files ... -# 1. be in the right directory -cd tests/letstest/testdata/ - -# 2. refer to the config with the same level of relativity that it itself -# contains :/ -OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade` -TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l` -REVOKED=`echo "$OUT" | grep REVOKED | wc -l` - -if [ "$TEST_CERTS" != 2 ] ; then - echo "Did not find two test certs as expected ($TEST_CERTS)" - exit 1 -fi - -if [ "$REVOKED" != 1 ] ; then - echo "Did not find one revoked cert as expected ($REVOKED)" - exit 1 -fi - if ! letsencrypt-auto --help --no-self-upgrade | grep -F "letsencrypt-auto [SUBCOMMAND]"; then echo "letsencrypt-auto not included in help output!" exit 1 diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index 0b9a91ffd..260a0acfb 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,6 +12,8 @@ export VENV_ARGS="-p $PYTHON" # setup venv tools/_venv_common.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate +# pytest is needed to run tests on some of our packages so we install a pinned version here. +tools/pip_install.py pytest # build sdists for pkg_dir in acme . $PLUGINS; do diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index e6ab836b8..d5fd6e14a 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -1,18 +1,20 @@ #!/bin/sh -xe -LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto" +REPO_ROOT="letsencrypt" +LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot_apache certbot_nginx" +PIP_INSTALL="$REPO_ROOT/tools/pip_install.py" VENV_NAME=venv # *-auto respects VENV_PATH $LE_AUTO --os-packages-only LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version . $VENV_NAME/bin/activate +"$PIP_INSTALL" pytest # change to an empty directory to ensure CWD doesn't affect tests cd $(mktemp -d) -pip install pytest==3.2.5 for module in $MODULES ; do echo testing $module diff --git a/tests/lock_test.py b/tests/lock_test.py index 0266cf029..aaa8ce2d9 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -2,6 +2,7 @@ from __future__ import print_function import atexit +import datetime import functools import logging import os @@ -11,6 +12,13 @@ import subprocess import sys import tempfile +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +# TODO: once mypy has cryptography types bundled, type: ignore can be removed. +# See https://github.com/python/typeshed/tree/master/third_party/2/cryptography +from cryptography.hazmat.primitives import serialization, hashes # type: ignore +from cryptography.hazmat.primitives.asymmetric import rsa + from certbot import lock from certbot import util @@ -102,12 +110,11 @@ def set_up_nginx_dir(root_path): repo_root = check_call('git rev-parse --show-toplevel'.split()).strip() conf_script = os.path.join( repo_root, 'certbot-nginx', 'tests', 'boulder-integration.conf.sh') - # boulder-integration.conf.sh uses the root environment variable as - # the Nginx server root when writing paths - os.environ['root'] = root_path + # Prepare self-signed certificates for Nginx + key_path, cert_path = setup_certificate(root_path) + # Generate Nginx configuration with open(os.path.join(root_path, 'nginx.conf'), 'w') as f: - f.write(check_call(['/bin/sh', conf_script])) - del os.environ['root'] + f.write(check_call(['/bin/sh', conf_script, root_path, key_path, cert_path])) def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): @@ -134,6 +141,51 @@ def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): config_dir, logs_dir, work_dir, nginx_dir).split()) +def setup_certificate(workspace): + """Generate a self-signed certificate for nginx. + :param workspace: path of folder where to put the certificate + :return: tuple containing the key path and certificate path + :rtype: `tuple` + """ + # Generate key + # See comment on cryptography import about type: ignore + private_key = rsa.generate_private_key( # type: ignore + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + subject = issuer = x509.Name([ + x509.NameAttribute(x509.NameOID.COMMON_NAME, u'nginx.wtf') + ]) + certificate = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + private_key.public_key() + ).serial_number( + 1 + ).not_valid_before( + datetime.datetime.utcnow() + ).not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=1) + ).sign(private_key, hashes.SHA256(), default_backend()) + + key_path = os.path.join(workspace, 'cert.key') + with open(key_path, 'wb') as file_handle: + file_handle.write(private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + )) + + cert_path = os.path.join(workspace, 'cert.pem') + with open(cert_path, 'wb') as file_handle: + file_handle.write(certificate.public_bytes(serialization.Encoding.PEM)) + + return key_path, cert_path + + def test_command(command, directories): """Assert Certbot acquires locks in a specific order. diff --git a/tests/pebble-fetch.sh b/tests/pebble-fetch.sh index b0ba08961..6b562eec2 100755 --- a/tests/pebble-fetch.sh +++ b/tests/pebble-fetch.sh @@ -2,7 +2,7 @@ # Download and run Pebble instance for integration testing set -xe -PEBBLE_VERSION=2018-11-02 +PEBBLE_VERSION=v1.0.1 # We reuse the same GOPATH-style directory than for Boulder. # Pebble does not need it, but it will make the installation consistent with Boulder's one. @@ -13,15 +13,32 @@ mkdir -p ${PEBBLEPATH} cat << UNLIKELY_EOF > "$PEBBLEPATH/docker-compose.yml" version: '3' - services: - pebble: - image: letsencrypt/pebble:${PEBBLE_VERSION} - command: pebble -strict ${PEBBLE_STRICT:-false} -dnsserver 10.77.77.1 - ports: - - 14000:14000 - environment: + pebble: + image: letsencrypt/pebble:${PEBBLE_VERSION} + command: pebble -dnsserver 10.30.50.3:8053 + environment: - PEBBLE_VA_NOSLEEP=1 + ports: + - 14000:14000 + networks: + acmenet: + ipv4_address: 10.30.50.2 + challtestsrv: + image: letsencrypt/pebble-challtestsrv:${PEBBLE_VERSION} + command: pebble-challtestsrv -defaultIPv6 "" -defaultIPv4 10.30.50.1 + ports: + - 8055:8055 + networks: + acmenet: + ipv4_address: 10.30.50.3 +networks: + acmenet: + driver: bridge + ipam: + driver: default + config: + - subnet: 10.30.50.0/24 UNLIKELY_EOF docker-compose -f "$PEBBLEPATH/docker-compose.yml" up -d pebble diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 540842773..09383b4c0 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -107,13 +107,13 @@ def subprocess_with_print(cmd, env=os.environ, shell=False): subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_bin_path(venv_path): +def get_venv_python_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return os.path.abspath(os.path.dirname(python_linux)) + return os.path.abspath(python_linux) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return os.path.abspath(os.path.dirname(python_windows)) + return os.path.abspath(python_windows) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -149,17 +149,12 @@ def main(venv_name, venv_args, args): command.extend(shlex.split(venv_args)) subprocess_with_print(command) - # We execute the following commands in the context of the virtual environment, to install - # the packages in it. To do so, we append the venv binary to the PATH that will be used for - # these commands. With this trick, correct python executable will be selected. - new_environ = os.environ.copy() - new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) - subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), - env=new_environ, shell=True) - subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', - env=new_environ, shell=True) - subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), - env=new_environ, shell=True) + # Using the python executable from venv, we ensure to execute following commands in this venv. + py_venv = get_venv_python_path(venv_name) + subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + command = [py_venv, os.path.abspath('tools/pip_install.py')] + command.extend(args) + subprocess_with_print(command) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -179,6 +174,4 @@ def main(venv_name, venv_args, args): if __name__ == '__main__': - main('venv', - '', - sys.argv[1:]) + main('venv', '', sys.argv[1:]) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..539791a69 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,4 +1,4 @@ -# Specifies Python package versions for development. +# Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). @@ -51,6 +51,7 @@ pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.22.5 +pytest-sugar==0.9.2 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b15c8eca5..288226527 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -15,7 +15,7 @@ import subprocess import re SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] def call_with_print(command, cwd=None): diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 4205e6bcf..0d41d12c4 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,27 +10,36 @@ from __future__ import print_function import sys -def read_file(file_path): - """Reads in a Python requirements file. +def process_entries(entries): + """ Ignore empty lines, comments and editable requirements - :param str file_path: path to requirements file + :param list entries: List of entries :returns: mapping from a project to its pinned version :rtype: dict - """ data = {} - with open(file_path) as file_h: - for line in file_h: - line = line.strip() - if line and not line.startswith('#') and not line.startswith('-e'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - data[project] = version + for e in entries: + e = e.strip() + if e and not e.startswith('#') and not e.startswith('-e'): + project, version = e.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(e)) + data[project] = version return data +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: list of entries in the file + :rtype: list + + """ + with open(file_path) as file_h: + return file_h.readlines() def output_requirements(requirements): """Prepare print requirements to stdout. @@ -46,14 +55,25 @@ def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier - files. + files. Files are read from file paths passed from the command line arguments. - :param tuple paths: paths to requirements files + If no command line arguments are defined, data is read from stdin instead. + + :param tuple paths: paths to requirements files provided on command line """ data = {} - for path in paths: - data.update(read_file(path)) + if paths: + for path in paths: + data.update(process_entries(read_file(path))) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) + return output_requirements(data) diff --git a/tools/pip_install.py b/tools/pip_install.py index dd6302b48..15dc2f0c0 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -19,6 +19,7 @@ import tempfile import merge_requirements as merge_module import readlink +import strip_hashes def find_tools_path(): @@ -47,10 +48,8 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - for line in data: - search = re.search(r'^(\S*==\S*).*$', line) - if search: - fd.write('{0}{1}'.format(search.group(1), os.linesep)) + data = "\n".join(strip_hashes.process_entries(data)) + fd.write(data) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py new file mode 100755 index 000000000..988e72eb8 --- /dev/null +++ b/tools/strip_hashes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +"""Removes hash information from requirement files passed to it as file path +arguments or simply piped to stdin.""" + +import re +import sys + + +def process_entries(entries): + """Strips off hash strings from dependencies. + + :param list entries: List of entries + + :returns: list of dependencies without hashes + :rtype: list + """ + out_lines = [] + for e in entries: + e = e.strip() + search = re.search(r'^(\S*==\S*).*$', e) + if search: + out_lines.append(search.group(1)) + return out_lines + +def main(*paths): + """ + Reads dependency definitions from a (list of) file(s) provided on the + command line. If no command line arguments are present, data is read from + stdin instead. + + Hashes are removed from returned entries. + """ + + deps = [] + if paths: + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) + + return "\n".join(deps) + +if __name__ == '__main__': + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tox.cover.py b/tox.cover.py index 008424641..6f5392b71 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -12,9 +12,12 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] COVER_THRESHOLDS = { - 'certbot': {'linux': 98, 'windows': 94}, + 'certbot': {'linux': 98, 'windows': 93}, 'acme': {'linux': 100, 'windows': 99}, - 'certbot_apache': {'linux': 100, 'windows': 100}, + # certbot_apache coverage not being at 100% is a workaround for + # https://github.com/certbot/certbot/issues/6813. We should increase + # the minimum coverage back to 100% when this issue is resolved. + 'certbot_apache': {'linux': 99, 'windows': 99}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, @@ -35,7 +38,8 @@ COVER_THRESHOLDS = { } SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + 'certbot-apache', 'certbot-postfix', 'letshelp-certbot'] + def cover(package): threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] @@ -54,6 +58,7 @@ def cover(package): sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) + def main(): description = """ This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order @@ -77,5 +82,6 @@ Option -e makes sure we fail fast and don't submit to codecov.""" for package in packages: cover(package) + if __name__ == '__main__': main() diff --git a/tox.ini b/tox.ini index b66e330da..2c5fe0644 100644 --- a/tox.ini +++ b/tox.ini @@ -166,7 +166,6 @@ passenv = HOME GOPATH PEBBLEPATH - PEBBLE_STRICT setenv = SERVER=https://localhost:14000/dir @@ -251,3 +250,15 @@ commands = whitelist_externals = docker-compose passenv = DOCKER_* + +[testenv:integration] +commands = + {[base]pip_install} acme . certbot-nginx certbot-ci + pytest {toxinidir}/certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} \ + --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ + --cov-config={toxinidir}/certbot-ci/certbot_integration_tests/.coveragerc \ + -W 'ignore:Unverified HTTPS request' + coverage report --fail-under=65 --show-missing +passenv = + DOCKER_*