Merge branch 'master' into test-everything-warnings-3

.travis.yml

@@ -48,10 +48,7 @@ matrix:
       sudo: required
       services: docker
     - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,postfix}-oldest'
-    - sudo: required
-      env: TOXENV=apache_compat
-      services: docker
+      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
     - sudo: required
       env: TOXENV=nginx_compat
       services: docker
@@ -157,7 +154,7 @@ script:
     - travis_retry tox
     - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
-after_success: '[ "$TOXENV" == "cover" ] && codecov'
+after_success: '[ "$TOXENV" == "py27-cover" ] && codecov'
 
 notifications:
   email: false

CHANGELOG.md

@@ -2,11 +2,13 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
-## 0.28.0 - master
+## 0.29.0 - master
 
 ### Added
 
-* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`.
+* Noninteractive renewals with `certbot renew` (those not started from a
+  terminal) now randomly sleep 1-480 seconds before beginning work in
+  order to spread out load spikes on the server side.
 
 ### Changed
 
@@ -14,8 +16,67 @@ Certbot adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+*
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+*
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/62?closed=1
+
+## 0.28.0 - 2018-11-7
+
+### Added
+
+* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`.
+* Use the ACMEv2 newNonce endpoint when a new nonce is needed, and newNonce is available in the directory.
+
+### Changed
+
+* Removed documentation mentions of `#letsencrypt` IRC on Freenode.
+* Write README to the base of (config-dir)/live directory
+* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges.
+* Warn when using deprecated acme.challenges.TLSSNI01
+* Log warning about TLS-SNI deprecation in Certbot
+* Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins
+* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies
+* Default time the Linode plugin waits for DNS changes to propogate is now 1200 seconds.
+
+### Fixed
+
 * Match Nginx parser update in allowing variable names to start with `${`.
+* Fix ranking of vhosts in Nginx so that all port-matching vhosts come first
 * Correct OVH integration tests on machines without internet access.
+* Stop caching the results of ipv6_info in http01.py
+* Test fix for Route53 plugin to prevent boto3 making outgoing connections.
+* The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors.
+* The CloudXNS, DNSimple, DNS Made Easy, Gehirn, Linode, LuaDNS, NS1, OVH, and
+  Sakura Cloud DNS plugins are now compatible with Lexicon 3.0+.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-ovh
+* certbot-dns-route53
+* certbot-dns-sakuracloud
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/59?closed=1
 
 ## 0.27.1 - 2018-09-06
 

Dockerfile-dev

@@ -16,6 +16,6 @@ RUN apt-get update && \
            /tmp/* \
            /var/tmp/*
 
-RUN VENV_NAME="../venv" tools/venv.sh
+RUN VENV_NAME="../venv" python tools/venv.py
 
 ENV PATH /opt/certbot/venv/bin:$PATH

README.rst

@@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha
 
 How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide <https://certbot.eff.org>`_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access <https://certbot.eff.org/faq/#does-certbot-require-root-administrator-privileges>`_ to your web server to run Certbot.
 
-If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
+Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
 
 Certbot is a fully-featured, extensible client for the Let's
 Encrypt CA (or any other CA that speaks the `ACME
@@ -91,8 +91,6 @@ Main Website: https://certbot.eff.org
 
 Let's Encrypt Website: https://letsencrypt.org
 
-IRC Channel: #letsencrypt on `Freenode`_
-
 Community: https://community.letsencrypt.org
 
 ACME spec: http://ietf-wg-acme.github.io/acme/
@@ -101,8 +99,6 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
 
 |build-status| |coverage| |docs| |container|
 
-.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt
-
 .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master
    :target: https://travis-ci.org/certbot/certbot
    :alt: Travis CI status

acme/acme/challenges.py

@@ -4,6 +4,7 @@ import functools
 import hashlib
 import logging
 import socket
+import warnings
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
@@ -493,6 +494,11 @@ class TLSSNI01(KeyAuthorizationChallenge):
     # boulder#962, ietf-wg-acme#22
     #n = jose.Field("n", encoder=int, decoder=int)
 
+    def __init__(self, *args, **kwargs):
+        warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.",
+            DeprecationWarning, stacklevel=2)
+        super(TLSSNI01, self).__init__(*args, **kwargs)
+
     def validation(self, account_key, **kwargs):
         """Generate validation.
 

acme/acme/challenges_test.py

@@ -1,5 +1,6 @@
 """Tests for acme.challenges."""
 import unittest
+import warnings
 
 import josepy as jose
 import mock
@@ -360,20 +361,29 @@ class TLSSNI01ResponseTest(unittest.TestCase):
 class TLSSNI01Test(unittest.TestCase):
 
     def setUp(self):
-        from acme.challenges import TLSSNI01
-        self.msg = TLSSNI01(
-            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
         self.jmsg = {
             'type': 'tls-sni-01',
             'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
         }
 
+    def _msg(self):
+        from acme.challenges import TLSSNI01
+        with warnings.catch_warnings(record=True) as warn:
+            warnings.simplefilter("always")
+            msg = TLSSNI01(
+                token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+            assert warn is not None # using a raw assert for mypy
+            self.assertTrue(len(warn) == 1)
+            self.assertTrue(issubclass(warn[-1].category, DeprecationWarning))
+            self.assertTrue('deprecated' in str(warn[-1].message))
+        return msg
+
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+        self.assertEqual(self.jmsg, self._msg().to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSSNI01
-        self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg))
+        self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg))
 
     def test_from_json_hashable(self):
         from acme.challenges import TLSSNI01
@@ -388,7 +398,7 @@ class TLSSNI01Test(unittest.TestCase):
     @mock.patch('acme.challenges.TLSSNI01Response.gen_cert')
     def test_validation(self, mock_gen_cert):
         mock_gen_cert.return_value = ('cert', 'key')
-        self.assertEqual(('cert', 'key'), self.msg.validation(
+        self.assertEqual(('cert', 'key'), self._msg().validation(
             KEY, cert_key=mock.sentinel.cert_key))
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 

acme/acme/client.py

@@ -89,6 +89,8 @@ class ClientBase(object):  # pylint: disable=too-many-instance-attributes
 
         """
         kwargs.setdefault('acme_version', self.acme_version)
+        if hasattr(self.directory, 'newNonce'):
+            kwargs.setdefault('new_nonce_url', getattr(self.directory, 'newNonce'))
         return self.net.post(*args, **kwargs)
 
     def update_registration(self, regr, update=None):
@@ -1106,10 +1108,15 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         else:
             raise errors.MissingNonce(response)
 
-    def _get_nonce(self, url):
+    def _get_nonce(self, url, new_nonce_url):
         if not self._nonces:
             logger.debug('Requesting fresh nonce')
-            self._add_nonce(self.head(url))
+            if new_nonce_url is None:
+                response = self.head(url)
+            else:
+                # request a new nonce from the acme newNonce endpoint
+                response = self._check_response(self.head(new_nonce_url), content_type=None)
+            self._add_nonce(response)
         return self._nonces.pop()
 
     def post(self, *args, **kwargs):
@@ -1130,8 +1137,13 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
 
     def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE,
             acme_version=1, **kwargs):
-        data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version)
+        try:
+            new_nonce_url = kwargs.pop('new_nonce_url')
+        except KeyError:
+            new_nonce_url = None
+        data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version)
         kwargs.setdefault('headers', {'Content-Type': content_type})
         response = self._send_request('POST', url, data=data, **kwargs)
+        response = self._check_response(response, content_type=content_type)
         self._add_nonce(response)
-        return self._check_response(response, content_type=content_type)
+        return response

acme/acme/client_test.py

@@ -805,7 +805,8 @@ class ClientV2Test(ClientTestBase):
     def test_revoke(self):
         self.client.revoke(messages_test.CERT, self.rsn)
         self.net.post.assert_called_once_with(
-            self.directory["revokeCert"], mock.ANY, acme_version=2)
+            self.directory["revokeCert"], mock.ANY, acme_version=2,
+            new_nonce_url=DIRECTORY_V2['newNonce'])
 
     def test_update_registration(self):
         # "Instance of 'Field' has no to_json/update member" bug:
@@ -1038,8 +1039,8 @@ class ClientNetworkTest(unittest.TestCase):
 
         # Requests Library Exceptions
         except requests.exceptions.ConnectionError as z: #pragma: no cover
-            self.assertEqual("('Connection aborted.', "
-                             "error(111, 'Connection refused'))", str(z))
+            self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))"
+                              == str(z) or "[WinError 10061]" in str(z))
 
 class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork which mock out response."""
@@ -1052,7 +1053,10 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.response = mock.MagicMock(ok=True, status_code=http_client.OK)
         self.response.headers = {}
         self.response.links = {}
-        self.checked_response = mock.MagicMock()
+        self.response.checked = False
+        self.acmev1_nonce_response = mock.MagicMock(ok=False,
+            status_code=http_client.METHOD_NOT_ALLOWED)
+        self.acmev1_nonce_response.headers = {}
         self.obj = mock.MagicMock()
         self.wrapped_obj = mock.MagicMock()
         self.content_type = mock.sentinel.content_type
@@ -1064,13 +1068,21 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
 
         def send_request(*args, **kwargs):
             # pylint: disable=unused-argument,missing-docstring
+            self.assertFalse("new_nonce_url" in kwargs)
+            method = args[0]
+            uri = args[1]
+            if method == 'HEAD' and uri != "new_nonce_uri":
+                response = self.acmev1_nonce_response
+            else:
+                response = self.response
+
             if self.available_nonces:
-                self.response.headers = {
+                response.headers = {
                     self.net.REPLAY_NONCE_HEADER:
                     self.available_nonces.pop().decode()}
             else:
-                self.response.headers = {}
-            return self.response
+                response.headers = {}
+            return response
 
         # pylint: disable=protected-access
         self.net._send_request = self.send_request = mock.MagicMock(
@@ -1082,28 +1094,39 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         # pylint: disable=missing-docstring
         self.assertEqual(self.response, response)
         self.assertEqual(self.content_type, content_type)
-        return self.checked_response
+        self.assertTrue(self.response.ok)
+        self.response.checked = True
+        return self.response
 
     def test_head(self):
-        self.assertEqual(self.response, self.net.head(
+        self.assertEqual(self.acmev1_nonce_response, self.net.head(
             'http://example.com/', 'foo', bar='baz'))
         self.send_request.assert_called_once_with(
             'HEAD', 'http://example.com/', 'foo', bar='baz')
 
+    def test_head_v2(self):
+        self.assertEqual(self.response, self.net.head(
+            'new_nonce_uri', 'foo', bar='baz'))
+        self.send_request.assert_called_once_with(
+            'HEAD', 'new_nonce_uri', 'foo', bar='baz')
+
     def test_get(self):
-        self.assertEqual(self.checked_response, self.net.get(
+        self.assertEqual(self.response, self.net.get(
             'http://example.com/', content_type=self.content_type, bar='baz'))
+        self.assertTrue(self.response.checked)
         self.send_request.assert_called_once_with(
             'GET', 'http://example.com/', bar='baz')
 
     def test_post_no_content_type(self):
         self.content_type = self.net.JOSE_CONTENT_TYPE
-        self.assertEqual(self.checked_response, self.net.post('uri', self.obj))
+        self.assertEqual(self.response, self.net.post('uri', self.obj))
+        self.assertTrue(self.response.checked)
 
     def test_post(self):
         # pylint: disable=protected-access
-        self.assertEqual(self.checked_response, self.net.post(
+        self.assertEqual(self.response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
+        self.assertTrue(self.response.checked)
         self.net._wrap_in_jws.assert_called_once_with(
             self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1)
 
@@ -1135,7 +1158,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     def test_post_not_retried(self):
         check_response = mock.MagicMock()
         check_response.side_effect = [messages.Error.with_code('malformed'),
-                                      self.checked_response]
+                                      self.response]
 
         # pylint: disable=protected-access
         self.net._check_response = check_response
@@ -1143,13 +1166,12 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
                           self.obj, content_type=self.content_type)
 
     def test_post_successful_retry(self):
-        check_response = mock.MagicMock()
-        check_response.side_effect = [messages.Error.with_code('badNonce'),
-                                      self.checked_response]
+        post_once = mock.MagicMock()
+        post_once.side_effect = [messages.Error.with_code('badNonce'),
+                                      self.response]
 
         # pylint: disable=protected-access
-        self.net._check_response = check_response
-        self.assertEqual(self.checked_response, self.net.post(
+        self.assertEqual(self.response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
 
     def test_head_get_post_error_passthrough(self):
@@ -1160,6 +1182,26 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertRaises(requests.exceptions.RequestException,
                           self.net.post, 'uri', obj=self.obj)
 
+    def test_post_bad_nonce_head(self):
+        # pylint: disable=protected-access
+        # regression test for https://github.com/certbot/certbot/issues/6092
+        bad_response = mock.MagicMock(ok=False, status_code=http_client.SERVICE_UNAVAILABLE)
+        self.net._send_request = mock.MagicMock()
+        self.net._send_request.return_value = bad_response
+        self.content_type = None
+        check_response = mock.MagicMock()
+        self.net._check_response = check_response
+        self.assertRaises(errors.ClientError, self.net.post, 'uri',
+                          self.obj, content_type=self.content_type, acme_version=2,
+                          new_nonce_url='new_nonce_uri')
+        self.assertEqual(check_response.call_count, 1)
+
+    def test_new_nonce_uri_removed(self):
+        self.content_type = None
+        self.net.post('uri', self.obj, content_type=None,
+            acme_version=2, new_nonce_url='new_nonce_uri')
+
+
 class ClientNetworkSourceAddressBindingTest(unittest.TestCase):
     """Tests that if ClientNetwork has a source IP set manually, the underlying library has
     used the provided source address."""

acme/acme/crypto_util.py

@@ -136,22 +136,16 @@ def probe_sni(name, host, port=443, timeout=300,
 
     socket_kwargs = {'source_address': source_address}
 
-    host_protocol_agnostic = host
-    if host == '::' or host == '0':
-        # https://github.com/python/typeshed/pull/2136
-        # while PR is not merged, we need to ignore
-        host_protocol_agnostic = None
-
     try:
         # pylint: disable=star-args
         logger.debug(
-            "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
+            "Attempting to connect to %s:%d%s.", host, port,
             " from {0}:{1}".format(
                 source_address[0],
                 source_address[1]
             ) if socket_kwargs else ""
         )
-        socket_tuple = (host_protocol_agnostic, port)  # type: Tuple[Optional[str], int]
+        socket_tuple = (host, port)  # type: Tuple[str, int]
         sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)

acme/acme/messages.py

@@ -523,7 +523,7 @@ class Order(ResourceBody):
     """
     identifiers = jose.Field('identifiers', omitempty=True)
     status = jose.Field('status', decoder=Status.from_json,
-                        omitempty=True, default=STATUS_PENDING)
+                        omitempty=True)
     authorizations = jose.Field('authorizations', omitempty=True)
     certificate = jose.Field('certificate', omitempty=True)
     finalize = jose.Field('finalize', omitempty=True)
@@ -553,4 +553,3 @@ class OrderResource(ResourceWithURI):
 class NewOrder(Order):
     """New order."""
     resource_type = 'new-order'
-    resource = fields.Resource(resource_type)

acme/acme/messages_test.py

@@ -424,6 +424,19 @@ class OrderResourceTest(unittest.TestCase):
             'authorizations': None,
         })
 
+class NewOrderTest(unittest.TestCase):
+    """Tests for acme.messages.NewOrder."""
+
+    def setUp(self):
+        from acme.messages import NewOrder
+        self.reg = NewOrder(
+            identifiers=mock.sentinel.identifiers)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.reg.to_json(), {
+            'identifiers': mock.sentinel.identifiers,
+        })
+
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/standalone_test.py

@@ -48,7 +48,7 @@ class TLSSNI01ServerTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01Server
-        self.server = TLSSNI01Server(("", 0), certs=self.certs)
+        self.server = TLSSNI01Server(('localhost', 0), certs=self.certs)
         # pylint: disable=no-member
         self.thread = threading.Thread(target=self.server.serve_forever)
         self.thread.start()
@@ -133,8 +133,11 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
                 self.address_family = socket.AF_INET
             socketserver.TCPServer.__init__(self, *args, **kwargs)
             if ipv6:
+                # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing.
+                # We use the corresponding value (41) instead.
+                level = getattr(socket, "IPPROTO_IPV6", 41)
                 # pylint: disable=no-member
-                self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+                self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1)
                 try:
                     self.server_bind()
                     self.server_activate()
@@ -147,15 +150,15 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
         mock_bind.side_effect = socket.error
         from acme.standalone import BaseDualNetworkedServers
         self.assertRaises(socket.error, BaseDualNetworkedServers,
-            BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
-            socketserver.BaseRequestHandler)
+                          BaseDualNetworkedServersTest.SingleProtocolServer,
+                          ('', 0),
+                          socketserver.BaseRequestHandler)
 
     def test_ports_equal(self):
         from acme.standalone import BaseDualNetworkedServers
         servers = BaseDualNetworkedServers(
             BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
+            ('', 0),
             socketserver.BaseRequestHandler)
         socknames = servers.getsocknames()
         prev_port = None
@@ -177,7 +180,7 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01DualNetworkedServers
-        self.servers = TLSSNI01DualNetworkedServers(("", 0), certs=self.certs)
+        self.servers = TLSSNI01DualNetworkedServers(('localhost', 0), certs=self.certs)
         self.servers.serve_forever()
 
     def tearDown(self):
@@ -245,6 +248,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
         self.assertFalse(self._test_http01(add=False))
 
 
+@test_util.broken_on_windows
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 

acme/acme/test_util.py

@@ -4,6 +4,7 @@
 
 """
 import os
+import sys
 import pkg_resources
 import unittest
 
@@ -94,3 +95,11 @@ def skip_unless(condition, reason):  # pragma: no cover
         return lambda cls: cls
     else:
         return lambda cls: None
+
+def broken_on_windows(function):
+    """Decorator to skip temporarily a broken test on Windows."""
+    reason = 'Test is broken and ignored on windows but should be fixed.'
+    return unittest.skipIf(
+        sys.platform == 'win32'
+        and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true',
+        reason)(function)

acme/setup.py

@@ -3,7 +3,7 @@ from setuptools import find_packages
 from setuptools.command.test import test as TestCommand
 import sys
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

appveyor.yml

@@ -1,12 +1,40 @@
-# AppVeyor CI pipeline, executed on Windows Server 2016/2012 R2
+environment:
+  matrix:
+    - FYI: Python 3.4 on Windows Server 2012 R2
+      TOXENV: py34
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+    - FYI: Python 3.4 on Windows Server 2016
+      TOXENV: py34
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - FYI: Python 3.5 on Windows Server 2016
+      TOXENV: py35
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - FYI: Python 3.7 on Windows Server 2016 + code coverage
+      TOXENV: py37-cover
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
 branches:
   only:
     - master
-    - /^\d+\.\d+\.x$/ # version branches like X.X.X
+    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
     - /^test-.*$/
 
+install:
+  # Use Python 3.7 by default
+  - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
+  # Check env
+  - "echo %APPVEYOR_BUILD_WORKER_IMAGE%"
+  - "python --version"
+  # Upgrade pip to avoid warnings
+  - "python -m pip install --upgrade pip"
+  # Ready to install tox and coverage
+  - "pip install tox codecov"
+
 build: off
 
 test_script:
-  - ps: Write-Host "Hello, world!"
-  
+  # Test env is set by TOXENV env variable
+  - tox
+
+on_success:
+  - if exist .coverage codecov

certbot-apache/certbot_apache/augeas_lens/httpd.aug

@@ -44,67 +44,134 @@ autoload xfm
  *****************************************************************)
 let dels (s:string)     = del s s
 
+(* The continuation sequence that indicates that we should consider the
+ * next line part of the current line *)
+let cont = /\\\\\r?\n/
+
+(* Whitespace within a line: space, tab, and the continuation sequence *)
+let ws = /[ \t]/ | cont
+
+(* Any possible character - '.' does not match \n *)
+let any = /(.|\n)/
+
+(* Any character preceded by a backslash *)
+let esc_any = /\\\\(.|\n)/
+
+(* Newline sequence - both for Unix and DOS newlines *)
+let nl = /\r?\n/
+
+(* Whitespace at the end of a line *)
+let eol = del (ws* . nl) "\n"
+
 (* deal with continuation lines *)
-let sep_spc             = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " "
-let sep_osp             = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ ""
-let sep_eq              = del /[ \t]*=[ \t]*/ "="
+let sep_spc             = del ws+ " "
+let sep_osp             = del ws* ""
+let sep_eq              = del (ws* . "=" . ws*) "="
 
 let nmtoken             = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/
 let word                = /[a-z][a-z0-9._-]*/i
 
-let eol                 = Util.doseol
-let empty               = Util.empty_dos
+(* A complete line that is either just whitespace or a comment that only
+ * contains whitespace *)
+let empty = [ del (ws* . /#?/ . ws* . nl) "\n" ]
+
 let indent              = Util.indent
 
-let comment_val_re      = /([^ \t\r\n](.|\\\\\r?\n)*[^ \\\t\r\n]|[^ \t\r\n])/
-let comment             = [ label "#comment" . del /[ \t]*#[ \t]*/ "# "
-                          . store comment_val_re . eol ]
+(* A comment that is not just whitespace. We define it in terms of the
+ * things that are not allowed as part of such a comment:
+ *   1) Starts with whitespace
+ *   2) Ends with whitespace, a backslash or \r
+ *   3) Unescaped newlines
+ *)
+let comment =
+  let comment_start = del (ws* . "#" . ws* ) "# " in
+  let unesc_eol = /[^\]?/ . nl in
+  let w = /[^\t\n\r \\]/ in
+  let r = /[\r\\]/ in
+  let s = /[\t\r ]/ in
+  (*
+   * we'd like to write
+   * let b = /\\\\/ in
+   * let t = /[\t\n\r ]/ in
+   * let x = b . (t? . (s|w)* ) in
+   * but the definition of b depends on commit 244c0edd in 1.9.0 and
+   * would make the lens unusable with versions before 1.9.0. So we write
+   * x out which works in older versions, too
+   *)
+  let x = /\\\\[\t\n\r ]?[^\n\\]*/ in
+  let line = ((r . s* . w|w|r) . (s|w)* . x*|(r.s* )?).w.(s*.w)* in
+  [ label "#comment" . comment_start . store line . eol ]
 
 (* borrowed from shellvars.aug *)
-let char_arg_dir  = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ /
 let char_arg_sec  = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ /
 let char_arg_wl   = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/
 
-let cdot = /\\\\./
-let cl = /\\\\\n/
 let dquot =
      let no_dquot = /[^"\\\r\n]/
-  in /"/ . (no_dquot|cdot|cl)* . /"/
+  in /"/ . (no_dquot|esc_any)* . /"/
 let dquot_msg =
      let no_dquot = /([^ \t"\\\r\n]|[^"\\\r\n]+[^ \t"\\\r\n])/
-  in /"/ . (no_dquot|cdot|cl)*
+  in /"/ . (no_dquot|esc_any)* . no_dquot
+
 let squot =
      let no_squot = /[^'\\\r\n]/
-  in /'/ . (no_squot|cdot|cl)* . /'/
+  in /'/ . (no_squot|esc_any)* . /'/
 let comp = /[<>=]?=/
 
 (******************************************************************
  *                            Attributes
  *****************************************************************)
 
-let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ]
+(* The arguments for a directive come in two flavors: quoted with single or
+ * double quotes, or bare. Bare arguments may not start with a single or
+ * double quote; since we also treat "word lists" special, i.e. lists
+ * enclosed in curly braces, bare arguments may not start with those,
+ * either.
+ *
+ * Bare arguments may not contain unescaped spaces, but we allow escaping
+ * with '\\'. Quoted arguments can contain anything, though the quote must
+ * be escaped with '\\'.
+ *)
+let bare = /([^{"' \t\n\r]|\\\\.)([^ \t\n\r]|\\\\.)*[^ \t\n\r\\]|[^{"' \t\n\r\\]/
+
+let arg_quoted = [ label "arg" . store (dquot|squot) ]
+let arg_bare = [ label "arg" . store bare ]
+
 (* message argument starts with " but ends at EOL *)
 let arg_dir_msg = [ label "arg" . store dquot_msg ]
-let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ]
 let arg_wl  = [ label "arg" . store (char_arg_wl+|dquot|squot) ]
 
 (* comma-separated wordlist as permitted in the SSLRequire directive *)
 let arg_wordlist =
-     let wl_start = Util.del_str "{" in
-     let wl_end   = Util.del_str "}" in
+     let wl_start = dels "{" in
+     let wl_end   = dels "}" in
      let wl_sep   = del /[ \t]*,[ \t]*/ ", "
   in [ label "wordlist" . wl_start . arg_wl . (wl_sep . arg_wl)* . wl_end ]
 
 let argv (l:lens) = l . (sep_spc . l)*
 
+(* the arguments of a directive. We use this once we have parsed the name
+ * of the directive, and the space right after it. When dir_args is used,
+ * we also know that we have at least one argument. We need to be careful
+ * with the spacing between arguments: quoted arguments and word lists do
+ * not need to have space between them, but bare arguments do.
+ *
+ * Apache apparently is also happy if the last argument starts with a double
+ * quote, but has no corresponding closing duoble quote, which is what
+ * arg_dir_msg handles
+ *)
+let dir_args =
+  let arg_nospc = arg_quoted|arg_wordlist in
+  (arg_bare . sep_spc | arg_nospc . sep_osp)* . (arg_bare|arg_nospc|arg_dir_msg)
+
 let directive =
-    (* arg_dir_msg may be the last or only argument *)
-     let dir_args = (argv (arg_dir|arg_wordlist) . (sep_spc . arg_dir_msg)?) | arg_dir_msg
-  in [ indent . label "directive" . store word .  (sep_spc . dir_args)? . eol ]
+  [ indent . label "directive" . store word .  (sep_spc . dir_args)? . eol ]
+
+let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ]
 
 let section (body:lens) =
     (* opt_eol includes empty lines *)
-    let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in
+    let opt_eol = del /([ \t]*#?[ \t]*\r?\n)*/ "\n" in
     let inner = (sep_spc . argv arg_sec)? . sep_osp .
              dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? .
              indent . dels "</" in
@@ -133,6 +200,7 @@ let filter = (incl "/etc/apache2/apache2.conf") .
              (incl "/etc/httpd/conf.d/*.conf") .
              (incl "/etc/httpd/httpd.conf") .
              (incl "/etc/httpd/conf/httpd.conf") .
+             (incl "/etc/httpd/conf.modules.d/*.conf") .
              Util.stdexcl
 
 let xfm = transform lns filter

certbot-apache/certbot_apache/configurator.py

@@ -2253,7 +2253,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01, challenges.HTTP01]
+        return [challenges.HTTP01, challenges.TLSSNI01]
 
     def perform(self, achalls):
         """Perform the configuration related challenge.

certbot-apache/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.27.1"
+LE_AUTO_VERSION="0.28.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then
 else
   SetRootAuthMechanism
   if [ -n "$SUDO" ]; then
-    echo "Requesting to rerun $0 with root privileges..."
+    say "Requesting to rerun $0 with root privileges..."
     $SUDO "$0" --cb-auto-has-root "$@"
     exit 0
   fi
@@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.27.1 \
-    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
-    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
-acme==0.27.1 \
-    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
-    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
-certbot-apache==0.27.1 \
-    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
-    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
-certbot-nginx==0.27.1 \
-    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
-    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
+certbot==0.28.0 \
+    --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \
+    --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a
+acme==0.28.0 \
+    --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \
+    --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85
+certbot-apache==0.28.0 \
+    --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \
+    --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb
+certbot-nginx==0.28.0 \
+    --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \
+    --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-compatibility-test/Dockerfile

@@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/
 
 # all above files are necessary for setup.py, however, package source
 # code directory has to be copied separately to a subdirectory...
@@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \
     /opt/certbot/venv/bin/pip install -U setuptools && \
     /opt/certbot/venv/bin/pip install -U pip
 ENV PATH /opt/certbot/venv/bin:$PATH
-RUN /opt/certbot/src/tools/pip_install_editable.sh \
+RUN /opt/certbot/venv/bin/python \
+    /opt/certbot/src/tools/pip_install_editable.py \
     /opt/certbot/src/acme \
     /opt/certbot/src \
     /opt/certbot/src/certbot-apache \

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py

@@ -70,6 +70,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient):
         super(_CloudXNSLexiconClient, self).__init__()
 
         self.provider = cloudxns.Provider({
+            'provider_name': 'cloudxns',
             'auth_username': api_key,
             'auth_token': secret_key,
             'ttl': ttl,

certbot-dns-cloudxns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py

@@ -66,6 +66,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient):
         super(_DNSimpleLexiconClient, self).__init__()
 
         self.provider = dnsimple.Provider({
+            'provider_name': 'dnssimple',
             'auth_token': token,
             'ttl': ttl,
         })

certbot-dns-dnsimple/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py

@@ -72,6 +72,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient):
         super(_DNSMadeEasyLexiconClient, self).__init__()
 
         self.provider = dnsmadeeasy.Provider({
+            'provider_name': 'dnsmadeeasy',
             'auth_username': api_key,
             'auth_token': secret_key,
             'ttl': ttl,

certbot-dns-dnsmadeeasy/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py

@@ -73,6 +73,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient):
         super(_GehirnLexiconClient, self).__init__()
 
         self.provider = gehirn.Provider({
+            'provider_name': 'gehirn',
             'auth_token': api_token,
             'auth_secret': api_secret,
             'ttl': ttl,

certbot-dns-gehirn/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-linode/certbot_dns_linode/__init__.py

@@ -14,7 +14,11 @@ Named Arguments
                                             DNS to propagate before asking the
                                             ACME server to verify the DNS
                                             record.
-                                            (Default: 960)
+                                            (Default: 1200 because Linode
+                                            updates its first DNS every 15
+                                            minutes and we allow 5 more minutes
+                                            for the update to reach the other 5
+                                            servers)
 ==========================================  ===================================
 
 
@@ -74,13 +78,15 @@ Examples
      -d www.example.com
 
 .. code-block:: bash
-   :caption: To acquire a certificate for ``example.com``, waiting 60 seconds
-             for DNS propagation
+   :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds
+             for DNS propagation (Linode updates its first DNS every 15 minutes
+             and we allow some extra time for the update to reach the other 5
+             servers)
 
    certbot certonly \\
      --dns-linode \\
      --dns-linode-credentials ~/.secrets/certbot/linode.ini \\
-     --dns-linode-propagation-seconds 60 \\
+     --dns-linode-propagation-seconds 1000 \\
      -d example.com
 
 """

certbot-dns-linode/certbot_dns_linode/dns_linode.py

@@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator):
 
     @classmethod
     def add_parser_arguments(cls, add):  # pylint: disable=arguments-differ
-        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960)
+        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200)
         add('credentials', help='Linode credentials INI file.')
 
     def more_info(self):  # pylint: disable=missing-docstring,no-self-use
@@ -62,6 +62,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_key):
         super(_LinodeLexiconClient, self).__init__()
         self.provider = linode.Provider({
+            'provider_name': 'linode',
             'auth_token': api_key
         })
 

certbot-dns-linode/setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py

@@ -69,6 +69,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient):
         super(_LuaDNSLexiconClient, self).__init__()
 
         self.provider = luadns.Provider({
+            'provider_name': 'luadns',
             'auth_username': email,
             'auth_token': token,
             'ttl': ttl,

certbot-dns-luadns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py

@@ -66,6 +66,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient):
         super(_NS1LexiconClient, self).__init__()
 
         self.provider = nsone.Provider({
+            'provider_name': 'nsone',
             'auth_token': api_key,
             'ttl': ttl,
         })

certbot-dns-nsone/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py

@@ -78,6 +78,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient):
         super(_OVHLexiconClient, self).__init__()
 
         self.provider = ovh.Provider({
+            'provider_name': 'ovh',
             'auth_entrypoint': endpoint,
             'auth_application_key': application_key,
             'auth_application_secret': application_secret,

certbot-dns-ovh/setup.py

@@ -4,14 +4,14 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.21.1',
     'certbot>=0.21.1',
-    'dns-lexicon>=2.7.3', # Correct OVH integration tests
+    'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-rfc2136/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/certbot_dns_route53/dns_route53_test.py

@@ -1,5 +1,6 @@
 """Tests for certbot_dns_route53.dns_route53.Authenticator"""
 
+import os
 import unittest
 
 import mock
@@ -20,8 +21,18 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest
 
         self.config = mock.MagicMock()
 
+        # Set up dummy credentials for testing
+        os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key"
+        os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key"
+
         self.auth = Authenticator(self.config, "route53")
 
+    def tearDown(self):
+        # Remove the dummy credentials from env vars
+        del os.environ["AWS_ACCESS_KEY_ID"]
+        del os.environ["AWS_SECRET_ACCESS_KEY"]
+        super(AuthenticatorTest, self).tearDown()
+
     def test_perform(self):
         self.auth._change_txt_record = mock.MagicMock()
         self.auth._wait_for_change = mock.MagicMock()
@@ -117,8 +128,18 @@ class ClientTest(unittest.TestCase):
 
         self.config = mock.MagicMock()
 
+        # Set up dummy credentials for testing
+        os.environ["AWS_ACCESS_KEY_ID"] = "dummy_access_key"
+        os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy_secret_access_key"
+
         self.client = Authenticator(self.config, "route53")
 
+    def tearDown(self):
+        # Remove the dummy credentials from env vars
+        del os.environ["AWS_ACCESS_KEY_ID"]
+        del os.environ["AWS_SECRET_ACCESS_KEY"]
+        super(ClientTest, self).tearDown()
+
     def test_find_zone_id_for_domain(self):
         self.client.r53.get_paginator = mock.MagicMock()
         self.client.r53.get_paginator().paginate.return_value = [

certbot-dns-route53/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py

@@ -76,6 +76,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient):
         super(_SakuraCloudLexiconClient, self).__init__()
 
         self.provider = sakuracloud.Provider({
+            'provider_name': 'sakuracloud',
             'auth_token': api_token,
             'auth_secret': api_secret,
             'ttl': ttl,

certbot-dns-sakuracloud/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/certbot_nginx/configurator.py

@@ -8,7 +8,6 @@ import tempfile
 import time
 
 import OpenSSL
-import six
 import zope.interface
 
 from acme import challenges
@@ -32,6 +31,12 @@ from certbot_nginx import obj # pylint: disable=unused-import
 from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module
 
 
+NAME_RANK = 0
+START_WILDCARD_RANK = 1
+END_WILDCARD_RANK = 2
+REGEX_RANK = 3
+NO_SSL_MODIFIER = 4
+
 
 logger = logging.getLogger(__name__)
 
@@ -405,7 +410,8 @@ class NginxConfigurator(common.Installer):
         """
         if not matches:
             return None
-        elif matches[0]['rank'] in six.moves.range(2, 6):
+        elif matches[0]['rank'] in [START_WILDCARD_RANK, END_WILDCARD_RANK,
+            START_WILDCARD_RANK + NO_SSL_MODIFIER, END_WILDCARD_RANK + NO_SSL_MODIFIER]:
             # Wildcard match - need to find the longest one
             rank = matches[0]['rank']
             wildcards = [x for x in matches if x['rank'] == rank]
@@ -414,10 +420,9 @@ class NginxConfigurator(common.Installer):
             # Exact or regex match
             return matches[0]['vhost']
 
-
-    def _rank_matches_by_name_and_ssl(self, vhost_list, target_name):
+    def _rank_matches_by_name(self, vhost_list, target_name):
         """Returns a ranked list of vhosts from vhost_list that match target_name.
-        The ranking gives preference to SSL vhosts.
+        This method should always be followed by a call to _select_best_name_match.
 
         :param list vhost_list: list of vhosts to filter and rank
         :param str target_name: The name to match
@@ -437,21 +442,37 @@ class NginxConfigurator(common.Installer):
             if name_type == 'exact':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 0 if vhost.ssl else 1})
+                                'rank': NAME_RANK})
             elif name_type == 'wildcard_start':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 2 if vhost.ssl else 3})
+                                'rank': START_WILDCARD_RANK})
             elif name_type == 'wildcard_end':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 4 if vhost.ssl else 5})
+                                'rank': END_WILDCARD_RANK})
             elif name_type == 'regex':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 6 if vhost.ssl else 7})
+                                'rank': REGEX_RANK})
         return sorted(matches, key=lambda x: x['rank'])
 
+    def _rank_matches_by_name_and_ssl(self, vhost_list, target_name):
+        """Returns a ranked list of vhosts from vhost_list that match target_name.
+        The ranking gives preference to SSLishness before name match level.
+
+        :param list vhost_list: list of vhosts to filter and rank
+        :param str target_name: The name to match
+        :returns: list of dicts containing the vhost, the matching name, and
+            the numerical rank
+        :rtype: list
+
+        """
+        matches = self._rank_matches_by_name(vhost_list, target_name)
+        for match in matches:
+            if not match['vhost'].ssl:
+                match['rank'] += NO_SSL_MODIFIER
+        return sorted(matches, key=lambda x: x['rank'])
 
     def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False):
         """Chooses a single virtual host for redirect enhancement.
@@ -531,9 +552,7 @@ class NginxConfigurator(common.Installer):
 
         matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)]
 
-        # We can use this ranking function because sslishness doesn't matter to us, and
-        # there shouldn't be conflicting plaintextish servers listening on 80.
-        return self._rank_matches_by_name_and_ssl(matching_vhosts, target_name)
+        return self._rank_matches_by_name(matching_vhosts, target_name)
 
     def get_all_names(self):
         """Returns all names found in the Nginx Configuration.
@@ -568,6 +587,7 @@ class NginxConfigurator(common.Installer):
         return util.get_filtered_names(all_names)
 
     def _get_snakeoil_paths(self):
+        """Generate invalid certs that let us create ssl directives for Nginx"""
         # TODO: generate only once
         tmp_dir = os.path.join(self.config.work_dir, "snakeoil")
         le_key = crypto_util.init_save_key(
@@ -1019,7 +1039,7 @@ class NginxConfigurator(common.Installer):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01, challenges.HTTP01]
+        return [challenges.HTTP01, challenges.TLSSNI01]
 
     # Entry point in main.py for performing challenges
     def perform(self, achalls):

certbot-nginx/certbot_nginx/http_01.py

@@ -40,8 +40,6 @@ class NginxHttp01(common.ChallengePerformer):
         super(NginxHttp01, self).__init__(configurator)
         self.challenge_conf = os.path.join(
             configurator.config.config_dir, "le_http_01_cert_challenge.conf")
-        self._ipv6 = None
-        self._ipv6only = None
 
     def perform(self):
         """Perform a challenge on Nginx.
@@ -102,6 +100,7 @@ class NginxHttp01(common.ChallengePerformer):
         config = [self._make_or_mod_server_block(achall) for achall in self.achalls]
         config = [x for x in config if x is not None]
         config = nginxparser.UnspacedList(config)
+        logger.debug("Generated server block:\n%s", str(config))
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
@@ -120,9 +119,7 @@ class NginxHttp01(common.ChallengePerformer):
             self.configurator.config.http01_port)
         port = self.configurator.config.http01_port
 
-        if self._ipv6 is None or self._ipv6only is None:
-            self._ipv6, self._ipv6only = self.configurator.ipv6_info(port)
-        ipv6, ipv6only = self._ipv6, self._ipv6only
+        ipv6, ipv6only = self.configurator.ipv6_info(port)
 
         if ipv6:
             # If IPv6 is active in Nginx configuration

certbot-nginx/certbot_nginx/parser_obj.py

@@ -0,0 +1,392 @@
+""" This file contains parsing routines and object classes to help derive meaning from
+raw lists of tokens from pyparsing. """
+
+import abc
+import logging
+import six
+
+from certbot import errors
+
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
+
+logger = logging.getLogger(__name__)
+COMMENT = " managed by Certbot"
+COMMENT_BLOCK = ["#", COMMENT]
+
+class Parsable(object):
+    """ Abstract base class for "Parsable" objects whose underlying representation
+    is a tree of lists.
+
+    :param .Parsable parent: This object's parsed parent in the tree
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, parent=None):
+        self._data = [] # type: List[object]
+        self._tabs = None
+        self.parent = parent
+
+    @classmethod
+    def parsing_hooks(cls):
+        """Returns object types that this class should be able to `parse` recusrively.
+        The order of the objects indicates the order in which the parser should
+        try to parse each subitem.
+        :returns: A list of Parsable classes.
+        :rtype list:
+        """
+        return (Block, Sentence, Statements)
+
+    @staticmethod
+    @abc.abstractmethod
+    def should_parse(lists):
+        """ Returns whether the contents of `lists` can be parsed into this object.
+
+        :returns: Whether `lists` can be parsed as this object.
+        :rtype bool:
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def parse(self, raw_list, add_spaces=False):
+        """ Loads information into this object from underlying raw_list structure.
+        Each Parsable object might make different assumptions about the structure of
+        raw_list.
+
+        :param list raw_list: A list or sublist of tokens from pyparsing, containing whitespace
+            as separate tokens.
+        :param bool add_spaces: If set, the method can and should manipulate and insert spacing
+            between non-whitespace tokens and lists to delimit them.
+        :raises .errors.MisconfigurationError: when the assumptions about the structure of
+            raw_list are not met.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def iterate(self, expanded=False, match=None):
+        """ Iterates across this object. If this object is a leaf object, only yields
+        itself. If it contains references other parsing objects, and `expanded` is set,
+        this function should first yield itself, then recursively iterate across all of them.
+        :param bool expanded: Whether to recursively iterate on possible children.
+        :param callable match: If provided, an object is only iterated if this callable
+            returns True when called on that object.
+
+        :returns: Iterator over desired objects.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_tabs(self):
+        """ Guess at the tabbing style of this parsed object, based on whitespace.
+
+        If this object is a leaf, it deducts the tabbing based on its own contents.
+        Other objects may guess by calling `get_tabs` recursively on child objects.
+
+        :returns: Guess at tabbing for this object. Should only return whitespace strings
+            that does not contain newlines.
+        :rtype str:
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def set_tabs(self, tabs="    "):
+        """This tries to set and alter the tabbing of the current object to a desired
+        whitespace string. Primarily meant for objects that were constructed, so they
+        can conform to surrounding whitespace.
+
+        :param str tabs: A whitespace string (not containing newlines).
+        """
+        raise NotImplementedError()
+
+    def dump(self, include_spaces=False):
+        """ Dumps back to pyparsing-like list tree. The opposite of `parse`.
+
+        Note: if this object has not been modified, `dump` with `include_spaces=True`
+        should always return the original input of `parse`.
+
+        :param bool include_spaces: If set to False, magically hides whitespace tokens from
+            dumped output.
+
+        :returns: Pyparsing-like list tree.
+        :rtype list:
+        """
+        return [elem.dump(include_spaces) for elem in self._data]
+
+class Statements(Parsable):
+    """ A group or list of "Statements". A Statement is either a Block or a Sentence.
+
+    The underlying representation is simply a list of these Statement objects, with
+    an extra `_trailing_whitespace` string to keep track of the whitespace that does not
+    precede any more statements.
+    """
+    def __init__(self, parent=None):
+        super(Statements, self).__init__(parent)
+        self._trailing_whitespace = None
+
+    # ======== Begin overridden functions
+
+    @staticmethod
+    def should_parse(lists):
+        return isinstance(lists, list)
+
+    def set_tabs(self, tabs="    "):
+        """ Sets the tabbing for this set of statements. Does this by calling `set_tabs`
+        on each of the child statements.
+
+        Then, if a parent is present, sets trailing whitespace to parent tabbing. This
+        is so that the trailing } of any Block that contains Statements lines up
+        with parent tabbing.
+        """
+        for statement in self._data:
+            statement.set_tabs(tabs)
+        if self.parent is not None:
+            self._trailing_whitespace = "\n" + self.parent.get_tabs()
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list of statements.
+        Expects all elements in `parse_this` to be parseable by `type(self).parsing_hooks`,
+        with an optional whitespace string at the last index of `parse_this`.
+        """
+        if not isinstance(parse_this, list):
+            raise errors.MisconfigurationError("Statements parsing expects a list!")
+        # If there's a trailing whitespace in the list of statements, keep track of it.
+        if len(parse_this) > 0 and isinstance(parse_this[-1], six.string_types) \
+                               and parse_this[-1].isspace():
+            self._trailing_whitespace = parse_this[-1]
+            parse_this = parse_this[:-1]
+        self._data = [parse_raw(elem, self, add_spaces) for elem in parse_this]
+
+    def get_tabs(self):
+        """ Takes a guess at the tabbing of all contained Statements by retrieving the
+        tabbing of the first Statement."""
+        if len(self._data) > 0:
+            return self._data[0].get_tabs()
+        return ""
+
+    def dump(self, include_spaces=False):
+        """ Dumps this object by first dumping each statement, then appending its
+        trailing whitespace (if `include_spaces` is set) """
+        data = super(Statements, self).dump(include_spaces)
+        if include_spaces and self._trailing_whitespace is not None:
+            return data + [self._trailing_whitespace]
+        return data
+
+    def iterate(self, expanded=False, match=None):
+        """ Combines each statement's iterator.  """
+        for elem in self._data:
+            for sub_elem in elem.iterate(expanded, match):
+                yield sub_elem
+
+    # ======== End overridden functions
+
+def _space_list(list_):
+    """ Inserts whitespace between adjacent non-whitespace tokens. """
+    spaced_statement = [] # type: List[str]
+    for i in reversed(six.moves.xrange(len(list_))):
+        spaced_statement.insert(0, list_[i])
+        if i > 0 and not list_[i].isspace() and not list_[i-1].isspace():
+            spaced_statement.insert(0, " ")
+    return spaced_statement
+
+class Sentence(Parsable):
+    """ A list of words. Non-whitespace words are typically separated with whitespace tokens. """
+
+    # ======== Begin overridden functions
+
+    @staticmethod
+    def should_parse(lists):
+        """ Returns True if `lists` can be parseable as a `Sentence`-- that is,
+        every element is a string type.
+
+        :param list lists: The raw unparsed list to check.
+
+        :returns: whether this lists is parseable by `Sentence`.
+        """
+        return isinstance(lists, list) and len(lists) > 0 and \
+            all([isinstance(elem, six.string_types) for elem in lists])
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list of string types into this object.
+        If add_spaces is set, adds whitespace tokens between adjacent non-whitespace tokens."""
+        if add_spaces:
+            parse_this = _space_list(parse_this)
+        if not isinstance(parse_this, list) or \
+                any([not isinstance(elem, six.string_types) for elem in parse_this]):
+            raise errors.MisconfigurationError("Sentence parsing expects a list of string types.")
+        self._data = parse_this
+
+    def iterate(self, expanded=False, match=None):
+        """ Simply yields itself. """
+        if match is None or match(self):
+            yield self
+
+    def set_tabs(self, tabs="    "):
+        """ Sets the tabbing on this sentence. Inserts a newline and `tabs` at the
+        beginning of `self._data`. """
+        if self._data[0].isspace():
+            return
+        self._data.insert(0, "\n" + tabs)
+
+    def dump(self, include_spaces=False):
+        """ Dumps this sentence. If include_spaces is set, includes whitespace tokens."""
+        if not include_spaces:
+            return self.words
+        return self._data
+
+    def get_tabs(self):
+        """ Guesses at the tabbing of this sentence. If the first element is whitespace,
+        returns the whitespace after the rightmost newline in the string. """
+        first = self._data[0]
+        if not first.isspace():
+            return ""
+        rindex = first.rfind("\n")
+        return first[rindex+1:]
+
+    # ======== End overridden functions
+
+    @property
+    def words(self):
+        """ Iterates over words, but without spaces. Like Unspaced List. """
+        return [word.strip("\"\'") for word in self._data if not word.isspace()]
+
+    def __getitem__(self, index):
+        return self.words[index]
+
+    def __contains__(self, word):
+        return word in self.words
+
+class Block(Parsable):
+    """ Any sort of bloc, denoted by a block name and curly braces, like so:
+    The parsed block:
+        block name {
+            content 1;
+            content 2;
+        }
+    might be represented with the list [names, contents], where
+        names = ["block", " ", "name", " "]
+        contents = [["\n    ", "content", " ", "1"], ["\n    ", "content", " ", "2"], "\n"]
+    """
+    def __init__(self, parent=None):
+        super(Block, self).__init__(parent)
+        self.names = None # type: Sentence
+        self.contents = None # type: Block
+
+    @staticmethod
+    def should_parse(lists):
+        """ Returns True if `lists` can be parseable as a `Block`-- that is,
+        it's got a length of 2, the first element is a `Sentence` and the second can be
+        a `Statements`.
+
+        :param list lists: The raw unparsed list to check.
+
+        :returns: whether this lists is parseable by `Block`. """
+        return isinstance(lists, list) and len(lists) == 2 and \
+            Sentence.should_parse(lists[0]) and isinstance(lists[1], list)
+
+    def set_tabs(self, tabs="    "):
+        """ Sets tabs by setting equivalent tabbing on names, then adding tabbing
+        to contents."""
+        self.names.set_tabs(tabs)
+        self.contents.set_tabs(tabs + "    ")
+
+    def iterate(self, expanded=False, match=None):
+        """ Iterator over self, and if expanded is set, over its contents. """
+        if match is None or match(self):
+            yield self
+        if expanded:
+            for elem in self.contents.iterate(expanded, match):
+                yield elem
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list that resembles a block.
+
+        The assumptions that this routine makes are:
+            1. the first element of `parse_this` is a valid Sentence.
+            2. the second element of `parse_this` is a valid Statement.
+        If add_spaces is set, we call it recursively on `names` and `contents`, and
+        add an extra trailing space to `names` (to separate the block's opening bracket
+        and the block name).
+        """
+        if not Block.should_parse(parse_this):
+            raise errors.MisconfigurationError("Block parsing expects a list of length 2. "
+                "First element should be a list of string types (the bloc names), "
+                "and second should be another list of statements (the bloc content).")
+        self.names = Sentence(self)
+        if add_spaces:
+            parse_this[0].append(" ")
+        self.names.parse(parse_this[0], add_spaces)
+        self.contents = Statements(self)
+        self.contents.parse(parse_this[1], add_spaces)
+        self._data = [self.names, self.contents]
+
+    def get_tabs(self):
+        """ Guesses tabbing by retrieving tabbing guess of self.names. """
+        return self.names.get_tabs()
+
+def _is_comment(parsed_obj):
+    """ Checks whether parsed_obj is a comment.
+
+    :param .Parsable parsed_obj:
+
+    :returns: whether parsed_obj represents a comment sentence.
+    :rtype bool:
+    """
+    if not isinstance(parsed_obj, Sentence):
+        return False
+    return parsed_obj.words[0] == "#"
+
+def _is_certbot_comment(parsed_obj):
+    """ Checks whether parsed_obj is a "managed by Certbot" comment.
+
+    :param .Parsable parsed_obj:
+
+    :returns: whether parsed_obj is a "managed by Certbot" comment.
+    :rtype bool:
+    """
+    if not _is_comment(parsed_obj):
+        return False
+    if len(parsed_obj.words) != len(COMMENT_BLOCK):
+        return False
+    for i, word in enumerate(parsed_obj.words):
+        if word != COMMENT_BLOCK[i]:
+            return False
+    return True
+
+def _certbot_comment(parent, preceding_spaces=4):
+    """ A "Managed by Certbot" comment.
+    :param int preceding_spaces: Number of spaces between the end of the previous
+        statement and the comment.
+    :returns: Sentence containing the comment.
+    :rtype: .Sentence
+    """
+    result = Sentence(parent)
+    result.parse([" " * preceding_spaces] + COMMENT_BLOCK)
+    return result
+
+def _choose_parser(parent, list_):
+    """ Choose a parser from type(parent).parsing_hooks, depending on whichever hook
+    returns True first. """
+    hooks = Parsable.parsing_hooks()
+    if parent:
+        hooks = type(parent).parsing_hooks()
+    for type_ in hooks:
+        if type_.should_parse(list_):
+            return type_(parent)
+    raise errors.MisconfigurationError(
+        "None of the parsing hooks succeeded, so we don't know how to parse this set of lists.")
+
+def parse_raw(lists_, parent=None, add_spaces=False):
+    """ Primary parsing factory function.
+
+    :param list lists_: raw lists from pyparsing to parse.
+    :param .Parent parent: The parent containing this object.
+    :param bool add_spaces: Whether to pass add_spaces to the parser.
+
+    :returns .Parsable: The parsed object.
+
+    :raises errors.MisconfigurationError: If no parsing hook passes, and we can't
+        determine which type to parse the raw lists into.
+    """
+    parser = _choose_parser(parent, lists_)
+    parser.parse(lists_, add_spaces)
+    return parser

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -103,7 +103,7 @@ class NginxConfiguratorTest(util.NginxTest):
             errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement')
 
     def test_get_chall_pref(self):
-        self.assertEqual([challenges.TLSSNI01, challenges.HTTP01],
+        self.assertEqual([challenges.HTTP01, challenges.TLSSNI01],
                          self.config.get_chall_pref('myhost'))
 
     def test_save(self):
@@ -128,22 +128,39 @@ class NginxConfiguratorTest(util.NginxTest):
                             ['#', parser.COMMENT]]]],
                          parsed[0])
 
-    def test_choose_vhosts(self):
-        localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.'])
-        server_conf = set(['somename', 'another.alias', 'alias'])
-        example_conf = set(['.example.com', 'example.*'])
-        foo_conf = set(['*.www.foo.com', '*.www.example.com'])
-        ipv6_conf = set(['ipv6.com'])
+    def test_choose_vhosts_alias(self):
+        self._test_choose_vhosts_common('alias', 'server_conf')
 
-        results = {'localhost': localhost_conf,
-                   'alias': server_conf,
-                   'example.com': example_conf,
-                   'example.com.uk.test': example_conf,
-                   'www.example.com': example_conf,
-                   'test.www.example.com': foo_conf,
-                   'abc.www.foo.com': foo_conf,
-                   'www.bar.co.uk': localhost_conf,
-                   'ipv6.com': ipv6_conf}
+    def test_choose_vhosts_example_com(self):
+        self._test_choose_vhosts_common('example.com', 'example_conf')
+
+    def test_choose_vhosts_localhost(self):
+        self._test_choose_vhosts_common('localhost', 'localhost_conf')
+
+    def test_choose_vhosts_example_com_uk_test(self):
+        self._test_choose_vhosts_common('example.com.uk.test', 'example_conf')
+
+    def test_choose_vhosts_www_example_com(self):
+        self._test_choose_vhosts_common('www.example.com', 'example_conf')
+
+    def test_choose_vhosts_test_www_example_com(self):
+        self._test_choose_vhosts_common('test.www.example.com', 'foo_conf')
+
+    def test_choose_vhosts_abc_www_foo_com(self):
+        self._test_choose_vhosts_common('abc.www.foo.com', 'foo_conf')
+
+    def test_choose_vhosts_www_bar_co_uk(self):
+        self._test_choose_vhosts_common('www.bar.co.uk', 'localhost_conf')
+
+    def test_choose_vhosts_ipv6_com(self):
+        self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf')
+
+    def _test_choose_vhosts_common(self, name, conf):
+        conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']),
+                 'server_conf': set(['somename', 'another.alias', 'alias']),
+                 'example_conf': set(['.example.com', 'example.*']),
+                 'foo_conf': set(['*.www.foo.com', '*.www.example.com']),
+                 'ipv6_conf': set(['ipv6.com'])}
 
         conf_path = {'localhost': "etc_nginx/nginx.conf",
                    'alias': "etc_nginx/nginx.conf",
@@ -155,22 +172,22 @@ class NginxConfiguratorTest(util.NginxTest):
                    'www.bar.co.uk': "etc_nginx/nginx.conf",
                    'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"}
 
+        vhost = self.config.choose_vhosts(name)[0]
+        path = os.path.relpath(vhost.filep, self.temp_dir)
+
+        self.assertEqual(conf_names[conf], vhost.names)
+        self.assertEqual(conf_path[name], path)
+        # IPv6 specific checks
+        if name == "ipv6.com":
+            self.assertTrue(vhost.ipv6_enabled())
+            # Make sure that we have SSL enabled also for IPv6 addr
+            self.assertTrue(
+                any([True for x in vhost.addrs if x.ssl and x.ipv6]))
+
+    def test_choose_vhosts_bad(self):
         bad_results = ['www.foo.com', 'example', 't.www.bar.co',
                        '69.255.225.155']
 
-        for name in results:
-            vhost = self.config.choose_vhosts(name)[0]
-            path = os.path.relpath(vhost.filep, self.temp_dir)
-
-            self.assertEqual(results[name], vhost.names)
-            self.assertEqual(conf_path[name], path)
-            # IPv6 specific checks
-            if name == "ipv6.com":
-                self.assertTrue(vhost.ipv6_enabled())
-                # Make sure that we have SSL enabled also for IPv6 addr
-                self.assertTrue(
-                    any([True for x in vhost.addrs if x.ssl and x.ipv6]))
-
         for name in bad_results:
             self.assertRaises(errors.MisconfigurationError,
                               self.config.choose_vhosts, name)

certbot-nginx/certbot_nginx/tests/http_01_test.py

@@ -12,6 +12,7 @@ from certbot import achallenges
 from certbot.plugins import common_test
 from certbot.tests import acme_util
 
+from certbot_nginx.obj import Addr
 from certbot_nginx.tests import util
 
 
@@ -108,6 +109,41 @@ class HttpPerformTest(util.NginxTest):
             #     self.assertEqual(vhost.addrs, set(v_addr2_print))
             # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')]))
 
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_no_memoization(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, True)
+        self.http01._default_listen_addresses()
+        self.assertEqual(ipv6_info.call_count, 1)
+        ipv6_info.return_value = (False, False)
+        self.http01._default_listen_addresses()
+        self.assertEqual(ipv6_info.call_count, 2)
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_t_t(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, True)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        http_ipv6_addr = Addr.fromstring("[::]:80")
+        self.assertEqual(addrs, [http_addr, http_ipv6_addr])
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_t_f(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, False)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        http_ipv6_addr = Addr.fromstring("[::]:80 ipv6only=on")
+        self.assertEqual(addrs, [http_addr, http_ipv6_addr])
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_f_f(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (False, False)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        self.assertEqual(addrs, [http_addr])
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tests/parser_obj_test.py

@@ -0,0 +1,253 @@
+""" Tests for functions and classes in parser_obj.py """
+
+import unittest
+import mock
+
+from certbot_nginx.parser_obj import parse_raw
+from certbot_nginx.parser_obj import COMMENT_BLOCK
+
+class CommentHelpersTest(unittest.TestCase):
+    def test_is_comment(self):
+        from certbot_nginx.parser_obj import _is_comment
+        self.assertTrue(_is_comment(parse_raw(['#'])))
+        self.assertTrue(_is_comment(parse_raw(['#', ' literally anything else'])))
+        self.assertFalse(_is_comment(parse_raw(['not', 'even', 'a', 'comment'])))
+
+    def test_is_certbot_comment(self):
+        from certbot_nginx.parser_obj import _is_certbot_comment
+        self.assertTrue(_is_certbot_comment(
+            parse_raw(COMMENT_BLOCK)))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['#', ' not a certbot comment'])))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['#', ' managed by Certbot', ' also not a certbot comment'])))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['not', 'even', 'a', 'comment'])))
+
+    def test_certbot_comment(self):
+        from certbot_nginx.parser_obj import _certbot_comment, _is_certbot_comment
+        comment = _certbot_comment(None)
+        self.assertTrue(_is_certbot_comment(comment))
+        self.assertEqual(comment.dump(), COMMENT_BLOCK)
+        self.assertEqual(comment.dump(True), ['    '] + COMMENT_BLOCK)
+        self.assertEqual(_certbot_comment(None, 2).dump(True),
+            ['  '] + COMMENT_BLOCK)
+
+class ParsingHooksTest(unittest.TestCase):
+    def test_is_sentence(self):
+        from certbot_nginx.parser_obj import Sentence
+        self.assertFalse(Sentence.should_parse([]))
+        self.assertTrue(Sentence.should_parse(['']))
+        self.assertTrue(Sentence.should_parse(['word']))
+        self.assertTrue(Sentence.should_parse(['two', 'words']))
+        self.assertFalse(Sentence.should_parse([[]]))
+        self.assertFalse(Sentence.should_parse(['word', []]))
+
+    def test_is_block(self):
+        from certbot_nginx.parser_obj import Block
+        self.assertFalse(Block.should_parse([]))
+        self.assertFalse(Block.should_parse(['']))
+        self.assertFalse(Block.should_parse(['two', 'words']))
+        self.assertFalse(Block.should_parse([[[]], []]))
+        self.assertFalse(Block.should_parse([['block_name'], ['hi', []], []]))
+        self.assertFalse(Block.should_parse([['block_name'], 'lol']))
+        self.assertTrue(Block.should_parse([['block_name'], ['hi', []]]))
+        self.assertTrue(Block.should_parse([['hello'], []]))
+        self.assertTrue(Block.should_parse([['block_name'], [['many'], ['statements'], 'here']]))
+        self.assertTrue(Block.should_parse([['if', ' ', '(whatever)'], ['hi']]))
+
+    def test_parse_raw(self):
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: True
+        fake_parser2 = mock.Mock()
+        fake_parser2.should_parse = lambda x: False
+        # First encountered "match" should parse.
+        parse_raw([])
+        fake_parser1.called_once()
+        fake_parser2.not_called()
+        fake_parser1.reset_mock()
+        # "match" that returns False shouldn't parse.
+        parse_raw([])
+        fake_parser1.not_called()
+        fake_parser2.called_once()
+
+    @mock.patch("certbot_nginx.parser_obj.Parsable.parsing_hooks")
+    def test_parse_raw_no_match(self, parsing_hooks):
+        from certbot import errors
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: False
+        parsing_hooks.return_value = (fake_parser1,)
+        self.assertRaises(errors.MisconfigurationError, parse_raw, [])
+        parsing_hooks.return_value = tuple()
+        self.assertRaises(errors.MisconfigurationError, parse_raw, [])
+
+    def test_parse_raw_passes_add_spaces(self):
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: True
+        parse_raw([])
+        fake_parser1.parse.called_with([None])
+        parse_raw([], add_spaces=True)
+        fake_parser1.parse.called_with([None, True])
+
+class SentenceTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Sentence
+        self.sentence = Sentence(None)
+
+    def test_parse_bad_sentence_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, 'lol')
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [[]])
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [5])
+
+    def test_parse_sentence_words_hides_spaces(self):
+        og_sentence = ['\r\n', 'hello', ' ', ' ', '\t\n  ', 'lol', ' ', 'spaces']
+        self.sentence.parse(og_sentence)
+        self.assertEquals(self.sentence.words, ['hello', 'lol', 'spaces'])
+        self.assertEquals(self.sentence.dump(), ['hello', 'lol', 'spaces'])
+        self.assertEquals(self.sentence.dump(True), og_sentence)
+
+    def test_parse_sentence_with_add_spaces(self):
+        self.sentence.parse(['hi', 'there'], add_spaces=True)
+        self.assertEquals(self.sentence.dump(True), ['hi', ' ', 'there'])
+        self.sentence.parse(['one', ' ', 'space', 'none'], add_spaces=True)
+        self.assertEquals(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none'])
+
+    def test_iterate(self):
+        expected = [['1', '2', '3']]
+        self.sentence.parse(['1', ' ', '2', ' ', '3'])
+        for i, sentence in enumerate(self.sentence.iterate()):
+            self.assertEquals(sentence.dump(), expected[i])
+
+    def test_set_tabs(self):
+        self.sentence.parse(['tabs', 'pls'], add_spaces=True)
+        self.sentence.set_tabs()
+        self.assertEquals(self.sentence.dump(True)[0], '\n    ')
+        self.sentence.parse(['tabs', 'pls'], add_spaces=True)
+
+    def test_get_tabs(self):
+        self.sentence.parse(['no', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '')
+        self.sentence.parse(['\n \n  ', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '  ')
+        self.sentence.parse(['\n\t  ', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '\t  ')
+        self.sentence.parse(['\n\t \n', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '')
+
+class BlockTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Block
+        self.bloc = Block(None)
+        self.name = ['server', 'name']
+        self.contents = [['thing', '1'], ['thing', '2'], ['another', 'one']]
+        self.bloc.parse([self.name, self.contents])
+
+    def test_iterate(self):
+        # Iterates itself normally
+        self.assertEquals(self.bloc, next(self.bloc.iterate()))
+        # Iterates contents while expanded
+        expected = [self.bloc.dump()] + self.contents
+        for i, elem in enumerate(self.bloc.iterate(expanded=True)):
+            self.assertEquals(expected[i], elem.dump())
+
+    def test_iterate_match(self):
+        # can match on contents while expanded
+        from certbot_nginx.parser_obj import Block, Sentence
+        expected = [['thing', '1'], ['thing', '2']]
+        for i, elem in enumerate(self.bloc.iterate(expanded=True,
+            match=lambda x: isinstance(x, Sentence) and 'thing' in x.words)):
+            self.assertEquals(expected[i], elem.dump())
+        # can match on self
+        self.assertEquals(self.bloc, next(self.bloc.iterate(
+            expanded=True,
+            match=lambda x: isinstance(x, Block) and 'server' in x.names)))
+
+    def test_parse_with_added_spaces(self):
+        import copy
+        self.bloc.parse([copy.copy(self.name), self.contents], add_spaces=True)
+        self.assertEquals(self.bloc.dump(), [self.name, self.contents])
+        self.assertEquals(self.bloc.dump(True), [
+            ['server', ' ', 'name', ' '],
+            [['thing', ' ', '1'],
+             ['thing', ' ', '2'],
+             ['another', ' ', 'one']]])
+
+    def test_bad_parse_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, [[[]], [[]]])
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['lol'])
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['fake', 'news'])
+
+    def test_set_tabs(self):
+        self.bloc.set_tabs()
+        self.assertEquals(self.bloc.names.dump(True)[0], '\n    ')
+        for elem in self.bloc.contents.dump(True)[:-1]:
+            self.assertEquals(elem[0], '\n        ')
+        self.assertEquals(self.bloc.contents.dump(True)[-1][0], '\n')
+
+    def test_get_tabs(self):
+        self.bloc.parse([[' \n  \t', 'lol'], []])
+        self.assertEquals(self.bloc.get_tabs(), '  \t')
+
+class StatementsTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Statements
+        self.statements = Statements(None)
+        self.raw = [
+            ['sentence', 'one'],
+            ['sentence', 'two'],
+            ['and', 'another']
+        ]
+        self.raw_spaced = [
+            ['\n  ', 'sentence', ' ', 'one'],
+            ['\n  ', 'sentence', ' ', 'two'],
+            ['\n  ', 'and', ' ', 'another'],
+            '\n\n'
+        ]
+
+    def test_set_tabs(self):
+        self.statements.parse(self.raw)
+        self.statements.set_tabs()
+        for statement in self.statements.iterate():
+            self.assertEquals(statement.dump(True)[0], '\n    ')
+
+    def test_set_tabs_with_parent(self):
+        # Trailing whitespace should inherit from parent tabbing.
+        self.statements.parse(self.raw)
+        self.statements.parent = mock.Mock()
+        self.statements.parent.get_tabs.return_value = '\t\t'
+        self.statements.set_tabs()
+        for statement in self.statements.iterate():
+            self.assertEquals(statement.dump(True)[0], '\n    ')
+        self.assertEquals(self.statements.dump(True)[-1], '\n\t\t')
+
+    def test_get_tabs(self):
+        self.raw[0].insert(0, '\n \n  \t')
+        self.statements.parse(self.raw)
+        self.assertEquals(self.statements.get_tabs(), '  \t')
+        self.statements.parse([])
+        self.assertEquals(self.statements.get_tabs(), '')
+
+    def test_parse_with_added_spaces(self):
+        self.statements.parse(self.raw, add_spaces=True)
+        self.assertEquals(self.statements.dump(True)[0], ['sentence', ' ', 'one'])
+
+    def test_parse_bad_list_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.statements.parse, 'lol not a list')
+
+    def test_parse_hides_trailing_whitespace(self):
+        self.statements.parse(self.raw + ['\n\n  '])
+        self.assertTrue(isinstance(self.statements.dump()[-1], list))
+        self.assertTrue(self.statements.dump(True)[-1].isspace())
+        self.assertEquals(self.statements.dump(True)[-1], '\n\n  ')
+
+    def test_iterate(self):
+        self.statements.parse(self.raw)
+        expected = [['sentence', 'one'], ['sentence', 'two']]
+        for i, elem in enumerate(self.statements.iterate(match=lambda x: 'sentence' in x)):
+            self.assertEquals(expected[i], elem.dump())
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tls_sni_01.py

@@ -51,9 +51,6 @@ class NginxTlsSni01(common.TLSSNI01):
         default_addr = "{0} ssl".format(
             self.configurator.config.tls_sni_01_port)
 
-        ipv6, ipv6only = self.configurator.ipv6_info(
-            self.configurator.config.tls_sni_01_port)
-
         for achall in self.achalls:
             vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True)
 
@@ -61,6 +58,9 @@ class NginxTlsSni01(common.TLSSNI01):
             if vhosts and vhosts[0].addrs:
                 addresses.append(list(vhosts[0].addrs))
             else:
+                # choose_vhosts might have modified vhosts, so put this after
+                ipv6, ipv6only = self.configurator.ipv6_info(
+                    self.configurator.config.tls_sni_01_port)
                 if ipv6:
                     # If IPv6 is active in Nginx configuration
                     ipv6_addr = "[::]:{0} ssl".format(
@@ -141,6 +141,8 @@ class NginxTlsSni01(common.TLSSNI01):
         with open(self.challenge_conf, "w") as new_conf:
             nginxparser.dump(config, new_conf)
 
+        logger.debug("Generated server block:\n%s", str(config))
+
     def _make_server_block(self, achall, addrs):
         """Creates a server block for a challenge.
 

certbot-nginx/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.28.0.dev0'
+version = '0.29.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-nginx/tests/boulder-integration.sh

@@ -35,9 +35,12 @@ test_deployment_and_rollback() {
 }
 
 export default_server="default_server"
+nginx -v
 reload_nginx
 certbot_test_nginx --domains nginx.wtf run
 test_deployment_and_rollback nginx.wtf
+certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni
+test_deployment_and_rollback nginx.wtf
 certbot_test_nginx --domains nginx2.wtf --preferred-challenges http
 test_deployment_and_rollback nginx2.wtf
 # Overlapping location block and server-block-level return 301

certbot-postfix/README.rst

@@ -7,7 +7,7 @@ feature requests for this plugin.
 
 To install this plugin, in the root of this repo, run::
 
-    ./tools/venv.sh
+    python tools/venv.py
     source venv/bin/activate
 
 You can use this installer with any `authenticator plugin

certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.28.0.dev0'
+__version__ = '0.29.0.dev0'

certbot/auth_handler.py

@@ -113,6 +113,12 @@ class AuthHandler(object):
                 aauthzr.authzr, path)
             aauthzr.achalls.extend(aauthzr_achalls)
 
+        for aauthzr in aauthzrs:
+            for achall in aauthzr.achalls:
+                if isinstance(achall.chall, challenges.TLSSNI01):
+                    logger.warning("TLS-SNI-01 is deprecated, and will stop working soon.")
+                    return
+
     def _has_challenges(self, aauthzrs):
         """Do we have any challenges to perform?"""
         return any(aauthzr.achalls for aauthzr in aauthzrs)

certbot/compat.py

@@ -2,7 +2,7 @@
 Compatibility layer to run certbot both on Linux and Windows.
 
 The approach used here is similar to Modernizr for Web browsers.
-We do not check the plateform type to determine if a particular logic is supported.
+We do not check the platform type to determine if a particular logic is supported.
 Instead, we apply a logic, and then fallback to another logic if first logic
 is not supported at runtime.
 
@@ -13,6 +13,7 @@ import select
 import sys
 import errno
 import ctypes
+import stat
 
 from certbot import errors
 
@@ -64,6 +65,30 @@ def os_geteuid():
         # Windows specific
         return 0
 
+def os_rename(src, dst):
+    """
+    Rename a file to a destination path and handles situations where the destination exists.
+
+    :param str src: The current file path.
+    :param str dst: The new file path.
+    """
+    try:
+        os.rename(src, dst)
+    except OSError as err:
+        # Windows specific, renaming a file on an existing path is not possible.
+        # On Python 3, the best fallback with atomic capabilities we have is os.replace.
+        if err.errno != errno.EEXIST:
+            # Every other error is a legitimate exception.
+            raise
+        if not hasattr(os, 'replace'):  # pragma: no cover
+            # We should never go on this line. Either we are on Linux and os.rename has succeeded,
+            # either we are on Windows, and only Python >= 3.4 is supported where os.replace is
+            # available.
+            raise RuntimeError('Error: tried to run os_rename on Python < 3.3. '
+                               'Certbot supports only Python 3.4 >= on Windows.')
+        getattr(os, 'replace')(src, dst)
+
+
 def readline_with_timeout(timeout, prompt):
     """
     Read user input to return the first line entered, or raise after specified timeout.
@@ -138,3 +163,39 @@ def release_locked_file(fd, path):
             raise
     finally:
         os.close(fd)
+
+def compare_file_modes(mode1, mode2):
+    """Return true if the two modes can be considered as equals for this platform"""
+    if 'fcntl' in sys.modules:
+        # Linux specific: standard compare
+        return oct(stat.S_IMODE(mode1)) == oct(stat.S_IMODE(mode2))
+    # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights.
+    return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD
+            and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE)
+
+WINDOWS_DEFAULT_FOLDERS = {
+    'config': 'C:\\Certbot',
+    'work': 'C:\\Certbot\\lib',
+    'logs': 'C:\\Certbot\\log',
+}
+LINUX_DEFAULT_FOLDERS = {
+    'config': '/etc/letsencrypt',
+    'work': '/var/letsencrypt/lib',
+    'logs': '/var/letsencrypt/log',
+}
+
+def get_default_folder(folder_type):
+    """
+    Return the relevant default folder for the current OS
+
+    :param str folder_type: The type of folder to retrieve (config, work or logs)
+
+    :returns: The relevant default folder.
+    :rtype: str
+
+    """
+    if 'fcntl' in sys.modules:
+        # Linux specific
+        return LINUX_DEFAULT_FOLDERS[folder_type]
+    # Windows specific
+    return WINDOWS_DEFAULT_FOLDERS[folder_type]

certbot/constants.py

@@ -4,7 +4,7 @@ import os
 import pkg_resources
 
 from acme import challenges
-
+from certbot import compat
 
 SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins"
 """Setuptools entry point group name for plugins."""
@@ -14,7 +14,7 @@ OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT = "letsencrypt.plugins"
 
 CLI_DEFAULTS = dict(
     config_files=[
-        "/etc/letsencrypt/cli.ini",
+        os.path.join(compat.get_default_folder('config'), 'cli.ini'),
         # http://freedesktop.org/wiki/Software/xdg-user-dirs/
         os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"),
                      "letsencrypt", "cli.ini"),
@@ -85,9 +85,9 @@ CLI_DEFAULTS = dict(
     auth_cert_path="./cert.pem",
     auth_chain_path="./chain.pem",
     key_path=None,
-    config_dir="/etc/letsencrypt",
-    work_dir="/var/lib/letsencrypt",
-    logs_dir="/var/log/letsencrypt",
+    config_dir=compat.get_default_folder('config'),
+    work_dir=compat.get_default_folder('work'),
+    logs_dir=compat.get_default_folder('logs'),
     server="https://acme-v02.api.letsencrypt.org/directory",
 
     # Plugins parsers

certbot/crypto_util.py

@@ -449,14 +449,17 @@ def _notAfterBefore(cert_path, method):
 def sha256sum(filename):
     """Compute a sha256sum of a file.
 
+    NB: In given file, platform specific newlines characters will be converted
+    into their equivalent unicode counterparts before calculating the hash.
+
     :param str filename: path to the file whose hash will be computed
 
     :returns: sha256 digest of the file in hexadecimal
     :rtype: str
     """
     sha256 = hashlib.sha256()
-    with open(filename, 'rb') as f:
-        sha256.update(f.read())
+    with open(filename, 'rU') as file_d:
+        sha256.update(file_d.read().encode('UTF-8'))
     return sha256.hexdigest()
 
 def cert_and_chain_from_fullchain(fullchain_pem):

certbot/display/completer.py

@@ -49,9 +49,9 @@ class Completer(object):
         readline.set_completer(self.complete)
         readline.set_completer_delims(' \t\n;')
 
-        # readline can be implemented using GNU readline or libedit
+        # readline can be implemented using GNU readline, pyreadline or libedit
         # which have different configuration syntax
-        if 'libedit' in readline.__doc__:
+        if readline.__doc__ is not None and 'libedit' in readline.__doc__:
             readline.parse_and_bind('bind ^I rl_complete')
         else:
             readline.parse_and_bind('tab: complete')

certbot/display/ops.py

@@ -4,9 +4,11 @@ import os
 
 import zope.component
 
+from certbot import compat
 from certbot import errors
 from certbot import interfaces
 from certbot import util
+
 from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
@@ -33,7 +35,8 @@ def get_email(invalid=False, optional=True):
     unsafe_suggestion = ("\n\nIf you really want to skip this, you can run "
                          "the client with --register-unsafely-without-email "
                          "but make sure you then backup your account key from "
-                         "/etc/letsencrypt/accounts\n\n")
+                         "{0}\n\n".format(os.path.join(
+                             compat.get_default_folder('config'), 'accounts')))
     if optional:
         if invalid:
             msg += unsafe_suggestion

certbot/display/util.py

@@ -53,7 +53,7 @@ def _wrap_lines(msg):
             break_long_words=False,
             break_on_hyphens=False))
 
-    return os.linesep.join(fixed_l)
+    return '\n'.join(fixed_l)
 
 
 def input_with_timeout(prompt=None, timeout=36000.0):

certbot/main.py

@@ -4,7 +4,9 @@ from __future__ import print_function
 import functools
 import logging.handlers
 import os
+import random
 import sys
+import time
 
 import configobj
 import josepy as jose
@@ -1135,7 +1137,8 @@ def _csr_get_and_save_cert(config, le_client):
             "Dry run: skipping saving certificate to %s", config.cert_path)
         return None, None
     cert_path, _, fullchain_path = le_client.save_certificate(
-            cert, chain, config.cert_path, config.chain_path, config.fullchain_path)
+        cert, chain, os.path.normpath(config.cert_path),
+        os.path.normpath(config.chain_path), os.path.normpath(config.fullchain_path))
     return cert_path, fullchain_path
 
 def renew_cert(config, plugins, lineage):
@@ -1242,6 +1245,16 @@ def renew(config, unused_plugins):
     :rtype: None
 
     """
+    if not sys.stdin.isatty():
+        # Noninteractive renewals include a random delay in order to spread
+        # out the load on the certificate authority servers, even if many
+        # users all pick the same time for renewals.  This delay precedes
+        # running any hooks, so that side effects of the hooks (such as
+        # shutting down a web service) aren't prolonged unnecessarily.
+        sleep_time = random.randint(1, 60*8)
+        logger.info("Non-interactive renewal: random delay of %s seconds", sleep_time)
+        time.sleep(sleep_time)
+
     try:
         renewal.handle_renewal_request(config)
     finally:

certbot/plugins/dns_common_lexicon.py

@@ -68,7 +68,12 @@ class LexiconClient(object):
 
         for domain_name in domain_name_guesses:
             try:
-                self.provider.options['domain'] = domain_name
+                if hasattr(self.provider, 'options'):
+                    # For Lexicon 2.x
+                    self.provider.options['domain'] = domain_name
+                else:
+                    # For Lexicon 3.x
+                    self.provider.domain = domain_name
 
                 self.provider.authenticate()
 

certbot/plugins/manual.py

@@ -94,6 +94,16 @@ using the secret key
 {key}
 when it receives a TLS ClientHello with the SNI extension set to
 {sni_domain}
+"""
+    _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """
+(This must be set up in addition to the previous challenges; do not remove,
+replace, or undo the previous challenge tasks yet.)
+"""
+    _SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS = """
+(This must be set up in addition to the previous challenges; do not remove,
+replace, or undo the previous challenge tasks yet. Note that you might be
+asked to create multiple distinct TXT records with the same name. This is
+permitted by DNS standards.)
 """
 
     def __init__(self, *args, **kwargs):
@@ -103,6 +113,8 @@ when it receives a TLS ClientHello with the SNI extension set to
         self.env = dict() \
         # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]]
         self.tls_sni_01 = None
+        self.subsequent_dns_challenge = False
+        self.subsequent_any_challenge = False
 
     @classmethod
     def add_parser_arguments(cls, add):
@@ -212,8 +224,17 @@ when it receives a TLS ClientHello with the SNI extension set to
                 key=self.tls_sni_01.get_key_path(achall),
                 port=self.config.tls_sni_01_port,
                 sni_domain=self.tls_sni_01.get_z_domain(achall))
+        if isinstance(achall.chall, challenges.DNS01):
+            if self.subsequent_dns_challenge:
+                # 2nd or later dns-01 challenge
+                msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS
+            self.subsequent_dns_challenge = True
+        elif self.subsequent_any_challenge:
+            # 2nd or later challenge of another type
+            msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS
         display = zope.component.getUtility(interfaces.IDisplay)
         display.notification(msg, wrap=False, force_interactive=True)
+        self.subsequent_any_challenge = True
 
     def cleanup(self, achalls):  # pylint: disable=missing-docstring
         if self.conf('cleanup-hook'):

certbot/plugins/manual_test.py

@@ -4,6 +4,7 @@ import unittest
 
 import six
 import mock
+import sys
 
 from acme import challenges
 
@@ -20,8 +21,9 @@ class AuthenticatorTest(test_util.TempDirTestCase):
         super(AuthenticatorTest, self).setUp()
         self.http_achall = acme_util.HTTP01_A
         self.dns_achall = acme_util.DNS01_A
+        self.dns_achall_2 = acme_util.DNS01_A_2
         self.tls_sni_achall = acme_util.TLSSNI01_A
-        self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall]
+        self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2]
         for d in ["config_dir", "work_dir", "in_progress"]:
             os.mkdir(os.path.join(self.tempdir, d))
             # "backup_dir" and "temp_checkpoint_dir" get created in
@@ -74,12 +76,14 @@ class AuthenticatorTest(test_util.TempDirTestCase):
     def test_script_perform(self):
         self.config.manual_public_ip_logging_ok = True
         self.config.manual_auth_hook = (
-            'echo ${CERTBOT_DOMAIN}; '
-            'echo ${CERTBOT_TOKEN:-notoken}; '
-            'echo ${CERTBOT_CERT_PATH:-nocert}; '
-            'echo ${CERTBOT_KEY_PATH:-nokey}; '
-            'echo ${CERTBOT_SNI_DOMAIN:-nosnidomain}; '
-            'echo ${CERTBOT_VALIDATION:-novalidation};')
+            '{0} -c "from __future__ import print_function;'
+            'import os;  print(os.environ.get(\'CERTBOT_DOMAIN\'));'
+            'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));'
+            'print(os.environ.get(\'CERTBOT_CERT_PATH\', \'nocert\'));'
+            'print(os.environ.get(\'CERTBOT_KEY_PATH\', \'nokey\'));'
+            'print(os.environ.get(\'CERTBOT_SNI_DOMAIN\', \'nosnidomain\'));'
+            'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"'
+            .format(sys.executable))
         dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format(
             self.dns_achall.domain, 'notoken',
             'nocert', 'nokey', 'nosnidomain',
@@ -127,6 +131,7 @@ class AuthenticatorTest(test_util.TempDirTestCase):
                         achall.validation(achall.account_key) in args[0])
             self.assertFalse(kwargs['wrap'])
 
+    @test_util.broken_on_windows
     def test_cleanup(self):
         self.config.manual_public_ip_logging_ok = True
         self.config.manual_auth_hook = 'echo foo;'

certbot/plugins/standalone.py

@@ -114,7 +114,7 @@ class ServerManager(object):
         return self._instances.copy()
 
 
-SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] \
+SUPPORTED_CHALLENGES = [challenges.HTTP01, challenges.TLSSNI01] \
 # type: List[Type[challenges.KeyAuthorizationChallenge]]
 
 

certbot/plugins/util_test.py

@@ -4,13 +4,14 @@ import unittest
 
 import mock
 
-
 class GetPrefixTest(unittest.TestCase):
     """Tests for certbot.plugins.get_prefixes."""
     def test_get_prefix(self):
         from certbot.plugins.util import get_prefixes
-        self.assertEqual(get_prefixes('/a/b/c'), ['/a/b/c', '/a/b', '/a', '/'])
-        self.assertEqual(get_prefixes('/'), ['/'])
+        self.assertEqual(
+            get_prefixes('/a/b/c'),
+            [os.path.normpath(path) for path in ['/a/b/c', '/a/b', '/a', '/']])
+        self.assertEqual(get_prefixes('/'), [os.path.normpath('/')])
         self.assertEqual(get_prefixes('a'), ['a'])
 
 class PathSurgeryTest(unittest.TestCase):

certbot/plugins/webroot_test.py

@@ -4,9 +4,9 @@ from __future__ import print_function
 
 import argparse
 import errno
+import json
 import os
 import shutil
-import stat
 import tempfile
 import unittest
 
@@ -17,6 +17,7 @@ import six
 from acme import challenges
 
 from certbot import achallenges
+from certbot import compat
 from certbot import errors
 from certbot.display import util as display_util
 
@@ -142,6 +143,7 @@ class AuthenticatorTest(unittest.TestCase):
             self.assertRaises(errors.PluginError, self.auth.perform, [])
         os.chmod(self.path, 0o700)
 
+    @test_util.skip_on_windows('On Windows, there is no chown.')
     @mock.patch("certbot.plugins.webroot.os.chown")
     def test_failed_chown(self, mock_chown):
         mock_chown.side_effect = OSError(errno.EACCES, "msg")
@@ -169,16 +171,14 @@ class AuthenticatorTest(unittest.TestCase):
         # Remove exec bit from permission check, so that it
         # matches the file
         self.auth.perform([self.achall])
-        path_permissions = stat.S_IMODE(os.stat(self.validation_path).st_mode)
-        self.assertEqual(path_permissions, 0o644)
+        self.assertTrue(compat.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644))
 
         # Check permissions of the directories
 
         for dirpath, dirnames, _ in os.walk(self.path):
             for directory in dirnames:
                 full_path = os.path.join(dirpath, directory)
-                dir_permissions = stat.S_IMODE(os.stat(full_path).st_mode)
-                self.assertEqual(dir_permissions, 0o755)
+                self.assertTrue(compat.compare_file_modes(os.stat(full_path).st_mode, 0o755))
 
         parent_gid = os.stat(self.path).st_gid
         parent_uid = os.stat(self.path).st_uid
@@ -274,7 +274,7 @@ class WebrootActionTest(unittest.TestCase):
 
     def test_webroot_map_action(self):
         args = self.parser.parse_args(
-            ["--webroot-map", '{{"thing.com":"{0}"}}'.format(self.path)])
+            ["--webroot-map", json.dumps({'thing.com': self.path})])
         self.assertEqual(args.webroot_map["thing.com"], self.path)
 
     def test_domain_before_webroot(self):

certbot/renewal.py

@@ -301,7 +301,7 @@ def renew_cert(config, domains, le_client, lineage):
         domains = lineage.names()
     # The private key is the existing lineage private key if reuse_key is set.
     # Otherwise, generate a fresh private key by passing None.
-    new_key = lineage.privkey if config.reuse_key else None
+    new_key = os.path.normpath(lineage.privkey) if config.reuse_key else None
     new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key)
     if config.dry_run:
         logger.debug("Dry run: skipping updating lineage at %s",

certbot/reverter.py

@@ -576,7 +576,7 @@ class Reverter(object):
             timestamp = self._checkpoint_timestamp()
             final_dir = os.path.join(self.config.backup_dir, timestamp)
             try:
-                os.rename(self.config.in_progress_dir, final_dir)
+                compat.os_rename(self.config.in_progress_dir, final_dir)
                 return
             except OSError:
                 logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp)

certbot/storage.py

@@ -14,6 +14,7 @@ import six
 
 import certbot
 from certbot import cli
+from certbot import compat
 from certbot import constants
 from certbot import crypto_util
 from certbot import errors
@@ -188,7 +189,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config):
     # Save only the config items that are relevant to renewal
     values = relevant_values(vars(cli_config.namespace))
     write_renewal_config(config_filename, temp_filename, archive_dir, target, values)
-    os.rename(temp_filename, config_filename)
+    compat.os_rename(temp_filename, config_filename)
 
     return configobj.ConfigObj(config_filename)
 
@@ -214,6 +215,26 @@ def get_link_target(link):
         target = os.path.join(os.path.dirname(link), target)
     return os.path.abspath(target)
 
+def _write_live_readme_to(readme_path, is_base_dir=False):
+    prefix = ""
+    if is_base_dir:
+        prefix = "[cert name]/"
+    with open(readme_path, "w") as f:
+        logger.debug("Writing README to %s.", readme_path)
+        f.write("This directory contains your keys and certificates.\n\n"
+                "`{prefix}privkey.pem`  : the private key for your certificate.\n"
+                "`{prefix}fullchain.pem`: the certificate file used in most server software.\n"
+                "`{prefix}chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.\n"
+                "`{prefix}cert.pem`     : will break many server configurations, and "
+                                    "should not be used\n"
+                "                 without reading further documentation (see link below).\n\n"
+                "WARNING: DO NOT MOVE OR RENAME THESE FILES!\n"
+                "         Certbot expects these files to remain in this location in order\n"
+                "         to function properly!\n\n"
+                "We recommend not moving these files. For more information, see the Certbot\n"
+                "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-"
+                                    "certificates.\n".format(prefix=prefix))
+
 
 def _relevant(option):
     """
@@ -1003,6 +1024,9 @@ class RenewableCert(object):
                 logger.debug("Creating directory %s.", i)
         config_file, config_filename = util.unique_lineage_name(
             cli_config.renewal_configs_dir, lineagename)
+        base_readme_path = os.path.join(cli_config.live_dir, README)
+        if not os.path.exists(base_readme_path):
+            _write_live_readme_to(base_readme_path, is_base_dir=True)
 
         # Determine where on disk everything will go
         # lineagename will now potentially be modified based on which
@@ -1045,21 +1069,7 @@ class RenewableCert(object):
 
         # Write a README file to the live directory
         readme_path = os.path.join(live_dir, README)
-        with open(readme_path, "w") as f:
-            logger.debug("Writing README to %s.", readme_path)
-            f.write("This directory contains your keys and certificates.\n\n"
-                    "`privkey.pem`  : the private key for your certificate.\n"
-                    "`fullchain.pem`: the certificate file used in most server software.\n"
-                    "`chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.\n"
-                    "`cert.pem`     : will break many server configurations, and "
-                                        "should not be used\n"
-                    "                 without reading further documentation (see link below).\n\n"
-                    "WARNING: DO NOT MOVE THESE FILES!\n"
-                    "         Certbot expects these files to remain in this location in order\n"
-                    "         to function properly!\n\n"
-                    "We recommend not moving these files. For more information, see the Certbot\n"
-                    "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-"
-                                        "certificates.\n")
+        _write_live_readme_to(readme_path)
 
         # Document what we've done in a new renewal config file
         config_file.close()

certbot/tests/account_test.py

@@ -116,6 +116,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
     def test_init_creates_dir(self):
         self.assertTrue(os.path.isdir(self.config.accounts_dir))
 
+    @test_util.broken_on_windows
     def test_save_and_restore(self):
         self.storage.save(self.acc, self.mock_client)
         account_path = os.path.join(self.config.accounts_dir, self.acc.id)
@@ -218,12 +219,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.assertEqual([], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_version_staging(self):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertEqual([self.acc], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_version_production(self):
         self._set_server('https://acme-v01.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
@@ -241,6 +244,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertEqual([], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_load(self):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
@@ -249,6 +253,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         account = self.storage.load(self.acc.id)
         self.assertEqual(prev_account, account)
 
+    @test_util.broken_on_windows
     def test_upgrade_load_single_account(self):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
@@ -273,6 +278,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
                 errors.AccountStorageError, self.storage.save,
                     self.acc, self.mock_client)
 
+    @test_util.broken_on_windows
     def test_delete(self):
         self.storage.save(self.acc, self.mock_client)
         self.storage.delete(self.acc.id)
@@ -307,10 +313,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id)
 
+    @test_util.broken_on_windows
     def test_delete_folders_up(self):
         self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory')
         self._assert_symlinked_account_removed()
 
+    @test_util.broken_on_windows
     def test_delete_folders_down(self):
         self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory')
         self._assert_symlinked_account_removed()
@@ -320,10 +328,12 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f:
             f.write('bar')
 
+    @test_util.broken_on_windows
     def test_delete_shared_account_up(self):
         self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory')
         self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory')
 
+    @test_util.broken_on_windows
     def test_delete_shared_account_down(self):
         self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory')
         self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory')

certbot/tests/acme_util.py

@@ -21,6 +21,7 @@ HTTP01 = challenges.HTTP01(
 TLSSNI01 = challenges.TLSSNI01(
     token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA"))
 DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a")
+DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac")
 
 CHALLENGES = [HTTP01, TLSSNI01, DNS01]
 
@@ -49,6 +50,7 @@ def chall_to_challb(chall, status):  # pylint: disable=redefined-outer-name
 TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING)
 HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING)
 DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING)
+DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING)
 
 CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P]
 
@@ -57,6 +59,7 @@ CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P]
 HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com")
 TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net")
 DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org")
+DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org")
 
 ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A]
 

certbot/tests/auth_handler_test.py

@@ -327,6 +327,11 @@ class HandleAuthorizationsTest(unittest.TestCase):
                 azr.body.combinations)
             aauthzrs[i] = type(aauthzr)(updated_azr, aauthzr.achalls)
 
+    @mock.patch("certbot.auth_handler.logger")
+    def test_tls_sni_logs(self, logger):
+        self._test_name1_tls_sni_01_1_common(combos=True)
+        self.assertTrue("deprecated" in logger.warning.call_args[0][0])
+
 
 class PollChallengesTest(unittest.TestCase):
     # pylint: disable=protected-access

certbot/tests/cert_manager_test.py

@@ -204,7 +204,7 @@ class CertificatesTest(BaseCertManagerTest):
         shutil.rmtree(empty_tempdir)
 
     @mock.patch('certbot.cert_manager.ocsp.RevocationChecker.ocsp_revoked')
-    def test_report_human_readable(self, mock_revoked):
+    def test_report_human_readable(self, mock_revoked): #pylint: disable=too-many-statements
         mock_revoked.return_value = None
         from certbot import cert_manager
         import datetime, pytz
@@ -228,7 +228,7 @@ class CertificatesTest(BaseCertManagerTest):
         cert.target_expiry += datetime.timedelta(hours=2)
         # pylint: disable=protected-access
         out = get_report()
-        self.assertTrue('1 hour(s)' in out)
+        self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out)
         self.assertTrue('VALID' in out and not 'INVALID' in out)
 
         cert.target_expiry += datetime.timedelta(days=1)

certbot/tests/cli_test.py

@@ -76,6 +76,7 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
         return output.getvalue()
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.cli.flag_default")
     def test_cli_ini_domains(self, mock_flag_default):
         tmp_config = tempfile.NamedTemporaryFile()

certbot/tests/compat_test.py

@@ -0,0 +1,21 @@
+"""Tests for certbot.compat."""
+import os
+
+from certbot import compat
+import certbot.tests.util as test_util
+
+class OsReplaceTest(test_util.TempDirTestCase):
+    """Test to ensure consistent behavior of os_rename method"""
+
+    def test_os_rename_to_existing_file(self):
+        """Ensure that os_rename will effectively rename src into dst for all platforms."""
+        src = os.path.join(self.tempdir, 'src')
+        dst = os.path.join(self.tempdir, 'dst')
+        open(src, 'w').close()
+        open(dst, 'w').close()
+
+        # On Windows, a direct call to os.rename will fail because dst already exists.
+        compat.os_rename(src, dst)
+
+        self.assertFalse(os.path.exists(src))
+        self.assertTrue(os.path.exists(dst))

certbot/tests/crypto_util_test.py

@@ -140,7 +140,7 @@ class ImportCSRFileTest(unittest.TestCase):
              util.CSR(file=csrfile,
                       data=data_pem,
                       form="pem"),
-             ["Example.com"],),
+             ["Example.com"]),
             self._call(csrfile, data))
 
     def test_pem_csr(self):
@@ -376,7 +376,6 @@ class NotAfterTest(unittest.TestCase):
 
 class Sha256sumTest(unittest.TestCase):
     """Tests for certbot.crypto_util.notAfter"""
-
     def test_sha256sum(self):
         from certbot.crypto_util import sha256sum
         self.assertEqual(sha256sum(CERT_PATH),

certbot/tests/display/completer_test.py

@@ -1,6 +1,9 @@
 """Test certbot.display.completer."""
 import os
-import readline
+try:
+    import readline # pylint: disable=import-error
+except ImportError:
+    import certbot.display.dummy_readline as readline # type: ignore
 import string
 import sys
 import unittest
@@ -9,9 +12,9 @@ import mock
 from six.moves import reload_module  # pylint: disable=import-error
 
 from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
-from certbot.tests.util import TempDirTestCase
+import certbot.tests.util as test_util
 
-class CompleterTest(TempDirTestCase):
+class CompleterTest(test_util.TempDirTestCase):
     """Test certbot.display.completer.Completer."""
 
     def setUp(self):
@@ -47,6 +50,8 @@ class CompleterTest(TempDirTestCase):
         completion = my_completer.complete(self.tempdir, num_paths)
         self.assertEqual(completion, None)
 
+    @unittest.skipIf('readline' not in sys.modules,
+                     reason='Not relevant if readline is not available.')
     def test_import_error(self):
         original_readline = sys.modules['readline']
         sys.modules['readline'] = None
@@ -91,7 +96,7 @@ class CompleterTest(TempDirTestCase):
 
 def enable_tab_completion(unused_command):
     """Enables readline tab completion using the system specific syntax."""
-    libedit = 'libedit' in readline.__doc__
+    libedit = readline.__doc__ is not None and 'libedit' in readline.__doc__
     command = 'bind ^I rl_complete' if libedit else 'tab: complete'
     readline.parse_and_bind(command)
 

certbot/tests/display/util_test.py

@@ -1,6 +1,5 @@
 """Test :mod:`certbot.display.util`."""
 import inspect
-import os
 import socket
 import tempfile
 import unittest
@@ -10,7 +9,6 @@ import mock
 
 from certbot import errors
 from certbot import interfaces
-
 from certbot.display import util as display_util
 
 
@@ -281,10 +279,10 @@ class FileOutputDisplayTest(unittest.TestCase):
         msg = ("This is just a weak test{0}"
                "This function is only meant to be for easy viewing{0}"
                "Test a really really really really really really really really "
-               "really really really really long line...".format(os.linesep))
+               "really really really really long line...".format('\n'))
         text = display_util._wrap_lines(msg)
 
-        self.assertEqual(text.count(os.linesep), 3)
+        self.assertEqual(text.count('\n'), 3)
 
     def test_get_valid_int_ans_valid(self):
         # pylint: disable=protected-access

certbot/tests/error_handler_test.py

@@ -10,6 +10,7 @@ import mock
 from acme.magic_typing import Callable, Dict, Union
 # pylint: enable=unused-import, no-name-in-module
 
+import certbot.tests.util as test_util
 
 def get_signals(signums):
     """Get the handlers for an iterable of signums."""
@@ -65,6 +66,8 @@ class ErrorHandlerTest(unittest.TestCase):
         self.init_func.assert_called_once_with(*self.init_args,
                                                **self.init_kwargs)
 
+    # On Windows, this test kills pytest itself !
+    @test_util.broken_on_windows
     def test_context_manager_with_signal(self):
         init_signals = get_signals(self.signals)
         with signal_receiver(self.signals) as signals_received:
@@ -95,6 +98,8 @@ class ErrorHandlerTest(unittest.TestCase):
                                                **self.init_kwargs)
         bad_func.assert_called_once_with()
 
+    # On Windows, this test kills pytest itself !
+    @test_util.broken_on_windows
     def test_bad_recovery_with_signal(self):
         sig1 = self.signals[0]
         sig2 = self.signals[-1]
@@ -144,5 +149,10 @@ class ExitHandlerTest(ErrorHandlerTest):
                                                **self.init_kwargs)
         func.assert_called_once_with()
 
+    # On Windows, this test kills pytest itself !
+    @test_util.broken_on_windows
+    def test_bad_recovery_with_signal(self):
+        super(ExitHandlerTest, self).test_bad_recovery_with_signal()
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot/tests/hook_test.py

@@ -37,6 +37,7 @@ class ValidateHookTest(util.TempDirTestCase):
         from certbot.hooks import validate_hook
         return validate_hook(*args, **kwargs)
 
+    @util.broken_on_windows
     def test_not_executable(self):
         file_path = os.path.join(self.tempdir, "foo")
         # create a non-executable file

certbot/tests/lock_test.py

@@ -10,6 +10,7 @@ from certbot import errors
 from certbot.tests import util as test_util
 
 
+@test_util.broken_on_windows
 class LockDirTest(test_util.TempDirTestCase):
     """Tests for certbot.lock.lock_dir."""
     @classmethod
@@ -24,6 +25,7 @@ class LockDirTest(test_util.TempDirTestCase):
         test_util.lock_and_call(assert_raises, lock_path)
 
 
+@test_util.broken_on_windows
 class LockFileTest(test_util.TempDirTestCase):
     """Tests for certbot.lock.LockFile."""
     @classmethod

certbot/tests/log_test.py

@@ -12,6 +12,7 @@ import six
 from acme import messages
 from acme.magic_typing import Optional  # pylint: disable=unused-import, no-name-in-module
 
+from certbot import compat
 from certbot import constants
 from certbot import errors
 from certbot import util
@@ -259,7 +260,7 @@ class TempHandlerTest(unittest.TestCase):
 
     def test_permissions(self):
         self.assertTrue(
-            util.check_permissions(self.handler.path, 0o600, os.getuid()))
+            util.check_permissions(self.handler.path, 0o600, compat.os_geteuid()))
 
     def test_delete(self):
         self.handler.close()

certbot/tests/main_test.py

@@ -4,6 +4,7 @@
 from __future__ import print_function
 
 import itertools
+import json
 import mock
 import os
 import shutil
@@ -11,6 +12,8 @@ import traceback
 import unittest
 import datetime
 import pytz
+import tempfile
+import sys
 
 import josepy as jose
 import six
@@ -517,6 +520,8 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
                               '--work-dir', self.config.work_dir,
                               '--logs-dir', self.config.logs_dir, '--text']
 
+        self.mock_sleep = mock.patch('time.sleep').start()
+
     def tearDown(self):
         # Reset globals in cli
         reload_module(cli)
@@ -588,12 +593,14 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
             self.assertTrue(message in str(exc))
         self.assertTrue(exc is not None)
 
+    @test_util.broken_on_windows
     def test_noninteractive(self):
         args = ['-n', 'certonly']
         self._cli_missing_flag(args, "specify a plugin")
         args.extend(['--standalone', '-d', 'eg.is'])
         self._cli_missing_flag(args, "register before running")
 
+    @test_util.broken_on_windows
     @mock.patch('certbot.main._report_new_cert')
     @mock.patch('certbot.main.client.acme_client.Client')
     @mock.patch('certbot.main._determine_account')
@@ -635,43 +642,46 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
     @mock.patch('certbot.main.plug_sel.record_chosen_plugins')
     @mock.patch('certbot.main.plug_sel.pick_installer')
     def test_installer_certname(self, _inst, _rec, mock_install):
-        mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain",
-                                      fullchain_path="/tmp/chain",
-                                      key_path="/tmp/privkey")
+        mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'),
+                                      chain_path=test_util.temp_join('chain'),
+                                      fullchain_path=test_util.temp_join('chain'),
+                                      key_path=test_util.temp_join('privkey'))
 
         with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin:
             mock_getlin.return_value = mock_lineage
             self._call(['install', '--cert-name', 'whatever'], mockisfile=True)
             call_config = mock_install.call_args[0][0]
-            self.assertEqual(call_config.cert_path, "/tmp/cert")
-            self.assertEqual(call_config.fullchain_path, "/tmp/chain")
-            self.assertEqual(call_config.key_path, "/tmp/privkey")
+            self.assertEqual(call_config.cert_path, test_util.temp_join('cert'))
+            self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain'))
+            self.assertEqual(call_config.key_path, test_util.temp_join('privkey'))
 
+    @test_util.broken_on_windows
     @mock.patch('certbot.main._install_cert')
     @mock.patch('certbot.main.plug_sel.record_chosen_plugins')
     @mock.patch('certbot.main.plug_sel.pick_installer')
     def test_installer_param_override(self, _inst, _rec, mock_install):
-        mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain",
-                                      fullchain_path="/tmp/chain",
-                                      key_path="/tmp/privkey")
+        mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'),
+                                      chain_path=test_util.temp_join('chain'),
+                                      fullchain_path=test_util.temp_join('chain'),
+                                      key_path=test_util.temp_join('privkey'))
         with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin:
             mock_getlin.return_value = mock_lineage
             self._call(['install', '--cert-name', 'whatever',
-                        '--key-path', '/tmp/overriding_privkey'], mockisfile=True)
+                        '--key-path', test_util.temp_join('overriding_privkey')], mockisfile=True)
             call_config = mock_install.call_args[0][0]
-            self.assertEqual(call_config.cert_path, "/tmp/cert")
-            self.assertEqual(call_config.fullchain_path, "/tmp/chain")
-            self.assertEqual(call_config.chain_path, "/tmp/chain")
-            self.assertEqual(call_config.key_path, "/tmp/overriding_privkey")
+            self.assertEqual(call_config.cert_path, test_util.temp_join('cert'))
+            self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain'))
+            self.assertEqual(call_config.chain_path, test_util.temp_join('chain'))
+            self.assertEqual(call_config.key_path, test_util.temp_join('overriding_privkey'))
 
             mock_install.reset()
 
             self._call(['install', '--cert-name', 'whatever',
-                        '--cert-path', '/tmp/overriding_cert'], mockisfile=True)
+                        '--cert-path', test_util.temp_join('overriding_cert')], mockisfile=True)
             call_config = mock_install.call_args[0][0]
-            self.assertEqual(call_config.cert_path, "/tmp/overriding_cert")
-            self.assertEqual(call_config.fullchain_path, "/tmp/chain")
-            self.assertEqual(call_config.key_path, "/tmp/privkey")
+            self.assertEqual(call_config.cert_path, test_util.temp_join('overriding_cert'))
+            self.assertEqual(call_config.fullchain_path, test_util.temp_join('chain'))
+            self.assertEqual(call_config.key_path, test_util.temp_join('privkey'))
 
     @mock.patch('certbot.main.plug_sel.record_chosen_plugins')
     @mock.patch('certbot.main.plug_sel.pick_installer')
@@ -686,15 +696,17 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
     @mock.patch('certbot.cert_manager.get_certnames')
     @mock.patch('certbot.main._install_cert')
     def test_installer_select_cert(self, mock_inst, mock_getcert, _inst, _rec):
-        mock_lineage = mock.MagicMock(cert_path="/tmp/cert", chain_path="/tmp/chain",
-                                      fullchain_path="/tmp/chain",
-                                      key_path="/tmp/privkey")
+        mock_lineage = mock.MagicMock(cert_path=test_util.temp_join('cert'),
+                                      chain_path=test_util.temp_join('chain'),
+                                      fullchain_path=test_util.temp_join('chain'),
+                                      key_path=test_util.temp_join('privkey'))
         with mock.patch("certbot.cert_manager.lineage_for_certname") as mock_getlin:
             mock_getlin.return_value = mock_lineage
             self._call(['install'], mockisfile=True)
         self.assertTrue(mock_getcert.called)
         self.assertTrue(mock_inst.called)
 
+    @test_util.broken_on_windows
     @mock.patch('certbot.main._report_new_cert')
     @mock.patch('certbot.util.exe_exists')
     def test_configurator_selection(self, mock_exe_exists, unused_report):
@@ -710,7 +722,8 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         #    ret, _, _, _ = self._call(args)
         #    self.assertTrue("Too many flags setting" in ret)
 
-        args = ["install", "--nginx", "--cert-path", "/tmp/blah", "--key-path", "/tmp/blah",
+        args = ["install", "--nginx", "--cert-path",
+                test_util.temp_join('blah'), "--key-path", test_util.temp_join('blah'),
                 "--nginx-server-root", "/nonexistent/thing", "-d",
                 "example.com", "--debug"]
         if "nginx" in real_plugins:
@@ -733,6 +746,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
             self._call(["auth", "--standalone"])
             self.assertEqual(1, mock_certonly.call_count)
 
+    @test_util.broken_on_windows
     def test_rollback(self):
         _, _, _, client = self._call(['rollback'])
         self.assertEqual(1, client.rollback.call_count)
@@ -760,6 +774,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         self._call_no_clientmock(['delete'])
         self.assertEqual(1, mock_cert_manager.call_count)
 
+    @test_util.broken_on_windows
     def test_plugins(self):
         flags = ['--init', '--prepare', '--authenticators', '--installers']
         for args in itertools.chain(
@@ -931,8 +946,8 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
     @mock.patch('certbot.crypto_util.notAfter')
     @test_util.patch_get_utility()
     def test_certonly_new_request_success(self, mock_get_utility, mock_notAfter):
-        cert_path = '/etc/letsencrypt/live/foo.bar'
-        key_path = '/etc/letsencrypt/live/baz.qux'
+        cert_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/foo.bar'))
+        key_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/baz.qux'))
         date = '1970-01-01'
         mock_notAfter().date.return_value = date
 
@@ -962,7 +977,8 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
                              reuse_key=False):
         # pylint: disable=too-many-locals,too-many-arguments,too-many-branches
         cert_path = test_util.vector_path('cert_512.pem')
-        chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem'
+        chain_path = os.path.normpath(os.path.join(self.config.config_dir,
+                                                   'live/foo.bar/fullchain.pem'))
         mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path,
                                       cert_path=cert_path, fullchain_path=chain_path)
         mock_lineage.should_autorenew.return_value = due_for_renewal
@@ -1014,7 +1030,8 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
                     # The location of the previous live privkey.pem is passed
                     # to obtain_certificate
                     mock_client.obtain_certificate.assert_called_once_with(['isnot.org'],
-                        os.path.join(self.config.config_dir, "live/sample-renewal/privkey.pem"))
+                        os.path.normpath(os.path.join(
+                            self.config.config_dir, "live/sample-renewal/privkey.pem")))
                 else:
                     mock_client.obtain_certificate.assert_called_once_with(['isnot.org'], None)
             else:
@@ -1039,6 +1056,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         self.assertTrue('fullchain.pem' in cert_msg)
         self.assertTrue('donate' in get_utility().add_message.call_args[0][0])
 
+    @test_util.broken_on_windows
     @mock.patch('certbot.crypto_util.notAfter')
     def test_certonly_renewal_triggers(self, unused_notafter):
         # --dry-run should force renewal
@@ -1077,6 +1095,26 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         args = ["renew", "--reuse-key"]
         self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True)
 
+    @mock.patch('sys.stdin')
+    def test_noninteractive_renewal_delay(self, stdin):
+        stdin.isatty.return_value = False
+        test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf')
+        args = ["renew", "--dry-run", "-tvv"]
+        self._test_renewal_common(True, [], args=args, should_renew=True)
+        self.assertEqual(self.mock_sleep.call_count, 1)
+        # in main.py:
+        #     sleep_time = random.randint(1, 60*8)
+        sleep_call_arg = self.mock_sleep.call_args[0][0]
+        self.assertTrue(1 <= sleep_call_arg <= 60*8)
+
+    @mock.patch('sys.stdin')
+    def test_interactive_no_renewal_delay(self, stdin):
+        stdin.isatty.return_value = True
+        test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf')
+        args = ["renew", "--dry-run", "-tvv"]
+        self._test_renewal_common(True, [], args=args, should_renew=True)
+        self.assertEqual(self.mock_sleep.call_count, 0)
+
     @mock.patch('certbot.renewal.should_renew')
     def test_renew_skips_recent_certs(self, should_renew):
         should_renew.return_value = False
@@ -1087,6 +1125,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         self.assertTrue('No renewals were attempted.' in stdout.getvalue())
         self.assertTrue('The following certs are not due for renewal yet:' in stdout.getvalue())
 
+    @test_util.broken_on_windows
     def test_quiet_renew(self):
         test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf')
         args = ["renew", "--dry-run"]
@@ -1204,7 +1243,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         renewalparams = {'authenticator': 'webroot'}
         self._test_renew_common(
             renewalparams=renewalparams, assert_oc_called=True,
-            args=['renew', '--webroot-map', '{"example.com": "/tmp"}'])
+            args=['renew', '--webroot-map', json.dumps({'example.com': tempfile.gettempdir()})])
 
     def test_renew_reconstitute_error(self):
         # pylint: disable=protected-access
@@ -1234,7 +1273,9 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
     def test_no_renewal_with_hooks(self):
         _, _, stdout = self._test_renewal_common(
             due_for_renewal=False, extra_args=None, should_renew=False,
-            args=['renew', '--post-hook', 'echo hello world'])
+            args=['renew', '--post-hook',
+                  '{0} -c "from __future__ import print_function; print(\'hello world\');"'
+                  .format(sys.executable)])
         self.assertTrue('No hooks were run.' in stdout.getvalue())
 
     @test_util.patch_get_utility()
@@ -1254,13 +1295,19 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
         chain = 'chain'
         mock_client = mock.MagicMock()
         mock_client.obtain_certificate_from_csr.return_value = (certr, chain)
-        cert_path = '/etc/letsencrypt/live/example.com/cert_512.pem'
-        full_path = '/etc/letsencrypt/live/example.com/fullchain.pem'
+        cert_path = os.path.normpath(os.path.join(
+            self.config.config_dir,
+            'live/example.com/cert_512.pem'))
+        full_path = os.path.normpath(os.path.join(
+            self.config.config_dir,
+            'live/example.com/fullchain.pem'))
         mock_client.save_certificate.return_value = cert_path, None, full_path
         with mock.patch('certbot.main._init_le_client') as mock_init:
             mock_init.return_value = mock_client
             with test_util.patch_get_utility() as mock_get_utility:
-                chain_path = '/etc/letsencrypt/live/example.com/chain.pem'
+                chain_path = os.path.normpath(os.path.join(
+                    self.config.config_dir,
+                    'live/example.com/chain.pem'))
                 args = ('-a standalone certonly --csr {0} --cert-path {1} '
                         '--chain-path {2} --fullchain-path {3}').format(
                             CSR, cert_path, chain_path, full_path).split()
@@ -1334,6 +1381,7 @@ class MainTest(test_util.ConfigTestCase):  # pylint: disable=too-many-public-met
             self._call(['-c', test_util.vector_path('cli.ini')])
         self.assertTrue(mocked_run.called)
 
+    @test_util.broken_on_windows
     def test_register(self):
         with mock.patch('certbot.main.client') as mocked_client:
             acc = mock.MagicMock()

certbot/tests/reverter_test.py

@@ -50,6 +50,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
             x = f.read()
         self.assertTrue("No changes" in x)
 
+    @test_util.broken_on_windows
     def test_basic_add_to_temp_checkpoint(self):
         # These shouldn't conflict even though they are both named config.txt
         self.reverter.add_to_temp_checkpoint(self.sets[0], "save1")
@@ -91,6 +92,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
         self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint,
                           set([config3]), "invalid save")
 
+    @test_util.broken_on_windows
     def test_multiple_saves_and_temp_revert(self):
         self.reverter.add_to_temp_checkpoint(self.sets[0], "save1")
         update_file(self.config1, "updated-directive")
@@ -120,6 +122,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
         self.assertFalse(os.path.isfile(config3))
         self.assertFalse(os.path.isfile(config4))
 
+    @test_util.broken_on_windows
     def test_multiple_registration_same_file(self):
         self.reverter.register_file_creation(True, self.config1)
         self.reverter.register_file_creation(True, self.config1)
@@ -144,6 +147,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
             errors.ReverterError, self.reverter.register_file_creation,
             "filepath")
 
+    @test_util.broken_on_windows
     def test_register_undo_command(self):
         coms = [
             ["a2dismod", "ssl"],
@@ -166,6 +170,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
                 errors.ReverterError, self.reverter.register_undo_command,
                 True, ["command"])
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.util.run_script")
     def test_run_undo_commands(self, mock_run):
         mock_run.side_effect = ["", errors.SubprocessError]
@@ -229,6 +234,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
             self.assertRaises(
                 errors.ReverterError, self.reverter.revert_temporary_config)
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.reverter.logger.warning")
     def test_recover_checkpoint_missing_new_files(self, mock_warn):
         self.reverter.register_file_creation(
@@ -243,6 +249,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase):
         self.assertRaises(
             errors.ReverterError, self.reverter.revert_temporary_config)
 
+    @test_util.broken_on_windows
     def test_recovery_routine_temp_and_perm(self):
         # Register a new perm checkpoint file
         config3 = os.path.join(self.dir1, "config3.txt")
@@ -306,6 +313,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase):
         self.assertRaises(
             errors.ReverterError, self.reverter.rollback_checkpoints, "one")
 
+    @test_util.broken_on_windows
     def test_rollback_finalize_checkpoint_valid_inputs(self):
 
         config3 = self._setup_three_checkpoints()
@@ -348,7 +356,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase):
         self.assertRaises(
             errors.ReverterError, self.reverter.finalize_checkpoint, "Title")
 
-    @mock.patch("certbot.reverter.os.rename")
+    @mock.patch("certbot.reverter.compat.os_rename")
     def test_finalize_checkpoint_no_rename_directory(self, mock_rename):
 
         self.reverter.add_to_checkpoint(self.sets[0], "perm save")
@@ -357,6 +365,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase):
         self.assertRaises(
             errors.ReverterError, self.reverter.finalize_checkpoint, "Title")
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.reverter.logger")
     def test_rollback_too_many(self, mock_logger):
         # Test no exist warning...
@@ -369,6 +378,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase):
         self.reverter.rollback_checkpoints(4)
         self.assertEqual(mock_logger.warning.call_count, 1)
 
+    @test_util.broken_on_windows
     def test_multi_rollback(self):
         config3 = self._setup_three_checkpoints()
         self.reverter.rollback_checkpoints(3)

certbot/tests/storage_test.py

@@ -480,6 +480,7 @@ class RenewableCertTests(BaseRenewableCertTest):
         self.assertTrue(self.test_rc.should_autorenew())
         mock_ocsp.return_value = False
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.storage.relevant_values")
     def test_save_successor(self, mock_rv):
         # Mock relevant_values() to claim that all values are relevant here
@@ -625,6 +626,8 @@ class RenewableCertTests(BaseRenewableCertTest):
         self.assertTrue(result._consistent())
         self.assertTrue(os.path.exists(os.path.join(
             self.config.renewal_configs_dir, "the-lineage.com.conf")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.config.live_dir, "README")))
         self.assertTrue(os.path.exists(os.path.join(
             self.config.live_dir, "the-lineage.com", "README")))
         with open(result.fullchain, "rb") as f:

certbot/tests/util.py

@@ -9,6 +9,8 @@ import pkg_resources
 import shutil
 import tempfile
 import unittest
+import sys
+import warnings
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
@@ -36,8 +38,15 @@ def vector_path(*names):
 def load_vector(*names):
     """Load contents of a test vector."""
     # luckily, resource_string opens file in binary mode
-    return pkg_resources.resource_string(
+    data = pkg_resources.resource_string(
         __name__, os.path.join('testdata', *names))
+    # Try at most to convert CRLF to LF when data is text
+    try:
+        return data.decode().replace('\r\n', '\n').encode()
+    except ValueError:
+        # Failed to process the file with standard encoding.
+        # Most likely not a text file, return its bytes untouched.
+        return data
 
 
 def _guess_loader(filename, loader_pem, loader_der):
@@ -314,10 +323,22 @@ class TempDirTestCase(unittest.TestCase):
     """Base test class which sets up and tears down a temporary directory"""
 
     def setUp(self):
+        """Execute before test"""
         self.tempdir = tempfile.mkdtemp()
 
     def tearDown(self):
-        shutil.rmtree(self.tempdir)
+        """Execute after test"""
+        # On Windows we have various files which are not correctly closed at the time of tearDown.
+        # For know, we log them until a proper file close handling is written.
+        # Useful for development only, so no warning when we are on a CI process.
+        def onerror_handler(_, path, excinfo):
+            """On error handler"""
+            if not os.environ.get('APPVEYOR'): # pragma: no cover
+                message = ('Following error occurred when deleting the tempdir {0}'
+                           ' for path {1} during tearDown process: {2}'
+                           .format(self.tempdir, path, str(excinfo)))
+                warnings.warn(message)
+        shutil.rmtree(self.tempdir, onerror=onerror_handler)
 
 class ConfigTestCase(TempDirTestCase):
     """Test class which sets up a NamespaceConfig object.
@@ -378,3 +399,25 @@ def hold_lock(cv, lock_path):  # pragma: no cover
     cv.notify()
     cv.wait()
     my_lock.release()
+
+def skip_on_windows(reason):
+    """Decorator to skip permanently a test on Windows. A reason is required."""
+    def wrapper(function):
+        """Wrapped version"""
+        return unittest.skipIf(sys.platform == 'win32', reason)(function)
+    return wrapper
+
+def broken_on_windows(function):
+    """Decorator to skip temporarily a broken test on Windows."""
+    reason = 'Test is broken and ignored on windows but should be fixed.'
+    return unittest.skipIf(
+        sys.platform == 'win32'
+        and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true',
+        reason)(function)
+
+def temp_join(path):
+    """
+    Return the given path joined to the tempdir path for the current platform
+    Eg.: 'cert' => /tmp/cert (Linux) or 'C:\\Users\\currentuser\\AppData\\Temp\\cert' (Windows)
+    """
+    return  os.path.join(tempfile.gettempdir(), path)

certbot/tests/util_test.py

@@ -3,7 +3,6 @@ import argparse
 import errno
 import os
 import shutil
-import stat
 import unittest
 
 import mock
@@ -89,6 +88,7 @@ class LockDirUntilExit(test_util.TempDirTestCase):
         import certbot.util
         reload_module(certbot.util)
 
+    @test_util.broken_on_windows
     @mock.patch('certbot.util.logger')
     @mock.patch('certbot.util.atexit_register')
     def test_it(self, mock_register, mock_logger):
@@ -140,9 +140,9 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase):
         super(MakeOrVerifyDirTest, self).setUp()
 
         self.path = os.path.join(self.tempdir, "foo")
-        os.mkdir(self.path, 0o400)
+        os.mkdir(self.path, 0o600)
 
-        self.uid = os.getuid()
+        self.uid = compat.os_geteuid()
 
     def _call(self, directory, mode):
         from certbot.util import make_or_verify_dir
@@ -152,14 +152,15 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase):
         path = os.path.join(self.tempdir, "bar")
         self._call(path, 0o650)
         self.assertTrue(os.path.isdir(path))
-        self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o650)
+        self.assertTrue(compat.compare_file_modes(os.stat(path).st_mode, 0o650))
 
     def test_existing_correct_mode_does_not_fail(self):
-        self._call(self.path, 0o400)
-        self.assertEqual(stat.S_IMODE(os.stat(self.path).st_mode), 0o400)
+        self._call(self.path, 0o600)
+        self.assertTrue(compat.compare_file_modes(os.stat(self.path).st_mode, 0o600))
 
+    @test_util.skip_on_windows('Umask modes are mostly ignored on Windows.')
     def test_existing_wrong_mode_fails(self):
-        self.assertRaises(errors.Error, self._call, self.path, 0o600)
+        self.assertRaises(errors.Error, self._call, self.path, 0o400)
 
     def test_reraises_os_error(self):
         with mock.patch.object(os, "makedirs") as makedirs:
@@ -178,7 +179,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase):
     def setUp(self):
         super(CheckPermissionsTest, self).setUp()
 
-        self.uid = os.getuid()
+        self.uid = compat.os_geteuid()
 
     def _call(self, mode):
         from certbot.util import check_permissions
@@ -212,8 +213,8 @@ class UniqueFileTest(test_util.TempDirTestCase):
         self.assertEqual(open(name).read(), "bar")
 
     def test_right_mode(self):
-        self.assertEqual(0o700, os.stat(self._call(0o700)[1]).st_mode & 0o777)
-        self.assertEqual(0o100, os.stat(self._call(0o100)[1]).st_mode & 0o777)
+        self.assertTrue(compat.compare_file_modes(0o700, os.stat(self._call(0o700)[1]).st_mode))
+        self.assertTrue(compat.compare_file_modes(0o600, os.stat(self._call(0o600)[1]).st_mode))
 
     def test_default_exists(self):
         name1 = self._call()[1]  # create 0000_foo.txt
@@ -513,17 +514,16 @@ class OsInfoTest(unittest.TestCase):
 
     def test_systemd_os_release(self):
         from certbot.util import (get_os_info, get_systemd_os_info,
-                                     get_os_info_ua)
+                                  get_os_info_ua)
 
         with mock.patch('os.path.isfile', return_value=True):
             self.assertEqual(get_os_info(
                 test_util.vector_path("os-release"))[0], 'systemdos')
             self.assertEqual(get_os_info(
                 test_util.vector_path("os-release"))[1], '42')
-            self.assertEqual(get_systemd_os_info("/dev/null"), ("", ""))
+            self.assertEqual(get_systemd_os_info(os.devnull), ("", ""))
             self.assertEqual(get_os_info_ua(
-                test_util.vector_path("os-release")),
-                "SystemdOS")
+                test_util.vector_path("os-release")), "SystemdOS")
         with mock.patch('os.path.isfile', return_value=False):
             self.assertEqual(get_systemd_os_info(), ("", ""))
 

certbot/util.py

@@ -12,7 +12,6 @@ import platform
 import re
 import six
 import socket
-import stat
 import subprocess
 import sys
 
@@ -21,6 +20,7 @@ from collections import OrderedDict
 import configargparse
 
 from acme.magic_typing import Tuple, Union  # pylint: disable=unused-import, no-name-in-module
+from certbot import compat
 from certbot import constants
 from certbot import errors
 from certbot import lock
@@ -204,7 +204,7 @@ def check_permissions(filepath, mode, uid=0):
 
     """
     file_stat = os.stat(filepath)
-    return stat.S_IMODE(file_stat.st_mode) == mode and file_stat.st_uid == uid
+    return compat.compare_file_modes(file_stat.st_mode, mode) and file_stat.st_uid == uid
 
 
 def safe_open(path, mode="w", chmod=None, buffering=None):

docs/cli-help.txt

@@ -24,7 +24,7 @@ obtain, install, and renew certificates:
 
 manage certificates:
     certificates    Display information about certificates you have from Certbot
-    revoke          Revoke a certificate (supply --cert-path)
+    revoke          Revoke a certificate (supply --cert-path or --cert-name)
     delete          Delete a certificate
 
 manage your account with Let's Encrypt:
@@ -108,7 +108,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/0.27.1
+                        "". (default: CertbotACMEClient/0.28.0
                         (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX
                         Installer/YYY (SUBCOMMAND; flags: FLAGS)
                         Py/major.minor.patchlevel). The flags encoded in the
@@ -261,7 +261,8 @@ manage:
   delete                Clean up all files related to a certificate
   renew                 Renew all certificates (or one specified with --cert-
                         name)
-  revoke                Revoke a certificate specified with --cert-path
+  revoke                Revoke a certificate specified with --cert-path or
+                        --cert-name
   update_symlinks       Recreate symlinks in your /etc/letsencrypt/live/
                         directory
 
@@ -475,10 +476,9 @@ apache:
   Apache Web Server plugin - Beta
 
   --apache-enmod APACHE_ENMOD
-                        Path to the Apache 'a2enmod' binary (default: a2enmod)
+                        Path to the Apache 'a2enmod' binary (default: None)
   --apache-dismod APACHE_DISMOD
-                        Path to the Apache 'a2dismod' binary (default:
-                        a2dismod)
+                        Path to the Apache 'a2dismod' binary (default: None)
   --apache-le-vhost-ext APACHE_LE_VHOST_EXT
                         SSL vhost configuration extension (default: -le-
                         ssl.conf)
@@ -492,16 +492,16 @@ apache:
                         /var/log/apache2)
   --apache-challenge-location APACHE_CHALLENGE_LOCATION
                         Directory path for challenge configuration (default:
-                        /etc/apache2)
+                        /etc/apache2/other)
   --apache-handle-modules APACHE_HANDLE_MODULES
                         Let installer handle enabling required modules for you
-                        (Only Ubuntu/Debian currently) (default: True)
+                        (Only Ubuntu/Debian currently) (default: False)
   --apache-handle-sites APACHE_HANDLE_SITES
                         Let installer handle enabling sites for you (Only
-                        Ubuntu/Debian currently) (default: True)
+                        Ubuntu/Debian currently) (default: False)
   --apache-ctl APACHE_CTL
                         Full path to Apache control script (default:
-                        apache2ctl)
+                        apachectl)
 
 certbot-route53:auth:
   Obtain certificates using a DNS TXT record (if you are using AWS Route53
@@ -602,7 +602,7 @@ dns-linode:
   --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS
                         The number of seconds to wait for DNS to propagate
                         before asking the ACME server to verify the DNS
-                        record. (default: 960)
+                        record. (default: 1200)
   --dns-linode-credentials DNS_LINODE_CREDENTIALS
                         Linode credentials INI file. (default: None)
 

docs/contributing.rst

@@ -38,13 +38,13 @@ Certbot.
 
    cd certbot
    ./certbot-auto --debug --os-packages-only
-   tools/venv.sh
+   python tools/venv.py
 
-If you have Python3 available and want to use it, run the ``venv3.sh`` script.
+If you have Python3 available and want to use it, run the ``venv3.py`` script.
 
 .. code-block:: shell
 
-   tools/venv3.sh
+   python tools/venv3.py
 
 .. note:: You may need to repeat this when
   Certbot's dependencies change or when a new plugin is introduced.
@@ -353,7 +353,7 @@ Steps:
 
 1. Write your code!
 2. Make sure your environment is set up properly and that you're in your
-   virtualenv. You can do this by running ``./tools/venv.sh``.
+   virtualenv. You can do this by running ``pip tools/venv.py``.
    (this is a **very important** step)
 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors.
 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite

docs/install.rst

@@ -9,6 +9,8 @@ Get Certbot
 About Certbot
 =============
 
+*Certbot is meant to be run directly on a web server*, normally by a system administrator. In most cases, running Certbot on your personal computer is not a useful option. The instructions below relate to installing and running Certbot on a server.
+
 Certbot is packaged for many common operating systems and web servers. Check whether
 ``certbot`` (or ``letsencrypt``) is packaged for your web server's OS by visiting
 certbot.eff.org_, where you will also find the correct installation instructions for

docs/using.rst

@@ -988,9 +988,6 @@ Getting help
 If you're having problems, we recommend posting on the Let's Encrypt
 `Community Forum <https://community.letsencrypt.org>`_.
 
-You can also chat with us on IRC: `(#letsencrypt @
-freenode) <https://webchat.freenode.net?channels=%23letsencrypt>`_
-
 If you find a bug in the software, please do report it in our `issue
 tracker <https://github.com/certbot/certbot/issues>`_. Remember to
 give us as much information as possible:

letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.27.1"
+LE_AUTO_VERSION="0.28.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then
 else
   SetRootAuthMechanism
   if [ -n "$SUDO" ]; then
-    echo "Requesting to rerun $0 with root privileges..."
+    say "Requesting to rerun $0 with root privileges..."
     $SUDO "$0" --cb-auto-has-root "$@"
     exit 0
   fi
@@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.27.1 \
-    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
-    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
-acme==0.27.1 \
-    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
-    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
-certbot-apache==0.27.1 \
-    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
-    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
-certbot-nginx==0.27.1 \
-    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
-    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
+certbot==0.28.0 \
+    --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \
+    --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a
+acme==0.28.0 \
+    --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \
+    --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85
+certbot-apache==0.28.0 \
+    --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \
+    --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb
+certbot-nginx==0.28.0 \
+    --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \
+    --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X
-dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+
-nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn
-ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa
-I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk
-AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm
-8/oqLVSfoqjhU6Fn/11Af+iCnpICUw==
-=QRnC
+iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X
+dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj
+zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i
+7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq
+rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO
+zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg
+PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q==
+=1hD2
 -----END PGP SIGNATURE-----

letsencrypt-auto-source/letsencrypt-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.28.0.dev0"
+LE_AUTO_VERSION="0.29.0.dev0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -594,7 +594,7 @@ BootstrapArchCommon() {
   #
   # "python-virtualenv" is Python3, but "python2-virtualenv" provides
   # only "virtualenv2" binary, not "virtualenv" necessary in
-  # ./tools/_venv_common.sh
+  # ./tools/_venv_common.py
 
   deps="
     python2
@@ -912,6 +912,35 @@ OldVenvExists() {
     [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
 }
 
+# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2.
+# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated
+# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1
+# is outdated, and "UP_TO_DATE" if not.
+# This function relies only on installed python environment (2.x or 3.x) by certbot-auto.
+CompareVersions() {
+    "$1" - "$2" "$3" << "UNLIKELY_EOF"
+import sys
+from distutils.version import StrictVersion
+
+try:
+    current = StrictVersion(sys.argv[1])
+except ValueError:
+    sys.stdout.write('UNOFFICIAL')
+    sys.exit()
+
+try:
+    remote = StrictVersion(sys.argv[2])
+except ValueError:
+    sys.stdout.write('UP_TO_DATE')
+    sys.exit()
+
+if current < remote:
+    sys.stdout.write('OUTDATED')
+else:
+    sys.stdout.write('UP_TO_DATE')
+UNLIKELY_EOF
+}
+
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
 
@@ -1017,43 +1046,39 @@ pycparser==2.14 \
 asn1crypto==0.22.0 \
     --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \
     --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a
-cffi==1.10.0 \
-    --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \
-    --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \
-    --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \
-    --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \
-    --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \
-    --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \
-    --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \
-    --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \
-    --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \
-    --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \
-    --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \
-    --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \
-    --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \
-    --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \
-    --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \
-    --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \
-    --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \
-    --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \
-    --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \
-    --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \
-    --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \
-    --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \
-    --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \
-    --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \
-    --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \
-    --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \
-    --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \
-    --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \
-    --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \
-    --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \
-    --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \
-    --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \
-    --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \
-    --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \
-    --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \
-    --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
+cffi==1.11.5 \
+    --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \
+    --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \
+    --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \
+    --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \
+    --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \
+    --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \
+    --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \
+    --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \
+    --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \
+    --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \
+    --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \
+    --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \
+    --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \
+    --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \
+    --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \
+    --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \
+    --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \
+    --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \
+    --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \
+    --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \
+    --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \
+    --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \
+    --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \
+    --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \
+    --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \
+    --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \
+    --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \
+    --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \
+    --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \
+    --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \
+    --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \
+    --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4
 ConfigArgParse==0.12.0 \
     --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
     --no-binary ConfigArgParse
@@ -1197,31 +1222,29 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.27.1 \
-    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
-    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
-acme==0.27.1 \
-    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
-    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
-certbot-apache==0.27.1 \
-    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
-    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
-certbot-nginx==0.27.1 \
-    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
-    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
+certbot==0.28.0 \
+    --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \
+    --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a
+acme==0.28.0 \
+    --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \
+    --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85
+certbot-apache==0.28.0 \
+    --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \
+    --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb
+certbot-nginx==0.28.0 \
+    --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \
+    --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py"
 #!/usr/bin/env python
 """A small script that can act as a trust root for installing pip >=8
-
 Embed this in your project, and your VCS checkout is all you have to trust. In
 a post-peep era, this lets you claw your way to a hash-checking version of pip,
 with which you can install the rest of your dependencies safely. All it assumes
 is Python 2.6 or better and *some* version of pip already installed. If
 anything goes wrong, it will exit with a non-zero status code.
-
 """
 # This is here so embedded copies are MIT-compliant:
 # Copyright (c) 2016 Erik Rose
@@ -1348,10 +1371,8 @@ def hashed_download(url, temp, digest):
 
 def get_index_base():
     """Return the URL to the dir containing the "packages" folder.
-
     Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the
     end if it's there; that is likely to give us the right dir.
-
     """
     env_var = environ.get('PIP_INDEX_URL', '').rstrip('/')
     if env_var:
@@ -1641,7 +1662,12 @@ UNLIKELY_EOF
       error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
     elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
-    elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
+    fi
+
+    LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"`
+    if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then
+      say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION"
+    elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."
 
       # Now we drop into Python so we don't have to install even more