Merge remote-tracking branch 'upstream/master' into soham4abc/master

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -79,6 +79,9 @@ jobs:
           TOXENV: integration-dns-rfc2136
         docker-dev:
           TOXENV: docker_dev
+        le-modification:
+          IMAGE_NAME: ubuntu-18.04
+          TOXENV: modification
         macos-farmtest-apache2:
           # We run one of these test farm tests on macOS to help ensure the
           # tests continue to work on the platform.

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -56,11 +56,6 @@ jobs:
         apache-compat:
           IMAGE_NAME: ubuntu-18.04
           TOXENV: apache_compat
-        # le-modification can be moved to the extended test suite once
-        # https://github.com/certbot/certbot/issues/8742 is resolved.
-        le-modification:
-          IMAGE_NAME: ubuntu-18.04
-          TOXENV: modification
         apacheconftest:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.6

acme/acme/challenges.py

@@ -314,6 +314,15 @@ class HTTP01Response(KeyAuthorizationChallengeResponse):
         except requests.exceptions.RequestException as error:
             logger.error("Unable to reach %s: %s", uri, error)
             return False
+        # By default, http_response.text will try to guess the encoding to use
+        # when decoding the response to Python unicode strings. This guesswork
+        # is error prone. RFC 8555 specifies that HTTP-01 responses should be
+        # key authorizations with possible trailing whitespace. Since key
+        # authorizations must be composed entirely of the base64url alphabet
+        # plus ".", we tell requests that the response should be ASCII. See
+        # https://datatracker.ietf.org/doc/html/rfc8555#section-8.3 for more
+        # info.
+        http_response.encoding = "ascii"
         logger.debug("Received %s: %s. Headers: %s", http_response,
                      http_response.text, http_response.headers)
 

acme/acme/client.py

@@ -14,6 +14,7 @@ from typing import List
 from typing import Set
 from typing import Text
 from typing import Union
+import warnings
 
 import josepy as jose
 import OpenSSL
@@ -224,6 +225,9 @@ class ClientBase:
 class Client(ClientBase):
     """ACME client for a v1 API.
 
+    .. deprecated:: 1.18.0
+       Use :class:`ClientV2` instead.
+
     .. todo::
        Clean up raised error types hierarchy, document, and handle (wrap)
        instances of `.DeserializationError` raised in `from_json()`.
@@ -246,6 +250,8 @@ class Client(ClientBase):
             URI from which the resource will be downloaded.
 
         """
+        warnings.warn("acme.client.Client (ACMEv1) is deprecated, "
+                      "use acme.client.ClientV2 instead.", PendingDeprecationWarning)
         self.key = key
         if net is None:
             net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl)
@@ -805,6 +811,9 @@ class BackwardsCompatibleClientV2:
     """ACME client wrapper that tends towards V2-style calls, but
     supports V1 servers.
 
+    .. deprecated:: 1.18.0
+       Use :class:`ClientV2` instead.
+
     .. note:: While this class handles the majority of the differences
         between versions of the ACME protocol, if you need to support an
         ACME server based on version 3 or older of the IETF ACME draft
@@ -821,6 +830,8 @@ class BackwardsCompatibleClientV2:
     """
 
     def __init__(self, net, key, server):
+        warnings.warn("acme.client.BackwardsCompatibleClientV2 is deprecated, use "
+                      "acme.client.ClientV2 instead.", PendingDeprecationWarning)
         directory = messages.Directory.from_json(net.get(server).json())
         self.acme_version = self._acme_version_from_directory(directory)
         self.client: Union[Client, ClientV2]

acme/acme/messages.py

@@ -114,7 +114,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
         :rtype: unicode
 
         """
-        code = str(self.typ).split(':')[-1]
+        code = str(self.typ).rsplit(':', maxsplit=1)[-1]
         if code in ERROR_CODES:
             return code
         return None

acme/setup.py

@@ -14,7 +14,7 @@ install_requires = [
     'PyOpenSSL>=17.3.0',
     'pyrfc3339',
     'pytz',
-    'requests>=2.6.0',
+    'requests>=2.14.2',
     'requests-toolbelt>=0.3.0',
     'setuptools>=39.0.1',
 ]

certbot-apache/certbot_apache/_internal/configurator.py

@@ -15,7 +15,6 @@ from typing import Optional
 from typing import Set
 from typing import Union
 
-import zope.component
 import zope.interface
 
 from acme import challenges
@@ -884,7 +883,7 @@ class ApacheConfigurator(common.Installer):
                         all_names.add(name)
 
         if vhost_macro:
-            zope.component.getUtility(interfaces.IDisplay).notification(
+            display_util.notification(
                 "Apache mod_macro seems to be in use in file(s):\n{0}"
                 "\n\nUnfortunately mod_macro is not yet supported".format(
                     "\n  ".join(vhost_macro)), force_interactive=True)

certbot-apache/certbot_apache/_internal/display_ops.py

@@ -1,12 +1,9 @@
 """Contains UI methods for Apache operations."""
 import logging
 
-import zope.component
-
 from certbot import errors
-from certbot import interfaces
 from certbot.compat import os
-import certbot.display.util as display_util
+from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +23,7 @@ def select_vhost_multiple(vhosts):
     # Remove the extra newline from the last entry
     if tags_list:
         tags_list[-1] = tags_list[-1][:-1]
-    code, names = zope.component.getUtility(interfaces.IDisplay).checklist(
+    code, names = display_util.checklist(
         "Which VirtualHosts would you like to install the wildcard certificate for?",
         tags=tags_list, force_interactive=True)
     if code == display_util.OK:
@@ -34,6 +31,7 @@ def select_vhost_multiple(vhosts):
         return return_vhosts
     return []
 
+
 def _reversemap_vhosts(names, vhosts):
     """Helper function for select_vhost_multiple for mapping string
     representations back to actual vhost objects"""
@@ -45,6 +43,7 @@ def _reversemap_vhosts(names, vhosts):
                 return_vhosts.append(vhost)
     return return_vhosts
 
+
 def select_vhost(domain, vhosts):
     """Select an appropriate Apache Vhost.
 
@@ -62,6 +61,7 @@ def select_vhost(domain, vhosts):
         return vhosts[tag]
     return None
 
+
 def _vhost_menu(domain, vhosts):
     """Select an appropriate Apache Vhost.
 
@@ -107,7 +107,7 @@ def _vhost_menu(domain, vhosts):
         )
 
     try:
-        code, tag = zope.component.getUtility(interfaces.IDisplay).menu(
+        code, tag = display_util.menu(
             "We were unable to find a vhost with a ServerName "
             "or Address of {0}.{1}Which virtual host would you "
             "like to choose?".format(domain, os.linesep),

certbot-apache/certbot_apache/_internal/override_centos.py

@@ -177,8 +177,8 @@ class CentOSParser(parser.ApacheParser):
     def parse_sysconfig_var(self):
         """ Parses Apache CLI options from CentOS configuration file """
         defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS")
-        for k in defines:
-            self.variables[k] = defines[k]
+        for k, v in defines.items():
+            self.variables[k] = v
 
     def not_modssl_ifmodule(self, path):
         """Checks if the provided Augeas path has argument !mod_ssl"""

certbot-apache/certbot_apache/_internal/override_fedora.py

@@ -87,5 +87,5 @@ class FedoraParser(parser.ApacheParser):
     def _parse_sysconfig_var(self):
         """ Parses Apache CLI options from Fedora configuration file """
         defines = apache_util.parse_define_file(self.sysconfig_filep, "OPTIONS")
-        for k in defines:
-            self.variables[k] = defines[k]
+        for k, v in defines.items():
+            self.variables[k] = v

certbot-apache/certbot_apache/_internal/override_gentoo.py

@@ -53,8 +53,8 @@ class GentooParser(parser.ApacheParser):
         """ Parses Apache CLI options from Gentoo configuration file """
         defines = apache_util.parse_define_file(self.apacheconfig_filep,
                                                 "APACHE2_OPTS")
-        for k in defines:
-            self.variables[k] = defines[k]
+        for k, v in defines.items():
+            self.variables[k] = v
 
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""

certbot-apache/certbot_apache/_internal/parser.py

@@ -440,7 +440,11 @@ class ApacheParser:
         :type args: list or str
         """
         first_dir = aug_conf_path + "/directive[1]"
-        self.aug.insert(first_dir, "directive", True)
+        if self.aug.get(first_dir):
+            self.aug.insert(first_dir, "directive", True)
+        else:
+            self.aug.set(first_dir, "directive")
+
         self.aug.set(first_dir, dirname)
         if isinstance(args, list):
             for i, value in enumerate(args, 1):

certbot-apache/tests/configurator_test.py

@@ -136,7 +136,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in
                         self.config.options.server_root)
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_get_all_names(self, mock_getutility):
         mock_utility = mock_getutility()
         mock_utility.notification = mock.MagicMock(return_value=True)
@@ -145,7 +145,7 @@ class MultipleVhostsTest(util.ApacheTest):
              "nonsym.link", "vhost.in.rootconf", "www.certbot.demo",
              "duplicate.example.com"})
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     @mock.patch("certbot_apache._internal.configurator.socket.gethostbyaddr")
     def test_get_all_names_addrs(self, mock_gethost, mock_getutility):
         mock_gethost.side_effect = [("google.com", "", ""), socket.error]

certbot-apache/tests/debian_test.py

@@ -9,6 +9,7 @@ except ImportError: # pragma: no cover
 
 from certbot import errors
 from certbot.compat import os
+from certbot.tests import util as certbot_util
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import obj
 import util
@@ -68,17 +69,18 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         self.config.parser.modules["ssl_module"] = None
         self.config.parser.modules["mod_ssl.c"] = None
         self.assertFalse(ssl_vhost.enabled)
-        self.config.deploy_cert(
-            "encryption-example.demo", "example/cert.pem", "example/key.pem",
-            "example/cert_chain.pem", "example/fullchain.pem")
-        self.assertTrue(ssl_vhost.enabled)
-        # Make sure that we don't error out if symlink already exists
-        ssl_vhost.enabled = False
-        self.assertFalse(ssl_vhost.enabled)
-        self.config.deploy_cert(
-            "encryption-example.demo", "example/cert.pem", "example/key.pem",
-            "example/cert_chain.pem", "example/fullchain.pem")
-        self.assertTrue(ssl_vhost.enabled)
+        with certbot_util.patch_display_util():
+            self.config.deploy_cert(
+                "encryption-example.demo", "example/cert.pem", "example/key.pem",
+                "example/cert_chain.pem", "example/fullchain.pem")
+            self.assertTrue(ssl_vhost.enabled)
+            # Make sure that we don't error out if symlink already exists
+            ssl_vhost.enabled = False
+            self.assertFalse(ssl_vhost.enabled)
+            self.config.deploy_cert(
+                "encryption-example.demo", "example/cert.pem", "example/key.pem",
+                "example/cert_chain.pem", "example/fullchain.pem")
+            self.assertTrue(ssl_vhost.enabled)
 
     def test_enable_site_failure(self):
         self.config.parser.root = "/tmp/nonexistent"
@@ -101,9 +103,10 @@ class MultipleVhostsTestDebian(util.ApacheTest):
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
-        self.config.deploy_cert(
-            "random.demo", "example/cert.pem", "example/key.pem",
-            "example/cert_chain.pem", "example/fullchain.pem")
+        with certbot_util.patch_display_util():
+            self.config.deploy_cert(
+                "random.demo", "example/cert.pem", "example/key.pem",
+                "example/cert_chain.pem", "example/fullchain.pem")
         self.config.save()
 
         # Verify ssl_module was enabled.

certbot-apache/tests/display_ops_test.py

@@ -3,8 +3,8 @@ import unittest
 
 try:
     import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+except ImportError:  # pragma: no cover
+    from unittest import mock  # type: ignore
 
 from certbot import errors
 from certbot.display import util as display_util
@@ -25,7 +25,7 @@ class SelectVhostMultiTest(unittest.TestCase):
     def test_select_no_input(self):
         self.assertFalse(select_vhost_multiple([]))
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_select_correct(self, mock_util):
         mock_util().checklist.return_value = (
             display_util.OK, [self.vhosts[3].display_repr(),
@@ -37,12 +37,13 @@ class SelectVhostMultiTest(unittest.TestCase):
         self.assertTrue(self.vhosts[3] in vhs)
         self.assertFalse(self.vhosts[1] in vhs)
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_select_cancel(self, mock_util):
         mock_util().checklist.return_value = (display_util.CANCEL, "whatever")
         vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]])
         self.assertFalse(vhs)
 
+
 class SelectVhostTest(unittest.TestCase):
     """Tests for certbot_apache._internal.display_ops.select_vhost."""
 
@@ -56,12 +57,12 @@ class SelectVhostTest(unittest.TestCase):
         from certbot_apache._internal.display_ops import select_vhost
         return select_vhost("example.com", vhosts)
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_successful_choice(self, mock_util):
         mock_util().menu.return_value = (display_util.OK, 3)
         self.assertEqual(self.vhosts[3], self._call(self.vhosts))
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_noninteractive(self, mock_util):
         mock_util().menu.side_effect = errors.MissingCommandlineFlag("no vhost default")
         try:
@@ -69,7 +70,7 @@ class SelectVhostTest(unittest.TestCase):
         except errors.MissingCommandlineFlag as e:
             self.assertTrue("vhost ambiguity" in str(e))
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_more_info_cancel(self, mock_util):
         mock_util().menu.side_effect = [
             (display_util.CANCEL, -1),
@@ -81,16 +82,15 @@ class SelectVhostTest(unittest.TestCase):
         self.assertEqual(self._call([]), None)
 
     @mock.patch("certbot_apache._internal.display_ops.display_util")
-    @certbot_util.patch_get_utility()
     @mock.patch("certbot_apache._internal.display_ops.logger")
-    def test_small_display(self, mock_logger, mock_util, mock_display_util):
+    def test_small_display(self, mock_logger, mock_display_util):
         mock_display_util.WIDTH = 20
-        mock_util().menu.return_value = (display_util.OK, 0)
+        mock_display_util.menu.return_value = (display_util.OK, 0)
         self._call(self.vhosts)
 
         self.assertEqual(mock_logger.debug.call_count, 1)
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_multiple_names(self, mock_util):
         mock_util().menu.return_value = (display_util.OK, 5)
 

certbot-apache/tests/parser_test.py

@@ -105,6 +105,11 @@ class BasicParserTest(util.ParserTest):
         for i, match in enumerate(matches):
             self.assertEqual(self.parser.aug.get(match), str(i + 1))
 
+        for name in ("empty.conf", "no-directives.conf"):
+            conf = "/files" + os.path.join(self.parser.root, "sites-available", name)
+            self.parser.add_dir_beginning(conf, "AddDirectiveBeginning", "testBegin")
+            self.assertTrue(self.parser.find_dir("AddDirectiveBeginning", "testBegin", conf))
+
     def test_empty_arg(self):
         self.assertEqual(None,
                           self.parser.get_arg("/files/whatever/nonexistent"))

certbot-apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/no-directives.conf

@@ -0,0 +1,5 @@
+<VirtualHost *:80>
+  <Location />
+        Require all denied
+  </Location>
+</VirtualHost>

certbot-apache/tests/util.py

@@ -5,16 +5,16 @@ import unittest
 
 import augeas
 import josepy as jose
+
 try:
     import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
-import zope.component
+except ImportError:  # pragma: no cover
+    from unittest import mock  # type: ignore
 
 from certbot.compat import os
-from certbot.display import util as display_util
 from certbot.plugins import common
 from certbot.tests import util as test_util
+from certbot.display import util as display_util
 from certbot_apache._internal import configurator
 from certbot_apache._internal import entrypoint
 from certbot_apache._internal import obj
@@ -69,9 +69,6 @@ class ParserTest(ApacheTest):
               vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"):
         super().setUp(test_dir, config_root, vhost_root)
 
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout,
-                                                               False))
-
         from certbot_apache._internal.parser import ApacheParser
         self.aug = augeas.Augeas(
             flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -346,7 +346,8 @@ def test_renew_empty_hook_scripts(context):
     for hook_dir in misc.list_renewal_hooks_dirs(context.config_dir):
         shutil.rmtree(hook_dir)
         os.makedirs(join(hook_dir, 'dir'))
-        open(join(hook_dir, 'file'), 'w').close()
+        with open(join(hook_dir, 'file'), 'w'):
+            pass
     context.certbot(['renew'])
 
     assert_cert_count_for_lineage(context.config_dir, certname, 2)
@@ -368,7 +369,8 @@ def test_renew_hook_override(context):
     assert_hook_execution(context.hook_probe, 'deploy')
 
     # Now we override all previous hooks during next renew.
-    open(context.hook_probe, 'w').close()
+    with open(context.hook_probe, 'w'):
+        pass
     context.certbot([
         'renew', '--cert-name', certname,
         '--pre-hook', misc.echo('pre_override', context.hook_probe),
@@ -387,7 +389,8 @@ def test_renew_hook_override(context):
         assert_hook_execution(context.hook_probe, 'deploy')
 
     # Expect that this renew will reuse new hooks registered in the previous renew.
-    open(context.hook_probe, 'w').close()
+    with open(context.hook_probe, 'w'):
+        pass
     context.certbot(['renew', '--cert-name', certname])
 
     assert_hook_execution(context.hook_probe, 'pre_override')

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -52,7 +52,7 @@ class ACMEServer:
         self._proxy = http_proxy
         self._workspace = tempfile.mkdtemp()
         self._processes: List[subprocess.Popen] = []
-        self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
+        self._stdout = sys.stdout if stdout else open(os.devnull, 'w') # pylint: disable=consider-using-with
         self._dns_server = dns_server
         self._http_01_port = http_01_port
         if http_01_port != DEFAULT_HTTP_01_PORT:

certbot-ci/certbot_integration_tests/utils/dns_server.py

@@ -45,6 +45,7 @@ class DNSServer:
 
         # Unfortunately the BIND9 image forces everything to stderr with -g and we can't
         # modify the verbosity.
+        # pylint: disable=consider-using-with
         self._output = sys.stderr if show_output else open(os.devnull, "w")
 
     def start(self):

certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py

@@ -79,11 +79,12 @@ def _get_names(config):
 def _get_server_names(root, filename):
     """Returns all names in a config file path"""
     all_names = set()
-    for line in open(os.path.join(root, filename)):
-        if line.strip().startswith("server_name"):
-            names = line.partition("server_name")[2].rpartition(";")[0]
-            for n in names.split():
-                # Filter out wildcards in both all_names and test_names
-                if not n.startswith("*."):
-                    all_names.add(n)
+    with open(os.path.join(root, filename)) as f:
+        for line in f:
+            if line.strip().startswith("server_name"):
+                names = line.partition("server_name")[2].rpartition(";")[0]
+                for n in names.split():
+                    # Filter out wildcards in both all_names and test_names
+                    if not n.startswith("*."):
+                        all_names.add(n)
     return all_names

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -10,7 +10,6 @@ import tempfile
 import time
 from typing import List
 from typing import Tuple
-import zope.component
 
 import OpenSSL
 from urllib3.util import connection
@@ -21,6 +20,7 @@ from acme import messages
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.display import util as display_util
+from certbot._internal.display import obj as display_obj
 from certbot.tests import acme_util
 from certbot_compatibility_test import errors
 from certbot_compatibility_test import util
@@ -332,7 +332,7 @@ def setup_logging(args):
 def setup_display():
     """"Prepares IDisplay for the Certbot plugins """
     displayer = display_util.NoninteractiveDisplay(sys.stdout)
-    zope.component.provideUtility(displayer)
+    display_obj.set_display(displayer)
 
 
 def main():

certbot-dns-cloudflare/tests/dns_cloudflare_test.py

@@ -41,7 +41,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         # _get_cloudflare_client | pylint: disable=protected-access
         self.auth._get_cloudflare_client = mock.MagicMock(return_value=self.mock_client)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self, unused_mock_get_utility):
         self.auth.perform([self.achall])
 
@@ -56,7 +56,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)]
         self.assertEqual(expected, self.mock_client.mock_calls)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_api_token(self, unused_mock_get_utility):
         dns_test_common.write({"cloudflare_api_token": API_TOKEN},
                               self.config.cloudflare_credentials)

certbot-dns-cloudxns/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-digitalocean/tests/dns_digitalocean_test.py

@@ -37,7 +37,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         # _get_digitalocean_client | pylint: disable=protected-access
         self.auth._get_digitalocean_client = mock.MagicMock(return_value=self.mock_client)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self, unused_mock_get_utility):
         self.auth.perform([self.achall])
 

certbot-dns-dnsimple/setup.py

@@ -7,6 +7,9 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
+    # This version of lexicon is required to address the problem described in
+    # https://github.com/AnalogJ/lexicon/issues/387.
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]
@@ -25,18 +28,6 @@ elif 'bdist_wheel' in sys.argv[1:]:
 if os.environ.get('SNAP_BUILD'):
     install_requires.append('packaging')
 
-# This package normally depends on dns-lexicon>=3.2.1 to address the
-# problem described in https://github.com/AnalogJ/lexicon/issues/387,
-# however, the fix there has been backported to older versions of
-# lexicon found in various Linux distros. This conditional helps us test
-# that we've maintained compatibility with these versions of lexicon
-# which allows us to potentially upgrade our packages in these distros
-# as necessary.
-if os.environ.get('CERTBOT_OLDEST') == '1':
-    install_requires.append('dns-lexicon>=3.1.0')  # Changed parameter name
-else:
-    install_requires.append('dns-lexicon>=3.2.1')
-
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

certbot-dns-dnsmadeeasy/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-gehirn/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-google/tests/dns_google_test.py

@@ -43,7 +43,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         # _get_google_client | pylint: disable=protected-access
         self.auth._get_google_client = mock.MagicMock(return_value=self.mock_client)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self, unused_mock_get_utility):
         self.auth.perform([self.achall])
 
@@ -59,7 +59,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         self.assertEqual(expected, self.mock_client.mock_calls)
 
     @mock.patch('httplib2.Http.request', side_effect=ServerNotFoundError)
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_without_auth(self, unused_mock_get_utility, unused_mock):
         self.config.google_credentials = None
         self.assertRaises(PluginError, self.auth.perform, [self.achall])

certbot-dns-linode/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-luadns/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-nsone/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-ovh/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-dns-rfc2136/tests/dns_rfc2136_test.py

@@ -42,7 +42,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
         # _get_rfc2136_client | pylint: disable=protected-access
         self.auth._get_rfc2136_client = mock.MagicMock(return_value=self.mock_client)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self, unused_mock_get_utility):
         self.auth.perform([self.achall])
 
@@ -66,7 +66,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
                           self.auth.perform,
                           [self.achall])
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_valid_algorithm_passes(self, unused_mock_get_utility):
         config = VALID_CONFIG.copy()
         config["rfc2136_algorithm"] = "HMAC-sha512"

certbot-dns-sakuracloud/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup
 version = '1.18.0.dev0'
 
 install_requires = [
-    'dns-lexicon>=3.1.0',  # Changed `rtype` parameter name
+    'dns-lexicon>=3.2.1',
     'setuptools>=39.0.1',
     'zope.interface',
 ]

certbot-nginx/certbot_nginx/_internal/display_ops.py

@@ -1,10 +1,7 @@
 """Contains UI methods for Nginx operations."""
 import logging
 
-import zope.component
-
-from certbot import interfaces
-import certbot.display.util as display_util
+from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +19,7 @@ def select_vhost_multiple(vhosts):
     # Remove the extra newline from the last entry
     if tags_list:
         tags_list[-1] = tags_list[-1][:-1]
-    code, names = zope.component.getUtility(interfaces.IDisplay).checklist(
+    code, names = display_util.checklist(
         "Which server blocks would you like to modify?",
         tags=tags_list, force_interactive=True)
     if code == display_util.OK:
@@ -30,6 +27,7 @@ def select_vhost_multiple(vhosts):
         return return_vhosts
     return []
 
+
 def _reversemap_vhosts(names, vhosts):
     """Helper function for select_vhost_multiple for mapping string
     representations back to actual vhost objects"""

certbot-nginx/certbot_nginx/_internal/parser.py

@@ -96,8 +96,8 @@ class NginxParser:
         servers = self._get_raw_servers()
 
         addr_to_ssl: Dict[Tuple[str, str], bool] = {}
-        for filename in servers:
-            for server, _ in servers[filename]:
+        for server_list in servers.values():
+            for server, _ in server_list:
                 # Parse the server block to save addr info
                 parsed_server = _parse_server_raw(server)
                 for addr in parsed_server['addrs']:
@@ -112,8 +112,7 @@ class NginxParser:
         """Get a map of unparsed all server blocks
         """
         servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {}
-        for filename in self.parsed:
-            tree = self.parsed[filename]
+        for filename, tree in self.parsed.items():
             servers[filename] = []
             srv = servers[filename]  # workaround undefined loop var in lambdas
 
@@ -141,8 +140,8 @@ class NginxParser:
         servers = self._get_raw_servers()
 
         vhosts = []
-        for filename in servers:
-            for server, path in servers[filename]:
+        for filename, server_list in servers.items():
+            for server, path in server_list:
                 # Parse the server block into a VirtualHost object
 
                 parsed_server = _parse_server_raw(server)
@@ -240,8 +239,7 @@ class NginxParser:
 
         """
         # Best-effort atomicity is enforced above us by reverter.py
-        for filename in self.parsed:
-            tree = self.parsed[filename]
+        for filename, tree in self.parsed.items():
             if ext:
                 filename = filename + os.path.extsep + ext
             if not isinstance(tree, UnspacedList):

certbot-nginx/tests/display_ops_test.py

@@ -19,7 +19,7 @@ class SelectVhostMultiTest(util.NginxTest):
     def test_select_no_input(self):
         self.assertFalse(select_vhost_multiple([]))
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_select_correct(self, mock_util):
         mock_util().checklist.return_value = (
             display_util.OK, [self.vhosts[3].display_repr(),
@@ -31,7 +31,7 @@ class SelectVhostMultiTest(util.NginxTest):
         self.assertTrue(self.vhosts[3] in vhs)
         self.assertFalse(self.vhosts[1] in vhs)
 
-    @certbot_util.patch_get_utility()
+    @certbot_util.patch_display_util()
     def test_select_cancel(self, mock_util):
         mock_util().checklist.return_value = (display_util.CANCEL, "whatever")
         vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]])

certbot-nginx/tests/test_util.py

@@ -6,8 +6,8 @@ import tempfile
 import josepy as jose
 try:
     import mock
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
+except ImportError:  # pragma: no cover
+    from unittest import mock  # type: ignore
 import pkg_resources
 
 from certbot import util

certbot/CHANGELOG.md

@@ -6,7 +6,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 
-*
+* New functions that Certbot plugins can use to interact with the user have
+  been added to `certbot.display.util`. We plan to deprecate using `IDisplay`
+  with `zope` in favor of these new functions in the future.
 
 ### Changed
 
@@ -17,6 +19,9 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
   the response which was error prone.
 * `acme`: the `.client.Client` and `.client.BackwardsCompatibleClientV2` classes
   are now deprecated in favor of `.client.ClientV2`.
+* The `certbot.tests.patch_get_utility*` functions have been deprecated.
+  Plugins should now patch `certbot.display.util` themselves in their tests or
+  use `certbot.tests.util.patch_display_util` as a temporary workaround.
 
 ### Fixed
 

certbot/certbot/_internal/account.py

@@ -311,8 +311,8 @@ class AccountFileStorage(interfaces.AccountStorage):
 
         # does an appropriate directory link to me? if so, make sure that's gone
         reused_servers = {}
-        for k in constants.LE_REUSE_SERVERS:
-            reused_servers[constants.LE_REUSE_SERVERS[k]] = k
+        for k, v in constants.LE_REUSE_SERVERS.items():
+            reused_servers[v] = k
 
         # is there a next one up?
         possible_next_link = True

certbot/certbot/_internal/auth_handler.py

@@ -6,14 +6,11 @@ from typing import Dict
 from typing import List
 from typing import Tuple
 
-import zope.component
-
 from acme import challenges
 from acme import errors as acme_errors
 from acme import messages
 from certbot import achallenges
 from certbot import errors
-from certbot import interfaces
 from certbot._internal import error_handler
 from certbot.display import util as display_util
 from certbot.plugins import common as plugin_common
@@ -74,9 +71,9 @@ class AuthHandler:
 
                 # If debug is on, wait for user input before starting the verification process.
                 if config.debug_challenges:
-                    notify = zope.component.getUtility(interfaces.IDisplay).notification
-                    notify('Challenges loaded. Press continue to submit to CA. '
-                           'Pass "-v" for more info about challenges.', pause=True)
+                    display_util.notification(
+                        'Challenges loaded. Press continue to submit to CA. '
+                        'Pass "-v" for more info about challenges.', pause=True)
             except errors.AuthorizationError as error:
                 logger.critical('Failure in setting up challenges.')
                 logger.info('Attempting to clean up outstanding challenges...')

certbot/certbot/_internal/cert_manager.py

@@ -6,11 +6,9 @@ import traceback
 from typing import List
 
 import pytz
-import zope.component
 
 from certbot import crypto_util
 from certbot import errors
-from certbot import interfaces
 from certbot import ocsp
 from certbot import util
 from certbot._internal import storage
@@ -23,6 +21,7 @@ logger = logging.getLogger(__name__)
 # Commands
 ###################
 
+
 def update_live_symlinks(config):
     """Update the certificate file family symlinks to use archive_dir.
 
@@ -38,6 +37,7 @@ def update_live_symlinks(config):
     for renewal_file in storage.renewal_conf_files(config):
         storage.RenewableCert(renewal_file, config, update_symlinks=True)
 
+
 def rename_lineage(config):
     """Rename the specified lineage to the new name.
 
@@ -45,15 +45,13 @@ def rename_lineage(config):
     :type config: :class:`certbot._internal.configuration.NamespaceConfig`
 
     """
-    disp = zope.component.getUtility(interfaces.IDisplay)
-
     certname = get_certnames(config, "rename")[0]
 
     new_certname = config.new_certname
     if not new_certname:
-        code, new_certname = disp.input(
+        code, new_certname = display_util.input_text(
             "Enter the new name for certificate {0}".format(certname),
-            flag="--updated-cert-name", force_interactive=True)
+            force_interactive=True)
         if code != display_util.OK or not new_certname:
             raise errors.Error("User ended interaction.")
 
@@ -62,8 +60,8 @@ def rename_lineage(config):
         raise errors.ConfigurationError("No existing certificate with name "
             "{0} found.".format(certname))
     storage.rename_renewal_config(certname, new_certname, config)
-    disp.notification("Successfully renamed {0} to {1}."
-        .format(certname, new_certname), pause=False)
+    display_util.notification("Successfully renamed {0} to {1}."
+                                 .format(certname, new_certname), pause=False)
 
 
 def certificates(config):
@@ -92,12 +90,11 @@ def certificates(config):
 def delete(config):
     """Delete Certbot files associated with a certificate lineage."""
     certnames = get_certnames(config, "delete", allow_multiple=True)
-    disp = zope.component.getUtility(interfaces.IDisplay)
     msg = ["The following certificate(s) are selected for deletion:\n"]
     for certname in certnames:
         msg.append("  * " + certname)
     msg.append("\nAre you sure you want to delete the above certificate(s)?")
-    if not disp.yesno("\n".join(msg), default=True):
+    if not display_util.yesno("\n".join(msg), default=True):
         logger.info("Deletion of certificate(s) canceled.")
         return
     for certname in certnames:
@@ -316,7 +313,6 @@ def get_certnames(config, verb, allow_multiple=False, custom_prompt=None):
     if certname:
         certnames = [certname]
     else:
-        disp = zope.component.getUtility(interfaces.IDisplay)
         filenames = storage.renewal_conf_files(config)
         choices = [storage.lineagename_for_filename(name) for name in filenames]
         if not choices:
@@ -326,7 +322,7 @@ def get_certnames(config, verb, allow_multiple=False, custom_prompt=None):
                 prompt = "Which certificate(s) would you like to {0}?".format(verb)
             else:
                 prompt = custom_prompt
-            code, certnames = disp.checklist(
+            code, certnames = display_util.checklist(
                 prompt, choices, cli_flag="--cert-name", force_interactive=True)
             if code != display_util.OK:
                 raise errors.Error("User ended interaction.")
@@ -336,7 +332,7 @@ def get_certnames(config, verb, allow_multiple=False, custom_prompt=None):
             else:
                 prompt = custom_prompt
 
-            code, index = disp.menu(
+            code, index = display_util.menu(
                 prompt, choices, cli_flag="--cert-name", force_interactive=True)
 
             if code != display_util.OK or index not in range(0, len(choices)):
@@ -382,8 +378,7 @@ def _describe_certs(config, parsed_certs, parse_failures):
                "were invalid:")
             notify(_report_lines(parse_failures))
 
-    disp = zope.component.getUtility(interfaces.IDisplay)
-    disp.notification("\n".join(out), pause=False, wrap=False)
+    display_util.notification("\n".join(out), pause=False, wrap=False)
 
 
 def _search_lineages(cli_config, func, initial_rv, *args):

certbot/certbot/_internal/cli/__init__.py

@@ -40,9 +40,9 @@ from certbot._internal.cli.plugins_parsing import _plugins_parsing
 from certbot._internal.cli.subparsers import _create_subparsers
 from certbot._internal.cli.verb_help import VERB_HELP
 from certbot._internal.cli.verb_help import VERB_HELP_MAP
+from certbot.plugins import enhancements
 from certbot._internal.plugins import disco as plugins_disco
 import certbot._internal.plugins.selection as plugin_selection
-import certbot.plugins.enhancements as enhancements
 
 logger = logging.getLogger(__name__)
 

certbot/certbot/_internal/cli/helpful.py

@@ -9,13 +9,9 @@ from typing import Any
 from typing import Dict
 
 import configargparse
-import zope.component
-import zope.interface
-from zope.interface import interfaces as zope_interfaces
 
 from certbot import crypto_util
 from certbot import errors
-from certbot import interfaces
 from certbot import util
 from certbot._internal import constants
 from certbot._internal import hooks
@@ -32,8 +28,8 @@ from certbot._internal.cli.cli_utils import flag_default
 from certbot._internal.cli.cli_utils import HelpfulArgumentGroup
 from certbot._internal.cli.verb_help import VERB_HELP
 from certbot._internal.cli.verb_help import VERB_HELP_MAP
+from certbot._internal.display import obj as display_obj
 from certbot.compat import os
-from certbot.display import util as display_util
 
 
 class HelpfulArgumentParser:
@@ -66,13 +62,7 @@ class HelpfulArgumentParser:
         }
 
         # Get notification function for printing
-        try:
-            self.notify = zope.component.getUtility(
-                interfaces.IDisplay).notification
-        except zope_interfaces.ComponentLookupError:
-            self.notify = display_util.NoninteractiveDisplay(
-                sys.stdout).notification
-
+        self.notify = display_obj.NoninteractiveDisplay(sys.stdout).notification
 
         # List of topics for which additional help can be provided
         HELP_TOPICS = ["all", "security", "paths", "automation", "testing"]

certbot/certbot/_internal/client.py

@@ -3,6 +3,7 @@ import datetime
 import logging
 import platform
 from typing import List, Optional, Union
+import warnings
 
 from cryptography.hazmat.backends import default_backend
 # See https://github.com/pyca/cryptography/issues/4275
@@ -32,13 +33,23 @@ from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
 
-
 def acme_from_config_key(config, key, regr=None):
     "Wrangle ACME client construction"
     # TODO: Allow for other alg types besides RS256
     net = acme_client.ClientNetwork(key, account=regr, verify_ssl=(not config.no_verify_ssl),
                                     user_agent=determine_user_agent(config))
-    return acme_client.BackwardsCompatibleClientV2(net, key, config.server)
+
+    with warnings.catch_warnings():
+        # TODO: full removal of ACMEv1 support: https://github.com/certbot/certbot/issues/6844
+        warnings.simplefilter("ignore", PendingDeprecationWarning)
+
+        client = acme_client.BackwardsCompatibleClientV2(net, key, config.server)
+        if client.acme_version == 1:
+            logger.warning(
+                "Certbot is configured to use an ACMEv1 server (%s). ACMEv1 support is deprecated"
+                " and will soon be removed. See https://community.letsencrypt.org/t/143839 for "
+                "more information.", config.server)
+        return client
 
 
 def determine_user_agent(config):

certbot/certbot/_internal/display/obj.py

@@ -0,0 +1,579 @@
+"""This modules define the actual display implementations used in Certbot"""
+import logging
+import sys
+import textwrap
+from typing import Any
+from typing import Optional
+
+import zope.component
+import zope.interface
+
+from certbot import errors
+from certbot import interfaces
+from certbot._internal import constants
+from certbot._internal.display import completer
+from certbot.compat import os
+from certbot.display import util
+
+logger = logging.getLogger(__name__)
+
+
+# This class holds the global state of the display service. Using this class
+# eliminates potential gotchas that exist if self.display was just a global
+# variable. In particular, in functions `_DISPLAY = <value>` would create a
+# local variable unless the programmer remembered to use the `global` keyword.
+# Adding a level of indirection causes the lookup of the global _DisplayService
+# object to happen first avoiding this potential bug.
+class _DisplayService:
+    def __init__(self):
+        self.display: Optional[interfaces.IDisplay] = None
+
+
+_SERVICE = _DisplayService()
+
+
+@zope.interface.implementer(interfaces.IDisplay)
+class FileDisplay:
+    """File-based display."""
+    # see https://github.com/certbot/certbot/issues/3915
+
+    def __init__(self, outfile, force_interactive):
+        super().__init__()
+        self.outfile = outfile
+        self.force_interactive = force_interactive
+        self.skipped_interaction = False
+
+    def notification(self, message, pause=True,
+                     wrap=True, force_interactive=False,
+                     decorate=True):
+        """Displays a notification and waits for user acceptance.
+
+        :param str message: Message to display
+        :param bool pause: Whether or not the program should pause for the
+            user's confirmation
+        :param bool wrap: Whether or not the application should wrap text
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+        :param bool decorate: Whether to surround the message with a
+            decorated frame
+
+        """
+        if wrap:
+            message = _wrap_lines(message)
+
+        logger.debug("Notifying user: %s", message)
+
+        self.outfile.write(
+            (("{line}{frame}{line}" if decorate else "") +
+             "{msg}{line}" +
+             ("{frame}{line}" if decorate else ""))
+                .format(line=os.linesep, frame=util.SIDE_FRAME, msg=message)
+        )
+        self.outfile.flush()
+
+        if pause:
+            if self._can_interact(force_interactive):
+                util.input_with_timeout("Press Enter to Continue")
+            else:
+                logger.debug("Not pausing for user confirmation")
+
+    def menu(self, message, choices, ok_label=None, cancel_label=None,  # pylint: disable=unused-argument
+             help_label=None, default=None,  # pylint: disable=unused-argument
+             cli_flag=None, force_interactive=False, **unused_kwargs):
+        """Display a menu.
+
+        .. todo:: This doesn't enable the help label/button (I wasn't sold on
+           any interface I came up with for this). It would be a nice feature
+
+        :param str message: title of menu
+        :param choices: Menu lines, len must be > 0
+        :type choices: list of tuples (tag, item) or
+            list of descriptions (tags will be enumerated)
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+
+        :returns: tuple of (`code`, `index`) where
+            `code` - str display exit code
+            `index` - int index of the user's selection
+
+        :rtype: tuple
+
+        """
+        if self._return_default(message, default, cli_flag, force_interactive):
+            return util.OK, default
+
+        self._print_menu(message, choices)
+
+        code, selection = self._get_valid_int_ans(len(choices))
+
+        return code, selection - 1
+
+    def input(self, message, default=None,
+              cli_flag=None, force_interactive=False, **unused_kwargs):
+        """Accept input from the user.
+
+        :param str message: message to display to the user
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+
+        :returns: tuple of (`code`, `input`) where
+            `code` - str display exit code
+            `input` - str of the user's input
+        :rtype: tuple
+
+        """
+        if self._return_default(message, default, cli_flag, force_interactive):
+            return util.OK, default
+
+        # Trailing space must be added outside of _wrap_lines to be preserved
+        message = _wrap_lines("%s (Enter 'c' to cancel):" % message) + " "
+        ans = util.input_with_timeout(message)
+
+        if ans in ("c", "C"):
+            return util.CANCEL, "-1"
+        return util.OK, ans
+
+    def yesno(self, message, yes_label="Yes", no_label="No", default=None,
+              cli_flag=None, force_interactive=False, **unused_kwargs):
+        """Query the user with a yes/no question.
+
+        Yes and No label must begin with different letters, and must contain at
+        least one letter each.
+
+        :param str message: question for the user
+        :param str yes_label: Label of the "Yes" parameter
+        :param str no_label: Label of the "No" parameter
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+
+        :returns: True for "Yes", False for "No"
+        :rtype: bool
+
+        """
+        if self._return_default(message, default, cli_flag, force_interactive):
+            return default
+
+        message = _wrap_lines(message)
+
+        self.outfile.write("{0}{frame}{msg}{0}{frame}".format(
+            os.linesep, frame=util.SIDE_FRAME + os.linesep, msg=message))
+        self.outfile.flush()
+
+        while True:
+            ans = util.input_with_timeout("{yes}/{no}: ".format(
+                yes=_parens_around_char(yes_label),
+                no=_parens_around_char(no_label)))
+
+            # Couldn't get pylint indentation right with elif
+            # elif doesn't matter in this situation
+            if (ans.startswith(yes_label[0].lower()) or
+                ans.startswith(yes_label[0].upper())):
+                return True
+            if (ans.startswith(no_label[0].lower()) or
+                ans.startswith(no_label[0].upper())):
+                return False
+
+    def checklist(self, message, tags, default=None,
+                  cli_flag=None, force_interactive=False, **unused_kwargs):
+        """Display a checklist.
+
+        :param str message: Message to display to user
+        :param list tags: `str` tags to select, len(tags) > 0
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+
+        :returns: tuple of (`code`, `tags`) where
+            `code` - str display exit code
+            `tags` - list of selected tags
+        :rtype: tuple
+
+        """
+        if self._return_default(message, default, cli_flag, force_interactive):
+            return util.OK, default
+
+        while True:
+            self._print_menu(message, tags)
+
+            code, ans = self.input("Select the appropriate numbers separated "
+                                   "by commas and/or spaces, or leave input "
+                                   "blank to select all options shown",
+                                   force_interactive=True)
+
+            if code == util.OK:
+                if not ans.strip():
+                    ans = " ".join(str(x) for x in range(1, len(tags)+1))
+                indices = util.separate_list_input(ans)
+                selected_tags = self._scrub_checklist_input(indices, tags)
+                if selected_tags:
+                    return code, selected_tags
+                self.outfile.write(
+                    "** Error - Invalid selection **%s" % os.linesep)
+                self.outfile.flush()
+            else:
+                return code, []
+
+    def _return_default(self, prompt, default, cli_flag, force_interactive):
+        """Should we return the default instead of prompting the user?
+
+        :param str prompt: prompt for the user
+        :param default: default answer to prompt
+        :param str cli_flag: command line option for setting an answer
+            to this question
+        :param bool force_interactive: if interactivity is forced by the
+            IDisplay call
+
+        :returns: True if we should return the default without prompting
+        :rtype: bool
+
+        """
+        # assert_valid_call(prompt, default, cli_flag, force_interactive)
+        if self._can_interact(force_interactive):
+            return False
+        if default is None:
+            msg = "Unable to get an answer for the question:\n{0}".format(prompt)
+            if cli_flag:
+                msg += (
+                    "\nYou can provide an answer on the "
+                    "command line with the {0} flag.".format(cli_flag))
+            raise errors.Error(msg)
+        logger.debug(
+            "Falling back to default %s for the prompt:\n%s",
+            default, prompt)
+        return True
+
+    def _can_interact(self, force_interactive):
+        """Can we safely interact with the user?
+
+        :param bool force_interactive: if interactivity is forced by the
+            IDisplay call
+
+        :returns: True if the display can interact with the user
+        :rtype: bool
+
+        """
+        if (self.force_interactive or force_interactive or
+            sys.stdin.isatty() and self.outfile.isatty()):
+            return True
+        if not self.skipped_interaction:
+            logger.warning(
+                "Skipped user interaction because Certbot doesn't appear to "
+                "be running in a terminal. You should probably include "
+                "--non-interactive or %s on the command line.",
+                constants.FORCE_INTERACTIVE_FLAG)
+            self.skipped_interaction = True
+        return False
+
+    def directory_select(self, message, default=None, cli_flag=None,
+                         force_interactive=False, **unused_kwargs):
+        """Display a directory selection screen.
+
+        :param str message: prompt to give the user
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+        :param bool force_interactive: True if it's safe to prompt the user
+            because it won't cause any workflow regressions
+
+        :returns: tuple of the form (`code`, `string`) where
+            `code` - display exit code
+            `string` - input entered by the user
+
+        """
+        with completer.Completer():
+            return self.input(message, default, cli_flag, force_interactive)
+
+    def _scrub_checklist_input(self, indices, tags):
+        """Validate input and transform indices to appropriate tags.
+
+        :param list indices: input
+        :param list tags: Original tags of the checklist
+
+        :returns: valid tags the user selected
+        :rtype: :class:`list` of :class:`str`
+
+        """
+        # They should all be of type int
+        try:
+            indices = [int(index) for index in indices]
+        except ValueError:
+            return []
+
+        # Remove duplicates
+        indices = list(set(indices))
+
+        # Check all input is within range
+        for index in indices:
+            if index < 1 or index > len(tags):
+                return []
+        # Transform indices to appropriate tags
+        return [tags[index - 1] for index in indices]
+
+    def _print_menu(self, message, choices):
+        """Print a menu on the screen.
+
+        :param str message: title of menu
+        :param choices: Menu lines
+        :type choices: list of tuples (tag, item) or
+            list of descriptions (tags will be enumerated)
+
+        """
+        # Can take either tuples or single items in choices list
+        if choices and isinstance(choices[0], tuple):
+            choices = ["%s - %s" % (c[0], c[1]) for c in choices]
+
+        # Write out the message to the user
+        self.outfile.write(
+            "{new}{msg}{new}".format(new=os.linesep, msg=message))
+        self.outfile.write(util.SIDE_FRAME + os.linesep)
+
+        # Write out the menu choices
+        for i, desc in enumerate(choices, 1):
+            msg = "{num}: {desc}".format(num=i, desc=desc)
+            self.outfile.write(_wrap_lines(msg))
+
+            # Keep this outside of the textwrap
+            self.outfile.write(os.linesep)
+
+        self.outfile.write(util.SIDE_FRAME + os.linesep)
+        self.outfile.flush()
+
+    def _get_valid_int_ans(self, max_):
+        """Get a numerical selection.
+
+        :param int max: The maximum entry (len of choices), must be positive
+
+        :returns: tuple of the form (`code`, `selection`) where
+            `code` - str display exit code ('ok' or cancel')
+            `selection` - int user's selection
+        :rtype: tuple
+
+        """
+        selection = -1
+        if max_ > 1:
+            input_msg = ("Select the appropriate number "
+                         "[1-{max_}] then [enter] (press 'c' to "
+                         "cancel): ".format(max_=max_))
+        else:
+            input_msg = ("Press 1 [enter] to confirm the selection "
+                         "(press 'c' to cancel): ")
+        while selection < 1:
+            ans = util.input_with_timeout(input_msg)
+            if ans.startswith("c") or ans.startswith("C"):
+                return util.CANCEL, -1
+            try:
+                selection = int(ans)
+                if selection < 1 or selection > max_:
+                    selection = -1
+                    raise ValueError
+
+            except ValueError:
+                self.outfile.write(
+                    "{0}** Invalid input **{0}".format(os.linesep))
+                self.outfile.flush()
+
+        return util.OK, selection
+
+
+@zope.interface.implementer(interfaces.IDisplay)
+class NoninteractiveDisplay:
+    """An iDisplay implementation that never asks for interactive user input"""
+
+    def __init__(self, outfile, *unused_args, **unused_kwargs):
+        super().__init__()
+        self.outfile = outfile
+
+    def _interaction_fail(self, message, cli_flag, extra=""):
+        "Error out in case of an attempt to interact in noninteractive mode"
+        msg = "Missing command line flag or config entry for this setting:\n"
+        msg += message
+        if extra:
+            msg += "\n" + extra
+        if cli_flag:
+            msg += "\n\n(You can set this with the {0} flag)".format(cli_flag)
+        raise errors.MissingCommandlineFlag(msg)
+
+    def notification(self, message, pause=False, wrap=True, decorate=True, **unused_kwargs):  # pylint: disable=unused-argument
+        """Displays a notification without waiting for user acceptance.
+
+        :param str message: Message to display to stdout
+        :param bool pause: The NoninteractiveDisplay waits for no keyboard
+        :param bool wrap: Whether or not the application should wrap text
+        :param bool decorate: Whether to apply a decorated frame to the message
+
+        """
+        if wrap:
+            message = _wrap_lines(message)
+
+        logger.debug("Notifying user: %s", message)
+
+        self.outfile.write(
+            (("{line}{frame}{line}" if decorate else "") +
+             "{msg}{line}" +
+             ("{frame}{line}" if decorate else ""))
+                .format(line=os.linesep, frame=util.SIDE_FRAME, msg=message)
+        )
+        self.outfile.flush()
+
+    def menu(self, message, choices, ok_label=None, cancel_label=None,
+             help_label=None, default=None, cli_flag=None, **unused_kwargs):
+        # pylint: disable=unused-argument
+        """Avoid displaying a menu.
+
+        :param str message: title of menu
+        :param choices: Menu lines, len must be > 0
+        :type choices: list of tuples (tag, item) or
+            list of descriptions (tags will be enumerated)
+        :param int default: the default choice
+        :param dict kwargs: absorbs various irrelevant labelling arguments
+
+        :returns: tuple of (`code`, `index`) where
+            `code` - str display exit code
+            `index` - int index of the user's selection
+        :rtype: tuple
+        :raises errors.MissingCommandlineFlag: if there was no default
+
+        """
+        if default is None:
+            self._interaction_fail(message, cli_flag, "Choices: " + repr(choices))
+
+        return util.OK, default
+
+    def input(self, message, default=None, cli_flag=None, **unused_kwargs):
+        """Accept input from the user.
+
+        :param str message: message to display to the user
+
+        :returns: tuple of (`code`, `input`) where
+            `code` - str display exit code
+            `input` - str of the user's input
+        :rtype: tuple
+        :raises errors.MissingCommandlineFlag: if there was no default
+
+        """
+        if default is None:
+            self._interaction_fail(message, cli_flag)
+        return util.OK, default
+
+    def yesno(self, message, yes_label=None, no_label=None,  # pylint: disable=unused-argument
+              default=None, cli_flag=None, **unused_kwargs):
+        """Decide Yes or No, without asking anybody
+
+        :param str message: question for the user
+        :param dict kwargs: absorbs yes_label, no_label
+
+        :raises errors.MissingCommandlineFlag: if there was no default
+        :returns: True for "Yes", False for "No"
+        :rtype: bool
+
+        """
+        if default is None:
+            self._interaction_fail(message, cli_flag)
+        return default
+
+    def checklist(self, message, tags, default=None,
+                  cli_flag=None, **unused_kwargs):
+        """Display a checklist.
+
+        :param str message: Message to display to user
+        :param list tags: `str` tags to select, len(tags) > 0
+        :param dict kwargs: absorbs default_status arg
+
+        :returns: tuple of (`code`, `tags`) where
+            `code` - str display exit code
+            `tags` - list of selected tags
+        :rtype: tuple
+
+        """
+        if default is None:
+            self._interaction_fail(message, cli_flag, "? ".join(tags))
+        return util.OK, default
+
+    def directory_select(self, message, default=None,
+                         cli_flag=None, **unused_kwargs):
+        """Simulate prompting the user for a directory.
+
+        This function returns default if it is not ``None``, otherwise,
+        an exception is raised explaining the problem. If cli_flag is
+        not ``None``, the error message will include the flag that can
+        be used to set this value with the CLI.
+
+        :param str message: prompt to give the user
+        :param default: default value to return (if one exists)
+        :param str cli_flag: option used to set this value with the CLI
+
+        :returns: tuple of the form (`code`, `string`) where
+            `code` - int display exit code
+            `string` - input entered by the user
+
+        """
+        return self.input(message, default, cli_flag)
+
+
+# The two following functions use "Any" for their parameter/output types. Normally interfaces from
+# certbot.interfaces would be used, but MyPy will not understand their semantic. These interfaces
+# will be removed soon and replaced by ABC classes that will be used also here for type checking.
+# TODO: replace Any by actual ABC classes once available
+
+def get_display() -> Any:
+    """Get the display utility.
+
+    :return: the display utility
+    :rtype: IDisplay
+    :raise: ValueError if the display utility is not configured yet.
+
+    """
+    if not _SERVICE.display:
+        raise ValueError("This function was called too early in Certbot's execution "
+                         "as the display utility hasn't been configured yet.")
+    return _SERVICE.display
+
+
+def set_display(display: Any) -> None:
+    """Set the display service.
+
+    :param IDisplay display: the display service
+
+    """
+    # This call is done only for retro-compatibility purposes.
+    # TODO: Remove this call once zope dependencies are removed from Certbot.
+    zope.component.provideUtility(display)
+
+    _SERVICE.display = display
+
+
+def _wrap_lines(msg):
+    """Format lines nicely to 80 chars.
+
+    :param str msg: Original message
+
+    :returns: Formatted message respecting newlines in message
+    :rtype: str
+
+    """
+    lines = msg.splitlines()
+    fixed_l = []
+
+    for line in lines:
+        fixed_l.append(textwrap.fill(
+            line,
+            80,
+            break_long_words=False,
+            break_on_hyphens=False))
+
+    return '\n'.join(fixed_l)
+
+
+def _parens_around_char(label):
+    """Place parens around first character of label.
+
+    :param str label: Must contain at least one character
+
+    """
+    return "({first}){rest}".format(first=label[0], rest=label[1:])

certbot/certbot/_internal/eff.py

@@ -3,9 +3,7 @@ import logging
 from typing import Optional
 
 import requests
-import zope.component
 
-from certbot import interfaces
 from certbot._internal import constants
 from certbot._internal.account import Account
 from certbot._internal.account import AccountFileStorage
@@ -75,8 +73,7 @@ def _want_subscription() -> bool:
         "founding partner of the Let's Encrypt project and the non-profit organization "
         "that develops Certbot? We'd like to send you email about our work encrypting "
         "the web, EFF news, campaigns, and ways to support digital freedom. ")
-    display = zope.component.getUtility(interfaces.IDisplay)
-    return display.yesno(prompt, default=False)
+    return display_util.yesno(prompt, default=False)
 
 
 def subscribe(email: str) -> None:

certbot/certbot/_internal/error_handler.py

@@ -139,8 +139,8 @@ class ErrorHandler:
 
     def _reset_signal_handlers(self):
         """Resets signal handlers for signals in _SIGNALS."""
-        for signum in self.prev_handlers:
-            signal.signal(signum, self.prev_handlers[signum])
+        for signum, handler in self.prev_handlers.items():
+            signal.signal(signum, handler)
         self.prev_handlers.clear()
 
     def _signal_handler(self, signum, unused_frame):

certbot/certbot/_internal/main.py

@@ -1,10 +1,10 @@
 """Certbot main entry point."""
 # pylint: disable=too-many-lines
 
+from contextlib import contextmanager
 import functools
 import logging.handlers
 import sys
-from contextlib import contextmanager
 from typing import Generator
 from typing import IO
 from typing import Iterable
@@ -37,6 +37,7 @@ from certbot._internal import reporter
 from certbot._internal import snap_config
 from certbot._internal import storage
 from certbot._internal import updater
+from certbot._internal.display import obj as display_obj
 from certbot._internal.plugins import disco as plugins_disco
 from certbot._internal.plugins import selection as plug_sel
 from certbot.compat import filesystem
@@ -67,9 +68,8 @@ def _suggest_donation_if_appropriate(config):
     if config.staging:
         # --dry-run implies --staging
         return
-    disp = zope.component.getUtility(interfaces.IDisplay)
     util.atexit_register(
-        disp.notification,
+        display_util.notification,
         "If you like Certbot, please consider supporting our work by:\n"
         " * Donating to ISRG / Let's Encrypt:   https://letsencrypt.org/donate\n"
         " * Donating to EFF:                    https://eff.org/donate-le",
@@ -191,10 +191,8 @@ def _handle_subset_cert_request(config: configuration.NamespaceConfig,
              existing,
              ", ".join(domains),
              br=os.linesep)
-    if config.expand or config.renew_by_default or zope.component.getUtility(
-            interfaces.IDisplay).yesno(question, "Expand", "Cancel",
-                                       cli_flag="--expand",
-                                       force_interactive=True):
+    if config.expand or config.renew_by_default or display_util.yesno(
+        question, "Expand", "Cancel", cli_flag="--expand", force_interactive=True):
         return "renew", cert
     display_util.notify(
         "To obtain a new certificate that contains these names without "
@@ -247,9 +245,8 @@ def _handle_identical_cert_request(config: configuration.NamespaceConfig,
     choices = [keep_opt,
                "Renew & replace the certificate (may be subject to CA rate limits)"]
 
-    display = zope.component.getUtility(interfaces.IDisplay)
-    response = display.menu(question, choices,
-                            default=0, force_interactive=True)
+    response = display_util.menu(question, choices,
+                                    default=0, force_interactive=True)
     if response[0] == display_util.CANCEL:
         # TODO: Add notification related to command-line options for
         #       skipping the menu for this case.
@@ -423,8 +420,7 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains):
                _format_list("+", added),
                _format_list("-", removed),
                br=os.linesep))
-    obj = zope.component.getUtility(interfaces.IDisplay)
-    if not obj.yesno(msg, "Update certificate", "Cancel", default=True):
+    if not display_util.yesno(msg, "Update certificate", "Cancel", default=True):
         raise errors.ConfigurationError("Specified mismatched certificate name and domains.")
 
 
@@ -652,8 +648,7 @@ def _determine_account(config):
         msg = ("Please read the Terms of Service at {0}. You "
                "must agree in order to register with the ACME "
                "server. Do you agree?".format(terms_of_service))
-        obj = zope.component.getUtility(interfaces.IDisplay)
-        result = obj.yesno(msg, cli_flag="--agree-tos", force_interactive=True)
+        result = display_util.yesno(msg, cli_flag="--agree-tos", force_interactive=True)
         if not result:
             raise errors.Error(
                 "Registration cannot proceed without accepting "
@@ -702,14 +697,12 @@ def _delete_if_appropriate(config):
     :raises errors.Error: If anything goes wrong, including bad user input, if an overlapping
         archive dir is found for the specified lineage, etc ...
     """
-    display = zope.component.getUtility(interfaces.IDisplay)
-
     attempt_deletion = config.delete_after_revoke
     if attempt_deletion is None:
         msg = ("Would you like to delete the certificate(s) you just revoked, "
                "along with all earlier and later versions of the certificate?")
-        attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No",
-                force_interactive=True, default=True)
+        attempt_deletion = display_util.yesno(msg, yes_label="Yes (recommended)", no_label="No",
+                                              force_interactive=True, default=True)
 
     if not attempt_deletion:
         return
@@ -788,11 +781,10 @@ def unregister(config, unused_plugins):
 
     if not accounts:
         return "Could not find existing account to deactivate."
-    yesno = zope.component.getUtility(interfaces.IDisplay).yesno
     prompt = ("Are you sure you would like to irrevocably deactivate "
               "your account?")
-    wants_deactivate = yesno(prompt, yes_label='Deactivate', no_label='Abort',
-                             default=True)
+    wants_deactivate = display_util.yesno(prompt, yes_label='Deactivate', no_label='Abort',
+                                          default=True)
 
     if not wants_deactivate:
         return "Deactivation aborted."
@@ -1034,8 +1026,7 @@ def plugins_cmd(config, plugins):
     filtered = plugins.visible().ifaces(ifaces)
     logger.debug("Filtered plugins: %r", filtered)
 
-    notify = functools.partial(zope.component.getUtility(
-        interfaces.IDisplay).notification, pause=False)
+    notify = functools.partial(display_util.notification, pause=False)
     if not config.init and not config.prepare:
         notify(str(filtered))
         return
@@ -1428,8 +1419,8 @@ def certonly(config, plugins):
     should_get_cert, lineage = _find_cert(config, domains, certname)
 
     if not should_get_cert:
-        notify = zope.component.getUtility(interfaces.IDisplay).notification
-        notify("Certificate not yet due for renewal; no action taken.", pause=False)
+        display_util.notification("Certificate not yet due for renewal; no action taken.",
+                                     pause=False)
         return
 
     lineage = _get_and_save_cert(le_client, config, domains, certname, lineage)
@@ -1563,12 +1554,13 @@ def main(cli_args=None):
         if config.func != plugins_cmd:  # pylint: disable=comparison-with-callable
             raise
 
-    # Reporter
+    # These calls are done only for retro-compatibility purposes.
+    # TODO: Remove these calls once zope dependencies are removed from Certbot.
     report = reporter.Reporter(config)
     zope.component.provideUtility(report)
     util.atexit_register(report.print_messages)
 
     with make_displayer(config) as displayer:
-        zope.component.provideUtility(displayer)
+        display_obj.set_display(displayer)
 
         return config.func(config, plugins)

certbot/certbot/_internal/plugins/disco.py

@@ -1,8 +1,8 @@
 """Utilities for plugins discovery and selection."""
+from collections.abc import Mapping
 import itertools
 import logging
 import sys
-from collections.abc import Mapping
 from typing import Dict
 from typing import Optional
 from typing import Union
@@ -10,6 +10,7 @@ from typing import Union
 import pkg_resources
 import zope.interface
 import zope.interface.verify
+
 from certbot import errors
 from certbot import interfaces
 from certbot._internal import constants

certbot/certbot/_internal/plugins/manual.py

@@ -2,7 +2,6 @@
 import logging
 from typing import Dict
 
-import zope.component
 import zope.interface
 
 from acme import challenges
@@ -11,15 +10,17 @@ from certbot import errors
 from certbot import interfaces
 from certbot import reverter
 from certbot import util
-from certbot._internal.cli import cli_constants
 from certbot._internal import hooks
+from certbot._internal.cli import cli_constants
 from certbot.compat import misc
 from certbot.compat import os
 from certbot.display import ops as display_ops
+from certbot.display import util as display_util
 from certbot.plugins import common
 
 logger = logging.getLogger(__name__)
 
+
 @zope.interface.implementer(interfaces.IAuthenticator)
 @zope.interface.provider(interfaces.IPluginFactory)
 class Authenticator(common.Plugin):
@@ -225,8 +226,7 @@ permitted by DNS standards.)
         elif self.subsequent_any_challenge:
             # 2nd or later challenge of another type
             msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS
-        display = zope.component.getUtility(interfaces.IDisplay)
-        display.notification(msg, wrap=False, force_interactive=True)
+        display_util.notification(msg, wrap=False, force_interactive=True)
         self.subsequent_any_challenge = True
 
     def cleanup(self, achalls):  # pylint: disable=missing-function-docstring

certbot/certbot/_internal/plugins/selection.py

@@ -1,9 +1,8 @@
 """Decide which plugins to use for authentication & installation"""
 
 import logging
-
-from typing import Optional, Tuple
-import zope.component
+from typing import Optional
+from typing import Tuple
 
 from certbot import errors
 from certbot import interfaces
@@ -12,7 +11,7 @@ from certbot.compat import os
 from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
-z_util = zope.component.getUtility
+
 
 def pick_configurator(
         config, default, plugins,
@@ -138,13 +137,12 @@ def choose_plugin(prepared, question):
             for plugin_ep in prepared]
 
     while True:
-        disp = z_util(interfaces.IDisplay)
-        code, index = disp.menu(question, opts, force_interactive=True)
+        code, index = display_util.menu(question, opts, force_interactive=True)
 
         if code == display_util.OK:
             plugin_ep = prepared[index]
             if plugin_ep.misconfigured:
-                z_util(interfaces.IDisplay).notification(
+                display_util.notification(
                     "The selected plugin encountered an error while parsing "
                     "your server configuration and cannot be used. The error "
                     "was:\n\n{0}".format(plugin_ep.prepare()), pause=False)

certbot/certbot/_internal/plugins/standalone.py

@@ -18,6 +18,7 @@ from acme import standalone as acme_standalone
 from certbot import achallenges
 from certbot import errors
 from certbot import interfaces
+from certbot.display import util as display_util
 from certbot.plugins import common
 
 logger = logging.getLogger(__name__)
@@ -28,6 +29,7 @@ if TYPE_CHECKING:
         Set[achallenges.KeyAuthorizationAnnotatedChallenge]
     ]
 
+
 class ServerManager:
     """Standalone servers manager.
 
@@ -202,14 +204,12 @@ def _handle_perform_error(error):
             "aren't running this program as "
             "root).".format(error.port))
     if error.socket_error.errno == errno.EADDRINUSE:
-        display = zope.component.getUtility(interfaces.IDisplay)
         msg = (
             "Could not bind TCP port {0} because it is already in "
             "use by another process on this system (such as a web "
             "server). Please stop the program in question and "
             "then try again.".format(error.port))
-        should_retry = display.yesno(msg, "Retry",
-                                     "Cancel", default=False)
+        should_retry = display_util.yesno(msg, "Retry", "Cancel", default=False)
         if not should_retry:
             raise errors.PluginError(msg)
     else:

certbot/certbot/_internal/plugins/webroot.py

@@ -8,7 +8,6 @@ from typing import Dict
 from typing import List
 from typing import Set
 
-import zope.component
 import zope.interface
 
 from acme import challenges
@@ -126,11 +125,10 @@ to serve all files under specified web root ({0})."""
         return webroot
 
     def _prompt_with_webroot_list(self, domain, known_webroots):
-        display = zope.component.getUtility(interfaces.IDisplay)
         path_flag = "--" + self.option_name("path")
 
         while True:
-            code, index = display.menu(
+            code, index = display_util.menu(
                 "Select the webroot for {0}:".format(domain),
                 ["Enter a new webroot"] + known_webroots,
                 cli_flag=path_flag, force_interactive=True)

certbot/certbot/_internal/renewal.py

@@ -429,8 +429,7 @@ def handle_renewal_request(config):
     apply_random_sleep = not sys.stdin.isatty() and config.random_sleep_on_renew
 
     for renewal_file in conf_files:
-        disp = zope.component.getUtility(interfaces.IDisplay)
-        disp.notification("Processing " + renewal_file, pause=False)
+        display_util.notification("Processing " + renewal_file, pause=False)
         lineage_config = copy.deepcopy(config)
         lineagename = storage.lineagename_for_filename(renewal_file)
 

certbot/certbot/_internal/storage.py

@@ -144,7 +144,8 @@ def write_renewal_config(o_filename, n_filename, archive_dir, target, relevant_d
     logger.debug("Writing new config %s.", n_filename)
 
     # Ensure that the file exists
-    open(n_filename, 'a').close()
+    with open(n_filename, 'a'):
+        pass
 
     # Copy permissions from the old version of the file, if it exists.
     if os.path.exists(o_filename):

certbot/certbot/_internal/updater.py

@@ -4,7 +4,7 @@ import logging
 from certbot import errors
 from certbot import interfaces
 from certbot._internal.plugins import selection as plug_sel
-import certbot.plugins.enhancements as enhancements
+from certbot.plugins import enhancements
 
 logger = logging.getLogger(__name__)
 

certbot/certbot/crypto_util.py

@@ -7,9 +7,10 @@
 import hashlib
 import logging
 import re
+from typing import List
+from typing import Set
 import warnings
 
-from typing import List, Set
 # See https://github.com/pyca/cryptography/issues/4275
 from cryptography import x509  # type: ignore
 from cryptography.exceptions import InvalidSignature

certbot/certbot/display/ops.py

@@ -2,19 +2,13 @@
 import logging
 from textwrap import indent
 
-import zope.component
-
 from certbot import errors
-from certbot import interfaces
 from certbot import util
 from certbot.compat import os
 from certbot.display import util as display_util
 
 logger = logging.getLogger(__name__)
 
-# Define a helper function to avoid verbose code
-z_util = zope.component.getUtility
-
 
 def get_email(invalid=False, optional=True):
     """Prompt for valid email address.
@@ -48,9 +42,8 @@ def get_email(invalid=False, optional=True):
 
     while True:
         try:
-            code, email = z_util(interfaces.IDisplay).input(
-                invalid_prefix + msg if invalid else msg,
-                force_interactive=True)
+            code, email = display_util.input_text(invalid_prefix + msg if invalid else msg,
+                                                  force_interactive=True)
         except errors.MissingCommandlineFlag:
             msg = ("You should register before running non-interactively, "
                    "or provide --agree-tos and --email <email_address> flags.")
@@ -81,12 +74,12 @@ def choose_account(accounts):
     # Note this will get more complicated once we start recording authorizations
     labels = [acc.slug for acc in accounts]
 
-    code, index = z_util(interfaces.IDisplay).menu(
-        "Please choose an account", labels, force_interactive=True)
+    code, index = display_util.menu("Please choose an account", labels, force_interactive=True)
     if code == display_util.OK:
         return accounts[index]
     return None
 
+
 def choose_values(values, question=None):
     """Display screen to let user pick one or multiple values from the provided
     list.
@@ -96,12 +89,12 @@ def choose_values(values, question=None):
     :returns: List of selected values
     :rtype: list
     """
-    code, items = z_util(interfaces.IDisplay).checklist(
-        question, tags=values, force_interactive=True)
+    code, items = display_util.checklist(question, tags=values, force_interactive=True)
     if code == display_util.OK and items:
         return items
     return []
 
+
 def choose_names(installer, question=None):
     """Display screen to select domains to validate.
 
@@ -147,6 +140,7 @@ def get_valid_domains(domains):
             continue
     return valid_domains
 
+
 def _sort_names(FQDNs):
     """Sort FQDNs by SLD (and if many, by their subdomains)
 
@@ -169,13 +163,13 @@ def _filter_names(names, override_question=None):
     :rtype: tuple
 
     """
-    #Sort by domain first, and then by subdomain
+    # Sort by domain first, and then by subdomain
     sorted_names = _sort_names(names)
     if override_question:
         question = override_question
     else:
         question = "Which names would you like to activate HTTPS for?"
-    code, names = z_util(interfaces.IDisplay).checklist(
+    code, names = display_util.checklist(
         question, tags=sorted_names, cli_flag="--domains", force_interactive=True)
     return code, [str(s) for s in names]
 
@@ -189,7 +183,7 @@ def _choose_names_manually(prompt_prefix=""):
     :rtype: `list` of `str`
 
     """
-    code, input_ = z_util(interfaces.IDisplay).input(
+    code, input_ = display_util.input_text(
         prompt_prefix +
         "Please enter the domain name(s) you would like on your certificate "
         "(comma and/or space separated)",
@@ -217,17 +211,16 @@ def _choose_names_manually(prompt_prefix=""):
             retry_message = (
                 "One or more of the entered domain names was not valid:"
                 "{0}{0}").format(os.linesep)
-            for domain in invalid_domains:
+            for invalid_domain, err in invalid_domains.items():
                 retry_message = retry_message + "{1}: {2}{0}".format(
-                    os.linesep, domain, invalid_domains[domain])
+                    os.linesep, invalid_domain, err)
             retry_message = retry_message + (
                 "{0}Would you like to re-enter the names?{0}").format(
                     os.linesep)
 
         if retry_message:
             # We had error in input
-            retry = z_util(interfaces.IDisplay).yesno(retry_message,
-                                                      force_interactive=True)
+            retry = display_util.yesno(retry_message, force_interactive=True)
             if retry:
                 return _choose_names_manually()
         else:
@@ -332,7 +325,7 @@ def _get_validated(method, validator, message, default=None, **kwargs):
                              raw,
                              message,
                              exc_info=True)
-                zope.component.getUtility(interfaces.IDisplay).notification(str(error), pause=False)
+                display_util.notification(str(error), pause=False)
         else:
             return code, raw
 
@@ -348,8 +341,7 @@ def validated_input(validator, *args, **kwargs):
     :return: as `~certbot.interfaces.IDisplay.input`
     :rtype: tuple
     """
-    return _get_validated(zope.component.getUtility(interfaces.IDisplay).input,
-                          validator, *args, **kwargs)
+    return _get_validated(display_util.input_text, validator, *args, **kwargs)
 
 
 def validated_directory(validator, *args, **kwargs):
@@ -364,5 +356,4 @@ def validated_directory(validator, *args, **kwargs):
     :return: as `~certbot.interfaces.IDisplay.directory_select`
     :rtype: tuple
     """
-    return _get_validated(zope.component.getUtility(interfaces.IDisplay).directory_select,
-                          validator, *args, **kwargs)
+    return _get_validated(display_util.directory_select, validator, *args, **kwargs)

certbot/certbot/display/util.py

@@ -11,18 +11,17 @@ Other messages can use the `logging` module. See `log.py`.
 """
 import logging
 import sys
-import textwrap
 from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
 
-import zope.component
-import zope.interface
 
-from certbot import errors
-from certbot import interfaces
-from certbot._internal import constants
-from certbot._internal.display import completer
 from certbot.compat import misc
-from certbot.compat import os
+# These imports are done to not break the public API of the module.
+from certbot._internal.display.obj import FileDisplay  # pylint: disable=unused-import
+from certbot._internal.display.obj import NoninteractiveDisplay  # pylint: disable=unused-import
+from certbot._internal.display import obj
 
 logger = logging.getLogger(__name__)
 
@@ -46,26 +45,145 @@ SIDE_FRAME = ("- " * 39) + "-"
 """Display boundary (alternates spaces, so when copy-pasted, markdown doesn't interpret
 it as a heading)"""
 
-def _wrap_lines(msg):
-    """Format lines nicely to 80 chars.
 
-    :param str msg: Original message
+def notify(msg: str) -> None:
+    """Display a basic status message.
 
-    :returns: Formatted message respecting newlines in message
-    :rtype: str
+    :param str msg: message to display
 
     """
-    lines = msg.splitlines()
-    fixed_l = []
+    obj.get_display().notification(msg, pause=False, decorate=False, wrap=False)
 
-    for line in lines:
-        fixed_l.append(textwrap.fill(
-            line,
-            80,
-            break_long_words=False,
-            break_on_hyphens=False))
 
-    return '\n'.join(fixed_l)
+def notification(message: str, pause: bool = True, wrap: bool = True,
+                 force_interactive: bool = False, decorate: bool = True) -> None:
+    """Displays a notification and waits for user acceptance.
+
+    :param str message: Message to display
+    :param bool pause: Whether or not the program should pause for the
+        user's confirmation
+    :param bool wrap: Whether or not the application should wrap text
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+    :param bool decorate: Whether to surround the message with a
+        decorated frame
+
+    """
+    obj.get_display().notification(message, pause=pause, wrap=wrap,
+                                   force_interactive=force_interactive, decorate=decorate)
+
+
+def menu(message: str, choices: Union[List[str], Tuple[str, str]],
+         default: Optional[int] = None, cli_flag: Optional[str] = None,
+         force_interactive: bool = False) -> Tuple[str, int]:
+    """Display a menu.
+
+    .. todo:: This doesn't enable the help label/button (I wasn't sold on
+        any interface I came up with for this). It would be a nice feature.
+
+    :param str message: title of menu
+    :param choices: Menu lines, len must be > 0
+    :type choices: list of tuples (tag, item) or
+        list of descriptions (tags will be enumerated)
+    :param default: default value to return (if one exists)
+    :param str cli_flag: option used to set this value with the CLI
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+
+    :returns: tuple of (`code`, `index`) where
+        `code` - str display exit code
+        `index` - int index of the user's selection
+
+    :rtype: tuple
+
+    """
+    return obj.get_display().menu(message, choices, default=default, cli_flag=cli_flag,
+                                  force_interactive=force_interactive)
+
+
+def input_text(message: str, default: Optional[str] = None, cli_flag: Optional[str] = None,
+               force_interactive: bool = False) -> Tuple[str, str]:
+    """Accept input from the user.
+
+    :param str message: message to display to the user
+    :param default: default value to return (if one exists)
+    :param str cli_flag: option used to set this value with the CLI
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+
+    :returns: tuple of (`code`, `input`) where
+        `code` - str display exit code
+        `input` - str of the user's input
+    :rtype: tuple
+
+    """
+    return obj.get_display().input(message, default=default, cli_flag=cli_flag,
+                                   force_interactive=force_interactive)
+
+
+def yesno(message: str, yes_label: str = "Yes", no_label: str = "No",
+          default: Optional[bool] = None, cli_flag: Optional[str] = None,
+          force_interactive: bool = False) -> bool:
+    """Query the user with a yes/no question.
+
+    Yes and No label must begin with different letters, and must contain at
+    least one letter each.
+
+    :param str message: question for the user
+    :param str yes_label: Label of the "Yes" parameter
+    :param str no_label: Label of the "No" parameter
+    :param default: default value to return (if one exists)
+    :param str cli_flag: option used to set this value with the CLI
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+
+    :returns: True for "Yes", False for "No"
+    :rtype: bool
+
+    """
+    return obj.get_display().yesno(message, yes_label=yes_label, no_label=no_label, default=default,
+                                   cli_flag=cli_flag, force_interactive=force_interactive)
+
+
+def checklist(message: str, tags: List[str], default: Optional[str] = None,
+              cli_flag: Optional[str] = None,
+              force_interactive: bool = False) -> Tuple[str, List[str]]:
+    """Display a checklist.
+
+    :param str message: Message to display to user
+    :param list tags: `str` tags to select, len(tags) > 0
+    :param default: default value to return (if one exists)
+    :param str cli_flag: option used to set this value with the CLI
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+
+    :returns: tuple of (`code`, `tags`) where
+        `code` - str display exit code
+        `tags` - list of selected tags
+    :rtype: tuple
+
+    """
+    return obj.get_display().checklist(message, tags, default=default, cli_flag=cli_flag,
+                                       force_interactive=force_interactive)
+
+
+def directory_select(message: str, default: Optional[str] = None, cli_flag: Optional[str] = None,
+                     force_interactive: bool = False) -> Tuple[int, str]:
+    """Display a directory selection screen.
+
+    :param str message: prompt to give the user
+    :param default: default value to return (if one exists)
+    :param str cli_flag: option used to set this value with the CLI
+    :param bool force_interactive: True if it's safe to prompt the user
+        because it won't cause any workflow regressions
+
+    :returns: tuple of the form (`code`, `string`) where
+        `code` - display exit code
+        `string` - input entered by the user
+
+    """
+    return obj.get_display().directory_select(message, default=default, cli_flag=cli_flag,
+                                              force_interactive=force_interactive)
 
 
 def input_with_timeout(prompt=None, timeout=36000.0):
@@ -98,366 +216,6 @@ def input_with_timeout(prompt=None, timeout=36000.0):
     return line.rstrip('\n')
 
 
-def notify(msg: str) -> None:
-    """Display a basic status message.
-
-    :param str msg: message to display
-
-    """
-    zope.component.getUtility(interfaces.IDisplay).notification(
-        msg, pause=False, decorate=False, wrap=False
-    )
-
-
-@zope.interface.implementer(interfaces.IDisplay)
-class FileDisplay:
-    """File-based display."""
-    # see https://github.com/certbot/certbot/issues/3915
-
-    def __init__(self, outfile, force_interactive):
-        super().__init__()
-        self.outfile = outfile
-        self.force_interactive = force_interactive
-        self.skipped_interaction = False
-
-    def notification(self, message, pause=True,
-                     wrap=True, force_interactive=False,
-                     decorate=True):
-        """Displays a notification and waits for user acceptance.
-
-        :param str message: Message to display
-        :param bool pause: Whether or not the program should pause for the
-            user's confirmation
-        :param bool wrap: Whether or not the application should wrap text
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-        :param bool decorate: Whether to surround the message with a
-            decorated frame
-
-        """
-        if wrap:
-            message = _wrap_lines(message)
-
-        logger.debug("Notifying user: %s", message)
-
-        self.outfile.write(
-            (("{line}{frame}{line}" if decorate else "") +
-             "{msg}{line}" +
-             ("{frame}{line}" if decorate else ""))
-            .format(line=os.linesep, frame=SIDE_FRAME, msg=message)
-        )
-        self.outfile.flush()
-
-        if pause:
-            if self._can_interact(force_interactive):
-                input_with_timeout("Press Enter to Continue")
-            else:
-                logger.debug("Not pausing for user confirmation")
-
-    def menu(self, message, choices, ok_label=None, cancel_label=None,  # pylint: disable=unused-argument
-             help_label=None, default=None,  # pylint: disable=unused-argument
-             cli_flag=None, force_interactive=False, **unused_kwargs):
-        """Display a menu.
-
-        .. todo:: This doesn't enable the help label/button (I wasn't sold on
-           any interface I came up with for this). It would be a nice feature
-
-        :param str message: title of menu
-        :param choices: Menu lines, len must be > 0
-        :type choices: list of tuples (tag, item) or
-            list of descriptions (tags will be enumerated)
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-
-        :returns: tuple of (`code`, `index`) where
-            `code` - str display exit code
-            `index` - int index of the user's selection
-
-        :rtype: tuple
-
-        """
-        if self._return_default(message, default, cli_flag, force_interactive):
-            return OK, default
-
-        self._print_menu(message, choices)
-
-        code, selection = self._get_valid_int_ans(len(choices))
-
-        return code, selection - 1
-
-    def input(self, message, default=None,
-              cli_flag=None, force_interactive=False, **unused_kwargs):
-        """Accept input from the user.
-
-        :param str message: message to display to the user
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-
-        :returns: tuple of (`code`, `input`) where
-            `code` - str display exit code
-            `input` - str of the user's input
-        :rtype: tuple
-
-        """
-        if self._return_default(message, default, cli_flag, force_interactive):
-            return OK, default
-
-        # Trailing space must be added outside of _wrap_lines to be preserved
-        message = _wrap_lines("%s (Enter 'c' to cancel):" % message) + " "
-        ans = input_with_timeout(message)
-
-        if ans in ("c", "C"):
-            return CANCEL, "-1"
-        return OK, ans
-
-    def yesno(self, message, yes_label="Yes", no_label="No", default=None,
-              cli_flag=None, force_interactive=False, **unused_kwargs):
-        """Query the user with a yes/no question.
-
-        Yes and No label must begin with different letters, and must contain at
-        least one letter each.
-
-        :param str message: question for the user
-        :param str yes_label: Label of the "Yes" parameter
-        :param str no_label: Label of the "No" parameter
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-
-        :returns: True for "Yes", False for "No"
-        :rtype: bool
-
-        """
-        if self._return_default(message, default, cli_flag, force_interactive):
-            return default
-
-        message = _wrap_lines(message)
-
-        self.outfile.write("{0}{frame}{msg}{0}{frame}".format(
-            os.linesep, frame=SIDE_FRAME + os.linesep, msg=message))
-        self.outfile.flush()
-
-        while True:
-            ans = input_with_timeout("{yes}/{no}: ".format(
-                yes=_parens_around_char(yes_label),
-                no=_parens_around_char(no_label)))
-
-            # Couldn't get pylint indentation right with elif
-            # elif doesn't matter in this situation
-            if (ans.startswith(yes_label[0].lower()) or
-                    ans.startswith(yes_label[0].upper())):
-                return True
-            if (ans.startswith(no_label[0].lower()) or
-                    ans.startswith(no_label[0].upper())):
-                return False
-
-    def checklist(self, message, tags, default=None,
-                  cli_flag=None, force_interactive=False, **unused_kwargs):
-        """Display a checklist.
-
-        :param str message: Message to display to user
-        :param list tags: `str` tags to select, len(tags) > 0
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-
-        :returns: tuple of (`code`, `tags`) where
-            `code` - str display exit code
-            `tags` - list of selected tags
-        :rtype: tuple
-
-        """
-        if self._return_default(message, default, cli_flag, force_interactive):
-            return OK, default
-
-        while True:
-            self._print_menu(message, tags)
-
-            code, ans = self.input("Select the appropriate numbers separated "
-                                   "by commas and/or spaces, or leave input "
-                                   "blank to select all options shown",
-                                   force_interactive=True)
-
-            if code == OK:
-                if not ans.strip():
-                    ans = " ".join(str(x) for x in range(1, len(tags)+1))
-                indices = separate_list_input(ans)
-                selected_tags = self._scrub_checklist_input(indices, tags)
-                if selected_tags:
-                    return code, selected_tags
-                self.outfile.write(
-                    "** Error - Invalid selection **%s" % os.linesep)
-                self.outfile.flush()
-            else:
-                return code, []
-
-    def _return_default(self, prompt, default, cli_flag, force_interactive):
-        """Should we return the default instead of prompting the user?
-
-        :param str prompt: prompt for the user
-        :param default: default answer to prompt
-        :param str cli_flag: command line option for setting an answer
-            to this question
-        :param bool force_interactive: if interactivity is forced by the
-            IDisplay call
-
-        :returns: True if we should return the default without prompting
-        :rtype: bool
-
-        """
-        # assert_valid_call(prompt, default, cli_flag, force_interactive)
-        if self._can_interact(force_interactive):
-            return False
-        if default is None:
-            msg = "Unable to get an answer for the question:\n{0}".format(prompt)
-            if cli_flag:
-                msg += (
-                    "\nYou can provide an answer on the "
-                    "command line with the {0} flag.".format(cli_flag))
-            raise errors.Error(msg)
-        logger.debug(
-            "Falling back to default %s for the prompt:\n%s",
-            default, prompt)
-        return True
-
-    def _can_interact(self, force_interactive):
-        """Can we safely interact with the user?
-
-        :param bool force_interactive: if interactivity is forced by the
-            IDisplay call
-
-        :returns: True if the display can interact with the user
-        :rtype: bool
-
-        """
-        if (self.force_interactive or force_interactive or
-                sys.stdin.isatty() and self.outfile.isatty()):
-            return True
-        if not self.skipped_interaction:
-            logger.warning(
-                "Skipped user interaction because Certbot doesn't appear to "
-                "be running in a terminal. You should probably include "
-                "--non-interactive or %s on the command line.",
-                constants.FORCE_INTERACTIVE_FLAG)
-            self.skipped_interaction = True
-        return False
-
-    def directory_select(self, message, default=None, cli_flag=None,
-                         force_interactive=False, **unused_kwargs):
-        """Display a directory selection screen.
-
-        :param str message: prompt to give the user
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-        :param bool force_interactive: True if it's safe to prompt the user
-            because it won't cause any workflow regressions
-
-        :returns: tuple of the form (`code`, `string`) where
-            `code` - display exit code
-            `string` - input entered by the user
-
-        """
-        with completer.Completer():
-            return self.input(message, default, cli_flag, force_interactive)
-
-    def _scrub_checklist_input(self, indices, tags):
-        """Validate input and transform indices to appropriate tags.
-
-        :param list indices: input
-        :param list tags: Original tags of the checklist
-
-        :returns: valid tags the user selected
-        :rtype: :class:`list` of :class:`str`
-
-        """
-        # They should all be of type int
-        try:
-            indices = [int(index) for index in indices]
-        except ValueError:
-            return []
-
-        # Remove duplicates
-        indices = list(set(indices))
-
-        # Check all input is within range
-        for index in indices:
-            if index < 1 or index > len(tags):
-                return []
-        # Transform indices to appropriate tags
-        return [tags[index - 1] for index in indices]
-
-    def _print_menu(self, message, choices):
-        """Print a menu on the screen.
-
-        :param str message: title of menu
-        :param choices: Menu lines
-        :type choices: list of tuples (tag, item) or
-            list of descriptions (tags will be enumerated)
-
-        """
-        # Can take either tuples or single items in choices list
-        if choices and isinstance(choices[0], tuple):
-            choices = ["%s - %s" % (c[0], c[1]) for c in choices]
-
-        # Write out the message to the user
-        self.outfile.write(
-            "{new}{msg}{new}".format(new=os.linesep, msg=message))
-        self.outfile.write(SIDE_FRAME + os.linesep)
-
-        # Write out the menu choices
-        for i, desc in enumerate(choices, 1):
-            msg = "{num}: {desc}".format(num=i, desc=desc)
-            self.outfile.write(_wrap_lines(msg))
-
-            # Keep this outside of the textwrap
-            self.outfile.write(os.linesep)
-
-        self.outfile.write(SIDE_FRAME + os.linesep)
-        self.outfile.flush()
-
-    def _get_valid_int_ans(self, max_):
-        """Get a numerical selection.
-
-        :param int max: The maximum entry (len of choices), must be positive
-
-        :returns: tuple of the form (`code`, `selection`) where
-            `code` - str display exit code ('ok' or cancel')
-            `selection` - int user's selection
-        :rtype: tuple
-
-        """
-        selection = -1
-        if max_ > 1:
-            input_msg = ("Select the appropriate number "
-                         "[1-{max_}] then [enter] (press 'c' to "
-                         "cancel): ".format(max_=max_))
-        else:
-            input_msg = ("Press 1 [enter] to confirm the selection "
-                         "(press 'c' to cancel): ")
-        while selection < 1:
-            ans = input_with_timeout(input_msg)
-            if ans.startswith("c") or ans.startswith("C"):
-                return CANCEL, -1
-            try:
-                selection = int(ans)
-                if selection < 1 or selection > max_:
-                    selection = -1
-                    raise ValueError
-
-            except ValueError:
-                self.outfile.write(
-                    "{0}** Invalid input **{0}".format(os.linesep))
-                self.outfile.flush()
-
-        return OK, selection
-
-
 def assert_valid_call(prompt, default, cli_flag, force_interactive):
     """Verify that provided arguments is a valid IDisplay call.
 
@@ -476,141 +234,6 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive):
     assert default is not None or force_interactive, msg
 
 
-@zope.interface.implementer(interfaces.IDisplay)
-class NoninteractiveDisplay:
-    """An iDisplay implementation that never asks for interactive user input"""
-
-    def __init__(self, outfile, *unused_args, **unused_kwargs):
-        super().__init__()
-        self.outfile = outfile
-
-    def _interaction_fail(self, message, cli_flag, extra=""):
-        "Error out in case of an attempt to interact in noninteractive mode"
-        msg = "Missing command line flag or config entry for this setting:\n"
-        msg += message
-        if extra:
-            msg += "\n" + extra
-        if cli_flag:
-            msg += "\n\n(You can set this with the {0} flag)".format(cli_flag)
-        raise errors.MissingCommandlineFlag(msg)
-
-    def notification(self, message, pause=False, wrap=True, decorate=True, **unused_kwargs):  # pylint: disable=unused-argument
-        """Displays a notification without waiting for user acceptance.
-
-        :param str message: Message to display to stdout
-        :param bool pause: The NoninteractiveDisplay waits for no keyboard
-        :param bool wrap: Whether or not the application should wrap text
-        :param bool decorate: Whether to apply a decorated frame to the message
-
-        """
-        if wrap:
-            message = _wrap_lines(message)
-
-        logger.debug("Notifying user: %s", message)
-
-        self.outfile.write(
-            (("{line}{frame}{line}" if decorate else "") +
-             "{msg}{line}" +
-             ("{frame}{line}" if decorate else ""))
-            .format(line=os.linesep, frame=SIDE_FRAME, msg=message)
-        )
-        self.outfile.flush()
-
-    def menu(self, message, choices, ok_label=None, cancel_label=None,
-             help_label=None, default=None, cli_flag=None, **unused_kwargs):
-        # pylint: disable=unused-argument
-        """Avoid displaying a menu.
-
-        :param str message: title of menu
-        :param choices: Menu lines, len must be > 0
-        :type choices: list of tuples (tag, item) or
-            list of descriptions (tags will be enumerated)
-        :param int default: the default choice
-        :param dict kwargs: absorbs various irrelevant labelling arguments
-
-        :returns: tuple of (`code`, `index`) where
-            `code` - str display exit code
-            `index` - int index of the user's selection
-        :rtype: tuple
-        :raises errors.MissingCommandlineFlag: if there was no default
-
-        """
-        if default is None:
-            self._interaction_fail(message, cli_flag, "Choices: " + repr(choices))
-
-        return OK, default
-
-    def input(self, message, default=None, cli_flag=None, **unused_kwargs):
-        """Accept input from the user.
-
-        :param str message: message to display to the user
-
-        :returns: tuple of (`code`, `input`) where
-            `code` - str display exit code
-            `input` - str of the user's input
-        :rtype: tuple
-        :raises errors.MissingCommandlineFlag: if there was no default
-
-        """
-        if default is None:
-            self._interaction_fail(message, cli_flag)
-        return OK, default
-
-    def yesno(self, message, yes_label=None, no_label=None,  # pylint: disable=unused-argument
-              default=None, cli_flag=None, **unused_kwargs):
-        """Decide Yes or No, without asking anybody
-
-        :param str message: question for the user
-        :param dict kwargs: absorbs yes_label, no_label
-
-        :raises errors.MissingCommandlineFlag: if there was no default
-        :returns: True for "Yes", False for "No"
-        :rtype: bool
-
-        """
-        if default is None:
-            self._interaction_fail(message, cli_flag)
-        return default
-
-    def checklist(self, message, tags, default=None,
-                  cli_flag=None, **unused_kwargs):
-        """Display a checklist.
-
-        :param str message: Message to display to user
-        :param list tags: `str` tags to select, len(tags) > 0
-        :param dict kwargs: absorbs default_status arg
-
-        :returns: tuple of (`code`, `tags`) where
-            `code` - str display exit code
-            `tags` - list of selected tags
-        :rtype: tuple
-
-        """
-        if default is None:
-            self._interaction_fail(message, cli_flag, "? ".join(tags))
-        return OK, default
-
-    def directory_select(self, message, default=None,
-                         cli_flag=None, **unused_kwargs):
-        """Simulate prompting the user for a directory.
-
-        This function returns default if it is not ``None``, otherwise,
-        an exception is raised explaining the problem. If cli_flag is
-        not ``None``, the error message will include the flag that can
-        be used to set this value with the CLI.
-
-        :param str message: prompt to give the user
-        :param default: default value to return (if one exists)
-        :param str cli_flag: option used to set this value with the CLI
-
-        :returns: tuple of the form (`code`, `string`) where
-            `code` - int display exit code
-            `string` - input entered by the user
-
-        """
-        return self.input(message, default, cli_flag)
-
-
 def separate_list_input(input_):
     """Separate a comma or space separated list.
 
@@ -626,15 +249,6 @@ def separate_list_input(input_):
     return [str(string) for string in no_commas.split()]
 
 
-def _parens_around_char(label):
-    """Place parens around first character of label.
-
-    :param str label: Must contain at least one character
-
-    """
-    return "({first}){rest}".format(first=label[0], rest=label[1:])
-
-
 def summarize_domain_list(domains: List[str]) -> str:
     """Summarizes a list of domains in the format of:
         example.com.com and N more domains

certbot/certbot/interfaces.py

@@ -549,7 +549,8 @@ class IReporter(zope.interface.Interface):
 class RenewableCert(object, metaclass=abc.ABCMeta):
     """Interface to a certificate lineage."""
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def cert_path(self):
         """Path to the certificate file.
 
@@ -557,7 +558,8 @@ class RenewableCert(object, metaclass=abc.ABCMeta):
 
         """
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def key_path(self):
         """Path to the private key file.
 
@@ -565,7 +567,8 @@ class RenewableCert(object, metaclass=abc.ABCMeta):
 
         """
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def chain_path(self):
         """Path to the certificate chain file.
 
@@ -573,7 +576,8 @@ class RenewableCert(object, metaclass=abc.ABCMeta):
 
         """
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def fullchain_path(self):
         """Path to the full chain file.
 
@@ -583,7 +587,8 @@ class RenewableCert(object, metaclass=abc.ABCMeta):
 
         """
 
-    @abc.abstractproperty
+    @property
+    @abc.abstractmethod
     def lineagename(self):
         """Name given to the certificate lineage.
 

certbot/certbot/plugins/dns_common.py

@@ -169,7 +169,7 @@ class DNSAuthenticator(common.Plugin):
             indicate any issue.
         """
 
-        def __validator(filename):
+        def __validator(filename): # pylint: disable=unused-private-member
             configuration = CredentialsConfiguration(filename, self.dest)
 
             if required_variables:
@@ -199,7 +199,7 @@ class DNSAuthenticator(common.Plugin):
         :rtype: str
         """
 
-        def __validator(i):
+        def __validator(i): # pylint: disable=unused-private-member
             if not i:
                 raise errors.PluginError('Please enter your {0}.'.format(label))
 
@@ -225,7 +225,7 @@ class DNSAuthenticator(common.Plugin):
         :rtype: str
         """
 
-        def __validator(filename):
+        def __validator(filename): # pylint: disable=unused-private-member
             if not filename:
                 raise errors.PluginError('Please enter a valid path to your {0}.'.format(label))
 

certbot/certbot/plugins/dns_test_common_lexicon.py

@@ -67,7 +67,7 @@ class _LexiconAwareTestCase(Protocol):
 
 class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest):
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self: _AuthenticatorCallableLexiconTestCase, unused_mock_get_utility):
         self.auth.perform([self.achall])
 

certbot/certbot/tests/util.py

@@ -37,9 +37,8 @@ try:
         "use unittest.mock. Be sure to update your code accordingly.",
         PendingDeprecationWarning
     )
-except ImportError: # pragma: no cover
-    from unittest import mock # type: ignore
-
+except ImportError:  # pragma: no cover
+    from unittest import mock  # type: ignore
 
 
 def vector_path(*names):
@@ -150,10 +149,7 @@ def make_lineage(config_dir, testfile, ec=False):
 
 
 def patch_get_utility(target='zope.component.getUtility'):
-    """Patch zope.component.getUtility to use a special mock IDisplay.
-
-    The mock IDisplay works like a regular mock object, except it also
-    also asserts that methods are called with valid arguments.
+    """Deprecated, patch certbot.display.util directly or use patch_display_util instead.
 
     :param str target: path to patch
 
@@ -161,18 +157,16 @@ def patch_get_utility(target='zope.component.getUtility'):
     :rtype: mock.MagicMock
 
     """
-    return mock.patch(target, new_callable=_create_get_utility_mock)
+    warnings.warn('Decorator certbot.tests.util.patch_get_utility is deprecated. You should now '
+                  'patch certbot.display.util yourself directly or use '
+                  'certbot.tests.util.patch_display_util as a temporary workaround.')
+    return mock.patch(target, new_callable=_create_display_util_mock)
 
 
 def patch_get_utility_with_stdout(target='zope.component.getUtility',
                                   stdout=None):
-    """Patch zope.component.getUtility to use a special mock IDisplay.
-
-    The mock IDisplay works like a regular mock object, except it also
-    also asserts that methods are called with valid arguments.
-
-    The `message` argument passed to the IDisplay methods is passed to
-    stdout's write method.
+    """Deprecated, patch certbot.display.util directly
+    or use patch_display_util_with_stdout instead.
 
     :param str target: path to patch
     :param object stdout: object to write standard output to; it is
@@ -181,11 +175,71 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility',
     :returns: mock zope.component.getUtility
     :rtype: mock.MagicMock
 
+    """
+    warnings.warn('Decorator certbot.tests.util.patch_get_utility_with_stdout is deprecated. You '
+                  'should now patch certbot.display.util yourself directly or use '
+                  'use certbot.tests.util.patch_display_util_with_stdout as a temporary '
+                  'workaround.')
+    stdout = stdout if stdout else io.StringIO()
+    freezable_mock = _create_display_util_mock_with_stdout(stdout)
+    return mock.patch(target, new=freezable_mock)
+
+
+def patch_display_util():
+    """Patch certbot.display.util to use a special mock IDisplay.
+
+    The mock IDisplay works like a regular mock object, except it also
+    also asserts that methods are called with valid arguments.
+
+    The mock created by this patch mocks out Certbot internals so this can be
+    used like the old patch_get_utility function. That is, the mock object will
+    be called by the certbot.display.util functions and the mock returned by
+    that call will be used as the IDisplay object. This was done to simplify
+    the transition from zope.component and mocking certbot.display.util
+    functions directly in test code should be preferred over using this
+    function in the future.
+
+    See https://github.com/certbot/certbot/issues/8948
+
+    :returns: patch on the function used internally by certbot.display.util to
+        get an IDisplay object
+    :rtype: unittest.mock._patch
+
+    """
+    return mock.patch('certbot._internal.display.obj.get_display',
+                      new_callable=_create_display_util_mock)
+
+
+def patch_display_util_with_stdout(stdout=None):
+    """Patch certbot.display.util to use a special mock IDisplay.
+
+    The mock IDisplay works like a regular mock object, except it also
+    asserts that methods are called with valid arguments.
+
+    The mock created by this patch mocks out Certbot internals so this can be
+    used like the old patch_get_utility function. That is, the mock object will
+    be called by the certbot.display.util functions and the mock returned by
+    that call will be used as the IDisplay object. This was done to simplify
+    the transition from zope.component and mocking certbot.display.util
+    functions directly in test code should be preferred over using this
+    function in the future.
+
+    See https://github.com/certbot/certbot/issues/8948
+
+    The `message` argument passed to the IDisplay methods is passed to
+    stdout's write method.
+
+    :param object stdout: object to write standard output to; it is
+        expected to have a `write` method
+    :returns: patch on the function used internally by certbot.display.util to
+        get an IDisplay object
+    :rtype: unittest.mock._patch
+
     """
     stdout = stdout if stdout else io.StringIO()
 
-    freezable_mock = _create_get_utility_mock_with_stdout(stdout)
-    return mock.patch(target, new=freezable_mock)
+    return mock.patch('certbot._internal.display.obj.get_display',
+                      new=_create_display_util_mock_with_stdout(stdout))
 
 
 class FreezableMock:
@@ -256,7 +310,7 @@ class FreezableMock:
         return object.__setattr__(self, name, value)
 
 
-def _create_get_utility_mock():
+def _create_display_util_mock():
     display = FreezableMock()
     # Use pylint code for disable to keep on single line under line length limit
     for name in interfaces.IDisplay.names():
@@ -267,7 +321,7 @@ def _create_get_utility_mock():
     return FreezableMock(frozen=True, return_value=display)
 
 
-def _create_get_utility_mock_with_stdout(stdout):
+def _create_display_util_mock_with_stdout(stdout):
     def _write_msg(message, *unused_args, **unused_kwargs):
         """Write to message to stdout.
         """
@@ -281,20 +335,17 @@ def _create_get_utility_mock_with_stdout(stdout):
         _assert_valid_call(args, kwargs)
         _write_msg(*args, **kwargs)
 
-
     display = FreezableMock()
     # Use pylint code for disable to keep on single line under line length limit
     for name in interfaces.IDisplay.names():
         if name == 'notification':
             frozen_mock = FreezableMock(frozen=True,
                                         func=_write_msg)
-            setattr(display, name, frozen_mock)
         else:
             frozen_mock = FreezableMock(frozen=True,
                                         func=mock_method)
-            setattr(display, name, frozen_mock)
+        setattr(display, name, frozen_mock)
     display.freeze()
-
     return FreezableMock(frozen=True, return_value=display)
 
 

certbot/docs/contributing.rst

@@ -566,6 +566,55 @@ and run the command:
 This would generate the HTML documentation in ``_build/html`` in your current
 ``docs/`` directory.
 
+Certbot's dependencies
+======================
+
+We attempt to pin all of Certbot's dependencies whenever we can for reliability
+and consistency. Some of the places we have Certbot's dependencies pinned
+include our snaps, Docker images, Windows installer, CI, and our development
+environments.
+
+In most cases, the file where dependency versions are specified is
+``tools/requirements.txt``. There are two exceptions to this. The first is our
+"oldest" tests where ``tools/oldest_constraints.txt`` is used instead. The
+purpose of the "oldest" tests is to ensure Certbot continues to work with the
+oldest versions of our dependencies which we claim to support. The oldest
+versions of the dependencies we support should also be declared in our setup.py
+files to communicate this information to our users.
+
+The second exception to using ``tools/requirements.txt`` is in our unpinned
+tests. As of writing this, there is one test we run nightly in CI where we
+leave Certbot's dependencies unpinned. The thinking behind this test is to help
+us learn about breaking changes in our dependencies so that we can respond
+accordingly.
+
+The choices of whether Certbot's dependencies are pinned and what file is used
+if they are should be automatically handled for you most of the time by
+Certbot's tooling. The way it works though is ``tools/pip_install.py`` (which
+many of our other tools build on) checks for the presence of environment
+variables. If ``CERTBOT_NO_PIN`` is set to 1, Certbot's dependencies will not
+be pinned. If that variable is not set and ``CERTBOT_OLDEST`` is set to 1,
+``tools/oldest_constraints.txt`` will be used as constraints for ``pip``.
+Otherwise, ``tools/requirements.txt`` is used as constraints.
+
+Updating dependency versions
+----------------------------
+
+``tools/requirements.txt`` and ``tools/oldest_constraints.txt`` can be updated
+using ``tools/pinning/current/repin.sh`` and ``tools/pinning/oldest/repin.sh``
+respectively. This works by using ``poetry`` to generate pinnings based on a
+Poetry project defined by the ``pyproject.toml`` file in the same directory as
+the script. In many cases, you can just run the script to generate updated
+dependencies, however, if you need to pin back packages or unpin packages that
+were previously restricted to an older version, you will need to modify the
+``pyproject.toml`` file. The syntax used by this file is described at
+https://python-poetry.org/docs/pyproject/ and how dependencies are specified in
+this file is further described at
+https://python-poetry.org/docs/dependency-specification/.
+
+If you want to learn more about the design used here, see
+``tools/pinning/DESIGN.md`` in the Certbot repo.
+
 .. _docker-dev:
 
 Running the client with Docker

certbot/setup.py

@@ -70,13 +70,10 @@ dev_extras = [
     'azure-devops',
     'ipdb',
     'PyGithub',
-    'pip',
     # poetry 1.2.0+ is required for it to pin pip, setuptools, and wheel. See
     # https://github.com/python-poetry/poetry/issues/1584.
     'poetry>=1.2.0a1',
-    'tox',
     'twine',
-    'wheel',
 ]
 
 docs_extras = [
@@ -87,16 +84,22 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+# Tools like pip, wheel, and tox are listed here to ensure they are properly
+# pinned and installed during automated testing.
 test_extras = [
     'coverage',
     'mypy',
+    'pip',
     'pylint',
     'pytest',
     'pytest-cov',
     'pytest-xdist',
+    'setuptools',
+    'tox',
     # typing-extensions is required to import typing.Protocol and make the mypy checks
     # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7
     'typing-extensions',
+    'wheel',
 ]
 
 

certbot/tests/auth_handler_test.py

@@ -5,9 +5,8 @@ import unittest
 
 try:
     import mock
-except ImportError: # pragma: no cover
+except ImportError:  # pragma: no cover
     from unittest import mock
-import zope.component
 
 from acme import challenges
 from acme import client as acme_client
@@ -15,8 +14,8 @@ from acme import errors as acme_errors
 from acme import messages
 from certbot import achallenges
 from certbot import errors
-from certbot import interfaces
 from certbot import util
+from certbot._internal.display import obj as display_obj
 from certbot.plugins import common as plugin_common
 from certbot.tests import acme_util
 from certbot.tests import util as test_util
@@ -70,8 +69,8 @@ class HandleAuthorizationsTest(unittest.TestCase):
 
         self.mock_display = mock.Mock()
         self.mock_config = mock.Mock(debug_challenges=False)
-        zope.component.provideUtility(
-            self.mock_display, interfaces.IDisplay)
+        with mock.patch("zope.component.provideUtility"):
+            display_obj.set_display(self.mock_display)
 
         self.mock_auth = mock.MagicMock(name="ApacheConfigurator")
 
@@ -307,7 +306,7 @@ class HandleAuthorizationsTest(unittest.TestCase):
         mock_order = mock.MagicMock(authorizations=authzrs)
         self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID)
 
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             with self.assertRaises(errors.AuthorizationError) as error:
                 self.handler.handle_authorizations(mock_order, self.mock_config, False)
         self.assertIn('Some challenges have failed.', str(error.exception))
@@ -342,7 +341,7 @@ class HandleAuthorizationsTest(unittest.TestCase):
 
         self.mock_net.poll.side_effect = _gen_mock_on_poll(status=messages.STATUS_INVALID)
 
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             with self.assertRaises(errors.AuthorizationError) as error:
                 self.handler.handle_authorizations(mock_order, self.mock_config, True)
 

certbot/tests/cert_manager_test.py

@@ -113,7 +113,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
         from certbot._internal import cert_manager
         cert_manager.delete(self.config)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot.display.util.notify')
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
@@ -129,7 +129,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
             "Deleted all files relating to certificate example.org."
         )
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
     def test_delete_from_config_no(self, mock_delete_files, mock_lineage_for_certname,
@@ -141,7 +141,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
         self._call()
         self.assertEqual(mock_delete_files.call_count, 0)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
     def test_delete_interactive_single_yes(self, mock_delete_files, mock_lineage_for_certname,
@@ -153,7 +153,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
         self._call()
         mock_delete_files.assert_called_once_with(self.config, "example.org")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
     def test_delete_interactive_single_no(self, mock_delete_files, mock_lineage_for_certname,
@@ -165,7 +165,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
         self._call()
         self.assertEqual(mock_delete_files.call_count, 0)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
     def test_delete_interactive_multiple_yes(self, mock_delete_files, mock_lineage_for_certname,
@@ -179,7 +179,7 @@ class DeleteTest(storage_test.BaseRenewableCertTest):
         mock_delete_files.assert_any_call(self.config, "other.org")
         self.assertEqual(mock_delete_files.call_count, 2)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     @mock.patch('certbot._internal.storage.delete_files')
     def test_delete_interactive_multiple_no(self, mock_delete_files, mock_lineage_for_certname,
@@ -200,14 +200,14 @@ class CertificatesTest(BaseCertManagerTest):
         return certificates(*args, **kwargs)
 
     @mock.patch('certbot._internal.cert_manager.logger')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_certificates_parse_fail(self, mock_utility, mock_logger):
         self._certificates(self.config)
         self.assertTrue(mock_logger.warning.called) #pylint: disable=no-member
         self.assertTrue(mock_utility.called)
 
     @mock.patch('certbot._internal.cert_manager.logger')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_certificates_quiet(self, mock_utility, mock_logger):
         self.config.quiet = True
         self._certificates(self.config)
@@ -216,7 +216,7 @@ class CertificatesTest(BaseCertManagerTest):
 
     @mock.patch('certbot.crypto_util.verify_renewable_cert')
     @mock.patch('certbot._internal.cert_manager.logger')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch("certbot._internal.storage.RenewableCert")
     @mock.patch('certbot._internal.cert_manager._report_human_readable')
     def test_certificates_parse_success(self, mock_report, mock_renewable_cert,
@@ -230,7 +230,7 @@ class CertificatesTest(BaseCertManagerTest):
         self.assertTrue(mock_renewable_cert.called)
 
     @mock.patch('certbot._internal.cert_manager.logger')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_certificates_no_files(self, mock_utility, mock_logger):
         empty_tempdir = tempfile.mkdtemp()
         empty_config = configuration.NamespaceConfig(mock.MagicMock(
@@ -408,7 +408,7 @@ class RenameLineageTest(BaseCertManagerTest):
         return cert_manager.rename_lineage(*args, **kwargs)
 
     @mock.patch('certbot._internal.storage.renewal_conf_files')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_no_certname(self, mock_get_utility, mock_renewal_conf_files):
         self.config.certname = None
         self.config.new_certname = "two"
@@ -425,7 +425,7 @@ class RenameLineageTest(BaseCertManagerTest):
         util_mock.menu.return_value = (display_util.OK, -1)
         self.assertRaises(errors.Error, self._call, self.config)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_no_new_certname(self, mock_get_utility):
         self.config.certname = "one"
         self.config.new_certname = None
@@ -437,7 +437,7 @@ class RenameLineageTest(BaseCertManagerTest):
         util_mock.input.return_value = (display_util.OK, None)
         self.assertRaises(errors.Error, self._call, self.config)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
     def test_no_existing_certname(self, mock_lineage_for_certname, unused_get_utility):
         self.config.certname = "one"
@@ -446,7 +446,7 @@ class RenameLineageTest(BaseCertManagerTest):
         self.assertRaises(errors.ConfigurationError,
             self._call, self.config)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch("certbot._internal.storage.RenewableCert._check_symlinks")
     def test_rename_cert(self, mock_check, unused_get_utility):
         mock_check.return_value = True
@@ -456,7 +456,7 @@ class RenameLineageTest(BaseCertManagerTest):
         self.assertIsNotNone(updated_lineage)
         self.assertEqual(updated_lineage.lineagename, self.config.new_certname)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch("certbot._internal.storage.RenewableCert._check_symlinks")
     def test_rename_cert_interactive_certname(self, mock_check, mock_get_utility):
         mock_check.return_value = True
@@ -469,7 +469,7 @@ class RenameLineageTest(BaseCertManagerTest):
         self.assertIsNotNone(updated_lineage)
         self.assertEqual(updated_lineage.lineagename, self.config.new_certname)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch("certbot._internal.storage.RenewableCert._check_symlinks")
     def test_rename_cert_bad_new_certname(self, mock_check, unused_get_utility):
         mock_check.return_value = True
@@ -619,7 +619,7 @@ class GetCertnameTest(unittest.TestCase):
     """Tests for certbot._internal.cert_manager."""
 
     def setUp(self):
-        get_utility_patch = test_util.patch_get_utility()
+        get_utility_patch = test_util.patch_display_util()
         self.mock_get_utility = get_utility_patch.start()
         self.addCleanup(get_utility_patch.stop)
         self.config = mock.MagicMock()

certbot/tests/cli_test.py

@@ -86,7 +86,7 @@ class ParseTest(unittest.TestCase):
     @staticmethod
     def parse(*args, **kwargs):
         """Mocks zope.component.getUtility and calls _unmocked_parse."""
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             return ParseTest._unmocked_parse(*args, **kwargs)
 
     def _help_output(self, args):
@@ -98,7 +98,7 @@ class ParseTest(unittest.TestCase):
             output.write(message)
 
         with mock.patch('certbot._internal.main.sys.stdout', new=output):
-            with test_util.patch_get_utility() as mock_get_utility:
+            with test_util.patch_display_util() as mock_get_utility:
                 mock_get_utility().notification.side_effect = write_msg
                 with mock.patch('certbot._internal.main.sys.stderr'):
                     self.assertRaises(SystemExit, self._unmocked_parse, args, output)
@@ -519,7 +519,7 @@ class SetByCliTest(unittest.TestCase):
 
 def _call_set_by_cli(var, args, verb):
     with mock.patch('certbot._internal.cli.helpful_parser') as mock_parser:
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             mock_parser.args = args
             mock_parser.verb = verb
             return cli.set_by_cli(var)

certbot/tests/client_test.py

@@ -3,25 +3,29 @@ import platform
 import shutil
 import tempfile
 import unittest
+from unittest.mock import MagicMock
 
 from josepy import interfaces
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock
 
 from certbot import errors
 from certbot import util
+from certbot._internal.display import obj as display_obj
 from certbot._internal import account
 from certbot.compat import os
 import certbot.tests.util as test_util
 
+try:
+    import mock
+except ImportError:  # pragma: no cover
+    from unittest import mock
+
 
 KEY = test_util.load_vector("rsa512_key.pem")
 CSR_SAN = test_util.load_vector("csr-san_512.pem")
 
 # pylint: disable=line-too-long
 
+
 class DetermineUserAgentTest(test_util.ConfigTestCase):
     """Tests for certbot._internal.client.determine_user_agent."""
 
@@ -62,6 +66,8 @@ class RegisterTest(test_util.ConfigTestCase):
         self.config.register_unsafely_without_email = False
         self.config.email = "alias@example.com"
         self.account_storage = account.AccountMemoryStorage()
+        with mock.patch("zope.component.provideUtility"):
+            display_obj.set_display(MagicMock())
 
     def _call(self):
         from certbot._internal.client import register
@@ -99,7 +105,7 @@ class RegisterTest(test_util.ConfigTestCase):
                 self._call()
                 self.assertIs(mock_prepare.called, True)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_it(self, unused_mock_get_utility):
         with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client:
             mock_client().external_account_required.side_effect = self._false_mock
@@ -160,7 +166,7 @@ class RegisterTest(test_util.ConfigTestCase):
                 # check Certbot created an account with no email. Contact should return empty
                 self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_with_eab_arguments(self, unused_mock_get_utility):
         with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client:
             mock_client().client.directory.__getitem__ = mock.Mock(
@@ -176,7 +182,7 @@ class RegisterTest(test_util.ConfigTestCase):
 
                     self.assertIs(mock_eab_from_data.called, True)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_without_eab_arguments(self, unused_mock_get_utility):
         with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client:
             mock_client().external_account_required.side_effect = self._false_mock
@@ -409,7 +415,7 @@ class ClientTest(ClientTestCommon):
         # Certificate should get issued despite one failed deactivation
         self.eg_order.authorizations = authzrs
         self.client.auth_handler.handle_authorizations.return_value = authzrs
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             result = self.client.obtain_certificate(self.eg_domains)
         self.assertEqual(result, (mock.sentinel.cert, mock.sentinel.chain, key, csr))
         self._check_obtain_certificate(1)
@@ -453,7 +459,7 @@ class ClientTest(ClientTestCommon):
         self.eg_order.authorizations = authzr
         self.client.auth_handler.handle_authorizations.return_value = authzr
 
-        with test_util.patch_get_utility():
+        with test_util.patch_display_util():
             result = self.client.obtain_certificate(self.eg_domains)
 
         self.assertEqual(
@@ -519,7 +525,7 @@ class ClientTest(ClientTestCommon):
 
         shutil.rmtree(tmp_path)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_deploy_certificate_success(self, mock_util):
         self.assertRaises(errors.Error, self.client.deploy_certificate,
                           ["foo.bar"], "key", "cert", "chain", "fullchain")
@@ -538,7 +544,7 @@ class ClientTest(ClientTestCommon):
         installer.restart.assert_called_once_with()
 
     @mock.patch('certbot._internal.client.display_util.notify')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_deploy_certificate_failure(self, mock_util, mock_notify):
         installer = mock.MagicMock()
         self.client.installer = installer
@@ -552,7 +558,7 @@ class ClientTest(ClientTestCommon):
         mock_notify.assert_any_call('Deploying certificate')
 
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_deploy_certificate_save_failure(self, mock_util):
         installer = mock.MagicMock()
         self.client.installer = installer
@@ -563,7 +569,7 @@ class ClientTest(ClientTestCommon):
         installer.recovery_routine.assert_called_once_with()
 
     @mock.patch('certbot._internal.client.display_util.notify')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_deploy_certificate_restart_failure(self, mock_get_utility, mock_notify):
         installer = mock.MagicMock()
         installer.restart.side_effect = [errors.PluginError, None]
@@ -578,7 +584,7 @@ class ClientTest(ClientTestCommon):
         self.assertEqual(installer.restart.call_count, 2)
 
     @mock.patch('certbot._internal.client.logger')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_deploy_certificate_restart_failure2(self, mock_get_utility, mock_logger):
         installer = mock.MagicMock()
         installer.restart.side_effect = errors.PluginError
@@ -706,7 +712,7 @@ class EnhanceConfigTest(ClientTestCommon):
     def _test_error(self, enhance_error=False, restart_error=False):
         self.config.redirect = True
         with mock.patch('certbot._internal.client.logger') as mock_logger, \
-             test_util.patch_get_utility() as mock_gu:
+             test_util.patch_display_util() as mock_gu:
             self.assertRaises(
                 errors.PluginError, self._test_with_all_supported)
 

certbot/tests/display/obj_test.py

@@ -0,0 +1,373 @@
+"""Test :mod:`certbot._internal.display.obj`."""
+import inspect
+import unittest
+from unittest import mock
+
+from certbot import errors, interfaces
+from certbot._internal.display import obj as display_obj
+from certbot.display import util as display_util
+
+CHOICES = [("First", "Description1"), ("Second", "Description2")]
+TAGS = ["tag1", "tag2", "tag3"]
+
+
+class FileOutputDisplayTest(unittest.TestCase):
+    """Test stdout display.
+
+    Most of this class has to deal with visual output.  In order to test how the
+    functions look to a user, uncomment the test_visual function.
+
+    """
+    def setUp(self):
+        super().setUp()
+        self.mock_stdout = mock.MagicMock()
+        self.displayer = display_obj.FileDisplay(self.mock_stdout, False)
+
+    @mock.patch("certbot._internal.display.obj.logger")
+    def test_notification_no_pause(self, mock_logger):
+        self.displayer.notification("message", False)
+        string = self.mock_stdout.write.call_args[0][0]
+
+        self.assertIn("message", string)
+        mock_logger.debug.assert_called_with("Notifying user: %s", "message")
+
+    def test_notification_pause(self):
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        with mock.patch(input_with_timeout, return_value="enter"):
+            self.displayer.notification("message", force_interactive=True)
+
+        self.assertIn("message", self.mock_stdout.write.call_args[0][0])
+
+    def test_notification_noninteractive(self):
+        self._force_noninteractive(self.displayer.notification, "message")
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertIn("message", string)
+
+    def test_notification_noninteractive2(self):
+        # The main purpose of this test is to make sure we only call
+        # logger.warning once which _force_noninteractive checks internally
+        self._force_noninteractive(self.displayer.notification, "message")
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertIn("message", string)
+
+        self.assertTrue(self.displayer.skipped_interaction)
+
+        self._force_noninteractive(self.displayer.notification, "message2")
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertIn("message2", string)
+
+    def test_notification_decoration(self):
+        from certbot.compat import os
+        self.displayer.notification("message", pause=False, decorate=False)
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertEqual(string, "message" + os.linesep)
+
+        self.displayer.notification("message2", pause=False)
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertIn("- - - ", string)
+        self.assertIn("message2" + os.linesep, string)
+
+    @mock.patch("certbot.display.util."
+                "FileDisplay._get_valid_int_ans")
+    def test_menu(self, mock_ans):
+        mock_ans.return_value = (display_util.OK, 1)
+        ret = self.displayer.menu("message", CHOICES, force_interactive=True)
+        self.assertEqual(ret, (display_util.OK, 0))
+
+    def test_menu_noninteractive(self):
+        default = 0
+        result = self._force_noninteractive(
+            self.displayer.menu, "msg", CHOICES, default=default)
+        self.assertEqual(result, (display_util.OK, default))
+
+    def test_input_cancel(self):
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        with mock.patch(input_with_timeout, return_value="c"):
+            code, _ = self.displayer.input("message", force_interactive=True)
+
+        self.assertTrue(code, display_util.CANCEL)
+
+    def test_input_normal(self):
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        with mock.patch(input_with_timeout, return_value="domain.com"):
+            code, input_ = self.displayer.input("message", force_interactive=True)
+
+        self.assertEqual(code, display_util.OK)
+        self.assertEqual(input_, "domain.com")
+
+    def test_input_noninteractive(self):
+        default = "foo"
+        code, input_ = self._force_noninteractive(
+            self.displayer.input, "message", default=default)
+
+        self.assertEqual(code, display_util.OK)
+        self.assertEqual(input_, default)
+
+    def test_input_assertion_fail(self):
+        # If the call to util.assert_valid_call is commented out, an
+        # error.Error is raised, otherwise, an AssertionError is raised.
+        self.assertRaises(Exception, self._force_noninteractive,
+                          self.displayer.input, "message", cli_flag="--flag")
+
+    def test_input_assertion_fail2(self):
+        with mock.patch("certbot.display.util.assert_valid_call"):
+            self.assertRaises(errors.Error, self._force_noninteractive,
+                              self.displayer.input, "msg", cli_flag="--flag")
+
+    def test_yesno(self):
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        with mock.patch(input_with_timeout, return_value="Yes"):
+            self.assertTrue(self.displayer.yesno(
+                "message", force_interactive=True))
+        with mock.patch(input_with_timeout, return_value="y"):
+            self.assertTrue(self.displayer.yesno(
+                "message", force_interactive=True))
+        with mock.patch(input_with_timeout, side_effect=["maybe", "y"]):
+            self.assertTrue(self.displayer.yesno(
+                "message", force_interactive=True))
+        with mock.patch(input_with_timeout, return_value="No"):
+            self.assertFalse(self.displayer.yesno(
+                "message", force_interactive=True))
+        with mock.patch(input_with_timeout, side_effect=["cancel", "n"]):
+            self.assertFalse(self.displayer.yesno(
+                "message", force_interactive=True))
+
+        with mock.patch(input_with_timeout, return_value="a"):
+            self.assertTrue(self.displayer.yesno(
+                "msg", yes_label="Agree", force_interactive=True))
+
+    def test_yesno_noninteractive(self):
+        self.assertTrue(self._force_noninteractive(
+            self.displayer.yesno, "message", default=True))
+
+    @mock.patch("certbot.display.util.input_with_timeout")
+    def test_checklist_valid(self, mock_input):
+        mock_input.return_value = "2 1"
+        code, tag_list = self.displayer.checklist(
+            "msg", TAGS, force_interactive=True)
+        self.assertEqual(
+            (code, set(tag_list)), (display_util.OK, {"tag1", "tag2"}))
+
+    @mock.patch("certbot.display.util.input_with_timeout")
+    def test_checklist_empty(self, mock_input):
+        mock_input.return_value = ""
+        code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True)
+        self.assertEqual(
+            (code, set(tag_list)), (display_util.OK, {"tag1", "tag2", "tag3"}))
+
+    @mock.patch("certbot.display.util.input_with_timeout")
+    def test_checklist_miss_valid(self, mock_input):
+        mock_input.side_effect = ["10", "tag1 please", "1"]
+
+        ret = self.displayer.checklist("msg", TAGS, force_interactive=True)
+        self.assertEqual(ret, (display_util.OK, ["tag1"]))
+
+    @mock.patch("certbot.display.util.input_with_timeout")
+    def test_checklist_miss_quit(self, mock_input):
+        mock_input.side_effect = ["10", "c"]
+
+        ret = self.displayer.checklist("msg", TAGS, force_interactive=True)
+        self.assertEqual(ret, (display_util.CANCEL, []))
+
+    def test_checklist_noninteractive(self):
+        default = TAGS
+        code, input_ = self._force_noninteractive(
+            self.displayer.checklist, "msg", TAGS, default=default)
+
+        self.assertEqual(code, display_util.OK)
+        self.assertEqual(input_, default)
+
+    def test_scrub_checklist_input_valid(self):
+        # pylint: disable=protected-access
+        indices = [
+            ["1"],
+            ["1", "2", "1"],
+            ["2", "3"],
+        ]
+        exp = [
+            {"tag1"},
+            {"tag1", "tag2"},
+            {"tag2", "tag3"},
+        ]
+        for i, list_ in enumerate(indices):
+            set_tags = set(
+                self.displayer._scrub_checklist_input(list_, TAGS))
+            self.assertEqual(set_tags, exp[i])
+
+    @mock.patch("certbot.display.util.input_with_timeout")
+    def test_directory_select(self, mock_input):
+        args = ["msg", "/var/www/html", "--flag", True]
+        user_input = "/var/www/html"
+        mock_input.return_value = user_input
+
+        returned = self.displayer.directory_select(*args)
+        self.assertEqual(returned, (display_util.OK, user_input))
+
+    def test_directory_select_noninteractive(self):
+        default = "/var/www/html"
+        code, input_ = self._force_noninteractive(
+            self.displayer.directory_select, "msg", default=default)
+
+        self.assertEqual(code, display_util.OK)
+        self.assertEqual(input_, default)
+
+    def _force_noninteractive(self, func, *args, **kwargs):
+        skipped_interaction = self.displayer.skipped_interaction
+
+        with mock.patch("certbot.display.util.sys.stdin") as mock_stdin:
+            mock_stdin.isatty.return_value = False
+            with mock.patch("certbot._internal.display.obj.logger") as mock_logger:
+                result = func(*args, **kwargs)
+
+        if skipped_interaction:
+            self.assertIs(mock_logger.warning.called, False)
+        else:
+            self.assertEqual(mock_logger.warning.call_count, 1)
+
+        return result
+
+    def test_scrub_checklist_input_invalid(self):
+        # pylint: disable=protected-access
+        indices = [
+            ["0"],
+            ["4"],
+            ["tag1"],
+            ["1", "tag1"],
+            ["2", "o"]
+        ]
+        for list_ in indices:
+            self.assertEqual(
+                self.displayer._scrub_checklist_input(list_, TAGS), [])
+
+    def test_print_menu(self):
+        # pylint: disable=protected-access
+        # This is purely cosmetic... just make sure there aren't any exceptions
+        self.displayer._print_menu("msg", CHOICES)
+        self.displayer._print_menu("msg", TAGS)
+
+    def test_wrap_lines(self):
+        # pylint: disable=protected-access
+        msg = ("This is just a weak test{0}"
+               "This function is only meant to be for easy viewing{0}"
+               "Test a really really really really really really really really "
+               "really really really really long line...".format('\n'))
+        text = display_obj._wrap_lines(msg)
+
+        self.assertEqual(text.count('\n'), 3)
+
+    def test_get_valid_int_ans_valid(self):
+        # pylint: disable=protected-access
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        with mock.patch(input_with_timeout, return_value="1"):
+            self.assertEqual(
+                self.displayer._get_valid_int_ans(1), (display_util.OK, 1))
+        ans = "2"
+        with mock.patch(input_with_timeout, return_value=ans):
+            self.assertEqual(
+                self.displayer._get_valid_int_ans(3),
+                (display_util.OK, int(ans)))
+
+    def test_get_valid_int_ans_invalid(self):
+        # pylint: disable=protected-access
+        answers = [
+            ["0", "c"],
+            ["4", "one", "C"],
+            ["c"],
+        ]
+        input_with_timeout = "certbot.display.util.input_with_timeout"
+        for ans in answers:
+            with mock.patch(input_with_timeout, side_effect=ans):
+                self.assertEqual(
+                    self.displayer._get_valid_int_ans(3),
+                    (display_util.CANCEL, -1))
+
+    def test_methods_take_force_interactive(self):
+        # Every IDisplay method implemented by FileDisplay must take
+        # force_interactive to prevent workflow regressions.
+        for name in interfaces.IDisplay.names():
+            arg_spec = inspect.getfullargspec(getattr(self.displayer, name))
+            self.assertIn("force_interactive", arg_spec.args)
+
+
+class NoninteractiveDisplayTest(unittest.TestCase):
+    """Test non-interactive display. These tests are pretty easy!"""
+    def setUp(self):
+        self.mock_stdout = mock.MagicMock()
+        self.displayer = display_util.NoninteractiveDisplay(self.mock_stdout)
+
+    @mock.patch("certbot._internal.display.obj.logger")
+    def test_notification_no_pause(self, mock_logger):
+        self.displayer.notification("message", 10)
+        string = self.mock_stdout.write.call_args[0][0]
+
+        self.assertIn("message", string)
+        mock_logger.debug.assert_called_with("Notifying user: %s", "message")
+
+    def test_notification_decoration(self):
+        from certbot.compat import os
+        self.displayer.notification("message", pause=False, decorate=False)
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertEqual(string, "message" + os.linesep)
+
+        self.displayer.notification("message2", pause=False)
+        string = self.mock_stdout.write.call_args[0][0]
+        self.assertTrue("- - - " in string and ("message2" + os.linesep) in string)
+
+    def test_input(self):
+        d = "an incomputable value"
+        ret = self.displayer.input("message", default=d)
+        self.assertEqual(ret, (display_util.OK, d))
+        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.input, "message")
+
+    def test_menu(self):
+        ret = self.displayer.menu("message", CHOICES, default=1)
+        self.assertEqual(ret, (display_util.OK, 1))
+        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.menu, "message", CHOICES)
+
+    def test_yesno(self):
+        d = False
+        ret = self.displayer.yesno("message", default=d)
+        self.assertEqual(ret, d)
+        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.yesno, "message")
+
+    def test_checklist(self):
+        d = [1, 3]
+        ret = self.displayer.checklist("message", TAGS, default=d)
+        self.assertEqual(ret, (display_util.OK, d))
+        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.checklist, "message", TAGS)
+
+    def test_directory_select(self):
+        default = "/var/www/html"
+        expected = (display_util.OK, default)
+        actual = self.displayer.directory_select("msg", default)
+        self.assertEqual(expected, actual)
+
+        self.assertRaises(
+            errors.MissingCommandlineFlag, self.displayer.directory_select, "msg")
+
+    def test_methods_take_kwargs(self):
+        # Every IDisplay method implemented by NoninteractiveDisplay
+        # should take **kwargs because every method of FileDisplay must
+        # take force_interactive which doesn't apply to
+        # NoninteractiveDisplay.
+
+        # Use pylint code for disable to keep on single line under line length limit
+        for name in interfaces.IDisplay.names():  # pylint: disable=E1120
+            method = getattr(self.displayer, name)
+            # asserts method accepts arbitrary keyword arguments
+            result = inspect.getfullargspec(method).varkw
+            self.assertIsNotNone(result)
+
+
+class PlaceParensTest(unittest.TestCase):
+    @classmethod
+    def _call(cls, label):  # pylint: disable=protected-access
+        from certbot._internal.display.obj import _parens_around_char
+        return _parens_around_char(label)
+
+    def test_single_letter(self):
+        self.assertEqual("(a)", self._call("a"))
+
+    def test_multiple(self):
+        self.assertEqual("(L)abel", self._call("Label"))
+        self.assertEqual("(y)es please", self._call("yes please"))

certbot/tests/display/ops_test.py

@@ -4,14 +4,10 @@ import sys
 import unittest
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock
-import zope.component
 
 from acme import messages
 from certbot import errors
+from certbot._internal.display import obj as display_obj
 from certbot._internal import account
 from certbot.compat import filesystem
 from certbot.compat import os
@@ -19,6 +15,12 @@ from certbot.display import ops
 from certbot.display import util as display_util
 import certbot.tests.util as test_util
 
+try:
+    import mock
+except ImportError:  # pragma: no cover
+    from unittest import mock
+
+
 KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
 
 
@@ -30,14 +32,14 @@ class GetEmailTest(unittest.TestCase):
         from certbot.display.ops import get_email
         return get_email(**kwargs)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_cancel_none(self, mock_get_utility):
         mock_input = mock_get_utility().input
         mock_input.return_value = (display_util.CANCEL, "foo@bar.baz")
         self.assertRaises(errors.Error, self._call)
         self.assertRaises(errors.Error, self._call, optional=False)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_ok_safe(self, mock_get_utility):
         mock_input = mock_get_utility().input
         mock_input.return_value = (display_util.OK, "foo@bar.baz")
@@ -45,7 +47,7 @@ class GetEmailTest(unittest.TestCase):
             mock_safe_email.return_value = True
             self.assertEqual(self._call(), "foo@bar.baz")
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_ok_not_safe(self, mock_get_utility):
         mock_input = mock_get_utility().input
         mock_input.return_value = (display_util.OK, "foo@bar.baz")
@@ -53,7 +55,7 @@ class GetEmailTest(unittest.TestCase):
             mock_safe_email.side_effect = [False, True]
             self.assertEqual(self._call(), "foo@bar.baz")
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_invalid_flag(self, mock_get_utility):
         invalid_txt = "There seem to be problems"
         mock_input = mock_get_utility().input
@@ -65,7 +67,7 @@ class GetEmailTest(unittest.TestCase):
             self._call(invalid=True)
             self.assertIn(invalid_txt, mock_input.call_args[0][0])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_optional_flag(self, mock_get_utility):
         mock_input = mock_get_utility().input
         mock_input.return_value = (display_util.OK, "foo@bar.baz")
@@ -75,7 +77,7 @@ class GetEmailTest(unittest.TestCase):
             for call in mock_input.call_args_list:
                 self.assertNotIn("--register-unsafely-without-email", call[0][0])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_optional_invalid_unsafe(self, mock_get_utility):
         invalid_txt = "There seem to be problems"
         mock_input = mock_get_utility().input
@@ -91,8 +93,7 @@ class ChooseAccountTest(test_util.TempDirTestCase):
     def setUp(self):
         super().setUp()
 
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout,
-                                                               False))
+        display_obj.set_display(display_obj.FileDisplay(sys.stdout, False))
 
         self.account_keys_dir = os.path.join(self.tempdir, "keys")
         filesystem.makedirs(self.account_keys_dir, 0o700)
@@ -114,17 +115,17 @@ class ChooseAccountTest(test_util.TempDirTestCase):
     def _call(cls, accounts):
         return ops.choose_account(accounts)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_one(self, mock_util):
         mock_util().menu.return_value = (display_util.OK, 0)
         self.assertEqual(self._call([self.acc1]), self.acc1)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_two(self, mock_util):
         mock_util().menu.return_value = (display_util.OK, 1)
         self.assertEqual(self._call([self.acc1, self.acc2]), self.acc2)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_cancel(self, mock_util):
         mock_util().menu.return_value = (display_util.CANCEL, 1)
         self.assertIsNone(self._call([self.acc1, self.acc2]))
@@ -133,8 +134,7 @@ class ChooseAccountTest(test_util.TempDirTestCase):
 class GenHttpsNamesTest(unittest.TestCase):
     """Test _gen_https_names."""
     def setUp(self):
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout,
-                                                               False))
+        display_obj.set_display(display_obj.FileDisplay(sys.stdout, False))
 
     @classmethod
     def _call(cls, domains):
@@ -181,8 +181,7 @@ class GenHttpsNamesTest(unittest.TestCase):
 class ChooseNamesTest(unittest.TestCase):
     """Test choose names."""
     def setUp(self):
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout,
-                                                               False))
+        display_obj.set_display(display_obj.FileDisplay(sys.stdout, False))
         self.mock_install = mock.MagicMock()
 
     @classmethod
@@ -195,12 +194,12 @@ class ChooseNamesTest(unittest.TestCase):
         self._call(None)
         self.assertEqual(mock_manual.call_count, 1)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_no_installer_cancel(self, mock_util):
         mock_util().input.return_value = (display_util.CANCEL, [])
         self.assertEqual(self._call(None), [])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_no_names_choose(self, mock_util):
         self.mock_install().get_all_names.return_value = set()
         domain = "example.com"
@@ -249,7 +248,7 @@ class ChooseNamesTest(unittest.TestCase):
         self.assertEqual(_sort_names(to_sort), sortd)
 
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_filter_names_valid_return(self, mock_util):
         self.mock_install.get_all_names.return_value = {"example.com"}
         mock_util().checklist.return_value = (display_util.OK, ["example.com"])
@@ -258,7 +257,7 @@ class ChooseNamesTest(unittest.TestCase):
         self.assertEqual(names, ["example.com"])
         self.assertEqual(mock_util().checklist.call_count, 1)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_filter_namees_override_question(self, mock_util):
         self.mock_install.get_all_names.return_value = {"example.com"}
         mock_util().checklist.return_value = (display_util.OK, ["example.com"])
@@ -267,14 +266,14 @@ class ChooseNamesTest(unittest.TestCase):
         self.assertEqual(mock_util().checklist.call_count, 1)
         self.assertEqual(mock_util().checklist.call_args[0][0], "Custom")
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_filter_names_nothing_selected(self, mock_util):
         self.mock_install.get_all_names.return_value = {"example.com"}
         mock_util().checklist.return_value = (display_util.OK, [])
 
         self.assertEqual(self._call(self.mock_install), [])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_filter_names_cancel(self, mock_util):
         self.mock_install.get_all_names.return_value = {"example.com"}
         mock_util().checklist.return_value = (
@@ -293,7 +292,7 @@ class ChooseNamesTest(unittest.TestCase):
         self.assertEqual(get_valid_domains(all_invalid), [])
         self.assertEqual(len(get_valid_domains(two_valid)), 2)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_choose_manually(self, mock_util):
         from certbot.display.ops import _choose_names_manually
         utility_mock = mock_util()
@@ -320,7 +319,7 @@ class ChooseNamesTest(unittest.TestCase):
                          ["example.com", "under_score.example.com",
                           "justtld", "valid.example.com"])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_choose_manually_retry(self, mock_util):
         from certbot.display.ops import _choose_names_manually
         utility_mock = mock_util()
@@ -339,10 +338,10 @@ class SuccessInstallationTest(unittest.TestCase):
         from certbot.display.ops import success_installation
         success_installation(names)
 
-    @test_util.patch_get_utility("certbot.display.util.notify")
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
-    def test_success_installation(self, mock_util, mock_notify):
-        mock_util().notification.return_value = None
+    @test_util.patch_display_util()
+    @mock.patch("certbot.display.util.notify")
+    def test_success_installation(self, mock_notify, mock_display):
+        mock_display().notification.return_value = None
         names = ["example.com", "abc.com"]
 
         self._call(names)
@@ -361,16 +360,17 @@ class SuccessRenewalTest(unittest.TestCase):
         from certbot.display.ops import success_renewal
         success_renewal(names)
 
-    @test_util.patch_get_utility("certbot.display.util.notify")
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
-    def test_success_renewal(self, mock_util, mock_notify):
-        mock_util().notification.return_value = None
+    @test_util.patch_display_util()
+    @mock.patch("certbot.display.util.notify")
+    def test_success_renewal(self, mock_notify, mock_display):
+        mock_display().notification.return_value = None
         names = ["example.com", "abc.com"]
 
         self._call(names)
 
         self.assertEqual(mock_notify.call_count, 1)
 
+
 class SuccessRevocationTest(unittest.TestCase):
     """Test the success revocation message."""
     @classmethod
@@ -378,9 +378,9 @@ class SuccessRevocationTest(unittest.TestCase):
         from certbot.display.ops import success_revocation
         success_revocation(path)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     @mock.patch("certbot.display.util.notify")
-    def test_success_revocation(self, mock_notify, unused_mock_util):
+    def test_success_revocation(self, mock_notify, unused_mock_display):
         path = "/path/to/cert.pem"
         self._call(path)
         mock_notify.assert_called_once_with(
@@ -402,7 +402,7 @@ class ValidatorTests(unittest.TestCase):
         if m == "":
             raise errors.PluginError(ValidatorTests.__ERROR)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_input_blank_with_validator(self, mock_util):
         mock_util().input.side_effect = [(display_util.OK, ""),
                                          (display_util.OK, ""),
@@ -413,14 +413,14 @@ class ValidatorTests(unittest.TestCase):
         self.assertEqual(ValidatorTests.__ERROR, mock_util().notification.call_args[0][0])
         self.assertEqual(returned, (display_util.OK, self.valid_input))
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_input_validation_with_default(self, mock_util):
         mock_util().input.side_effect = [(display_util.OK, self.valid_input)]
 
         returned = ops.validated_input(self.__validator, "msg", default="other")
         self.assertEqual(returned, (display_util.OK, self.valid_input))
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_input_validation_with_bad_default(self, mock_util):
         mock_util().input.side_effect = [(display_util.OK, self.valid_input)]
 
@@ -428,14 +428,14 @@ class ValidatorTests(unittest.TestCase):
                           ops.validated_input,
                           self.__validator, "msg", default="")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_input_cancel_with_validator(self, mock_util):
         mock_util().input.side_effect = [(display_util.CANCEL, "")]
 
         code, unused_raw = ops.validated_input(self.__validator, "message", force_interactive=True)
         self.assertEqual(code, display_util.CANCEL)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_directory_select_validation(self, mock_util):
         mock_util().directory_select.side_effect = [(display_util.OK, ""),
                                                     (display_util.OK, self.valid_directory)]
@@ -444,14 +444,14 @@ class ValidatorTests(unittest.TestCase):
         self.assertEqual(ValidatorTests.__ERROR, mock_util().notification.call_args[0][0])
         self.assertEqual(returned, (display_util.OK, self.valid_directory))
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_directory_select_validation_with_default(self, mock_util):
         mock_util().directory_select.side_effect = [(display_util.OK, self.valid_directory)]
 
         returned = ops.validated_directory(self.__validator, "msg", default="other")
         self.assertEqual(returned, (display_util.OK, self.valid_directory))
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_directory_select_validation_with_bad_default(self, mock_util):
         mock_util().directory_select.side_effect = [(display_util.OK, self.valid_directory)]
 
@@ -467,7 +467,7 @@ class ChooseValuesTest(unittest.TestCase):
         from certbot.display.ops import choose_values
         return choose_values(values, question)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_choose_names_success(self, mock_util):
         items = ["first", "second", "third"]
         mock_util().checklist.return_value = (display_util.OK, [items[2]])
@@ -476,7 +476,7 @@ class ChooseValuesTest(unittest.TestCase):
         self.assertIs(mock_util().checklist.called, True)
         self.assertIsNone(mock_util().checklist.call_args[0][0])
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_choose_names_success_question(self, mock_util):
         items = ["first", "second", "third"]
         question = "Which one?"
@@ -486,7 +486,7 @@ class ChooseValuesTest(unittest.TestCase):
         self.assertIs(mock_util().checklist.called, True)
         self.assertEqual(mock_util().checklist.call_args[0][0], question)
 
-    @test_util.patch_get_utility("certbot.display.ops.z_util")
+    @test_util.patch_display_util()
     def test_choose_names_user_cancel(self, mock_util):
         items = ["first", "second", "third"]
         question = "Want to cancel?"

certbot/tests/display/util_test.py

@@ -1,25 +1,101 @@
 """Test :mod:`certbot.display.util`."""
-import inspect
 import io
 import socket
 import tempfile
 import unittest
 
-
 from certbot import errors
-from certbot import interfaces
-from certbot.display import util as display_util
 import certbot.tests.util as test_util
 
 try:
     import mock
-except ImportError: # pragma: no cover
+except ImportError:  # pragma: no cover
     from unittest import mock
 
 
-CHOICES = [("First", "Description1"), ("Second", "Description2")]
-TAGS = ["tag1", "tag2", "tag3"]
-TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")]
+class NotifyTest(unittest.TestCase):
+    """Tests for certbot.display.util.notify"""
+
+    @test_util.patch_display_util()
+    def test_notify(self, mock_util):
+        from certbot.display.util import notify
+        notify("Hello World")
+        mock_util().notification.assert_called_with(
+            "Hello World", pause=False, decorate=False, wrap=False
+        )
+
+
+class NotificationTest(unittest.TestCase):
+    """Tests for certbot.display.util.notification"""
+
+    @test_util.patch_display_util()
+    def test_notification(self, mock_util):
+        from certbot.display.util import notification
+        notification("Hello World")
+        mock_util().notification.assert_called_with(
+            "Hello World", pause=True, decorate=True, wrap=True, force_interactive=False
+        )
+
+
+class MenuTest(unittest.TestCase):
+    """Tests for certbot.display.util.menu"""
+
+    @test_util.patch_display_util()
+    def test_menu(self, mock_util):
+        from certbot.display.util import menu
+        menu("Hello World", ["one", "two"], default=0)
+        mock_util().menu.assert_called_with(
+            "Hello World", ["one", "two"], default=0, cli_flag=None, force_interactive=False
+        )
+
+
+class InputTextTest(unittest.TestCase):
+    """Tests for certbot.display.util.input_text"""
+
+    @test_util.patch_display_util()
+    def test_input_text(self, mock_util):
+        from certbot.display.util import input_text
+        input_text("Hello World", default="something")
+        mock_util().input.assert_called_with(
+            "Hello World", default='something', cli_flag=None, force_interactive=False
+        )
+
+
+class YesNoTest(unittest.TestCase):
+    """Tests for certbot.display.util.yesno"""
+
+    @test_util.patch_display_util()
+    def test_yesno(self, mock_util):
+        from certbot.display.util import yesno
+        yesno("Hello World", default=True)
+        mock_util().yesno.assert_called_with(
+            "Hello World", yes_label='Yes', no_label='No', default=True, cli_flag=None,
+            force_interactive=False
+        )
+
+
+class ChecklistTest(unittest.TestCase):
+    """Tests for certbot.display.util.checklist"""
+
+    @test_util.patch_display_util()
+    def test_checklist(self, mock_util):
+        from certbot.display.util import checklist
+        checklist("Hello World", ["one", "two"], default="one")
+        mock_util().checklist.assert_called_with(
+            "Hello World", ['one', 'two'], default='one', cli_flag=None, force_interactive=False
+        )
+
+
+class DirectorySelectTest(unittest.TestCase):
+    """Tests for certbot.display.util.directory_select"""
+
+    @test_util.patch_display_util()
+    def test_directory_select(self, mock_util):
+        from certbot.display.util import directory_select
+        directory_select("Hello World", default="something")
+        mock_util().directory_select.assert_called_with(
+            "Hello World", default='something', cli_flag=None, force_interactive=False
+        )
 
 
 class InputWithTimeoutTest(unittest.TestCase):
@@ -57,354 +133,6 @@ class InputWithTimeoutTest(unittest.TestCase):
         stdin.close()
 
 
-class FileOutputDisplayTest(unittest.TestCase):
-    """Test stdout display.
-
-    Most of this class has to deal with visual output.  In order to test how the
-    functions look to a user, uncomment the test_visual function.
-
-    """
-    def setUp(self):
-        super().setUp()
-        self.mock_stdout = mock.MagicMock()
-        self.displayer = display_util.FileDisplay(self.mock_stdout, False)
-
-    @mock.patch("certbot.display.util.logger")
-    def test_notification_no_pause(self, mock_logger):
-        self.displayer.notification("message", False)
-        string = self.mock_stdout.write.call_args[0][0]
-
-        self.assertIn("message", string)
-        mock_logger.debug.assert_called_with("Notifying user: %s", "message")
-
-    def test_notification_pause(self):
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        with mock.patch(input_with_timeout, return_value="enter"):
-            self.displayer.notification("message", force_interactive=True)
-
-        self.assertIn("message", self.mock_stdout.write.call_args[0][0])
-
-    def test_notification_noninteractive(self):
-        self._force_noninteractive(self.displayer.notification, "message")
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertIn("message", string)
-
-    def test_notification_noninteractive2(self):
-        # The main purpose of this test is to make sure we only call
-        # logger.warning once which _force_noninteractive checks internally
-        self._force_noninteractive(self.displayer.notification, "message")
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertIn("message", string)
-
-        self.assertTrue(self.displayer.skipped_interaction)
-
-        self._force_noninteractive(self.displayer.notification, "message2")
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertIn("message2", string)
-
-    def test_notification_decoration(self):
-        from certbot.compat import os
-        self.displayer.notification("message", pause=False, decorate=False)
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertEqual(string, "message" + os.linesep)
-
-        self.displayer.notification("message2", pause=False)
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertIn("- - - ", string)
-        self.assertIn("message2" + os.linesep, string)
-
-    @mock.patch("certbot.display.util."
-                "FileDisplay._get_valid_int_ans")
-    def test_menu(self, mock_ans):
-        mock_ans.return_value = (display_util.OK, 1)
-        ret = self.displayer.menu("message", CHOICES, force_interactive=True)
-        self.assertEqual(ret, (display_util.OK, 0))
-
-    def test_menu_noninteractive(self):
-        default = 0
-        result = self._force_noninteractive(
-            self.displayer.menu, "msg", CHOICES, default=default)
-        self.assertEqual(result, (display_util.OK, default))
-
-    def test_input_cancel(self):
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        with mock.patch(input_with_timeout, return_value="c"):
-            code, _ = self.displayer.input("message", force_interactive=True)
-
-        self.assertTrue(code, display_util.CANCEL)
-
-    def test_input_normal(self):
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        with mock.patch(input_with_timeout, return_value="domain.com"):
-            code, input_ = self.displayer.input("message", force_interactive=True)
-
-        self.assertEqual(code, display_util.OK)
-        self.assertEqual(input_, "domain.com")
-
-    def test_input_noninteractive(self):
-        default = "foo"
-        code, input_ = self._force_noninteractive(
-            self.displayer.input, "message", default=default)
-
-        self.assertEqual(code, display_util.OK)
-        self.assertEqual(input_, default)
-
-    def test_input_assertion_fail(self):
-        # If the call to util.assert_valid_call is commented out, an
-        # error.Error is raised, otherwise, an AssertionError is raised.
-        self.assertRaises(Exception, self._force_noninteractive,
-                          self.displayer.input, "message", cli_flag="--flag")
-
-    def test_input_assertion_fail2(self):
-        with mock.patch("certbot.display.util.assert_valid_call"):
-            self.assertRaises(errors.Error, self._force_noninteractive,
-                              self.displayer.input, "msg", cli_flag="--flag")
-
-    def test_yesno(self):
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        with mock.patch(input_with_timeout, return_value="Yes"):
-            self.assertTrue(self.displayer.yesno(
-                "message", force_interactive=True))
-        with mock.patch(input_with_timeout, return_value="y"):
-            self.assertTrue(self.displayer.yesno(
-                "message", force_interactive=True))
-        with mock.patch(input_with_timeout, side_effect=["maybe", "y"]):
-            self.assertTrue(self.displayer.yesno(
-                "message", force_interactive=True))
-        with mock.patch(input_with_timeout, return_value="No"):
-            self.assertFalse(self.displayer.yesno(
-                "message", force_interactive=True))
-        with mock.patch(input_with_timeout, side_effect=["cancel", "n"]):
-            self.assertFalse(self.displayer.yesno(
-                "message", force_interactive=True))
-
-        with mock.patch(input_with_timeout, return_value="a"):
-            self.assertTrue(self.displayer.yesno(
-                "msg", yes_label="Agree", force_interactive=True))
-
-    def test_yesno_noninteractive(self):
-        self.assertTrue(self._force_noninteractive(
-            self.displayer.yesno, "message", default=True))
-
-    @mock.patch("certbot.display.util.input_with_timeout")
-    def test_checklist_valid(self, mock_input):
-        mock_input.return_value = "2 1"
-        code, tag_list = self.displayer.checklist(
-            "msg", TAGS, force_interactive=True)
-        self.assertEqual(
-            (code, set(tag_list)), (display_util.OK, {"tag1", "tag2"}))
-
-    @mock.patch("certbot.display.util.input_with_timeout")
-    def test_checklist_empty(self, mock_input):
-        mock_input.return_value = ""
-        code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True)
-        self.assertEqual(
-            (code, set(tag_list)), (display_util.OK, {"tag1", "tag2", "tag3"}))
-
-    @mock.patch("certbot.display.util.input_with_timeout")
-    def test_checklist_miss_valid(self, mock_input):
-        mock_input.side_effect = ["10", "tag1 please", "1"]
-
-        ret = self.displayer.checklist("msg", TAGS, force_interactive=True)
-        self.assertEqual(ret, (display_util.OK, ["tag1"]))
-
-    @mock.patch("certbot.display.util.input_with_timeout")
-    def test_checklist_miss_quit(self, mock_input):
-        mock_input.side_effect = ["10", "c"]
-
-        ret = self.displayer.checklist("msg", TAGS, force_interactive=True)
-        self.assertEqual(ret, (display_util.CANCEL, []))
-
-    def test_checklist_noninteractive(self):
-        default = TAGS
-        code, input_ = self._force_noninteractive(
-            self.displayer.checklist, "msg", TAGS, default=default)
-
-        self.assertEqual(code, display_util.OK)
-        self.assertEqual(input_, default)
-
-    def test_scrub_checklist_input_valid(self):
-        # pylint: disable=protected-access
-        indices = [
-            ["1"],
-            ["1", "2", "1"],
-            ["2", "3"],
-        ]
-        exp = [
-            {"tag1"},
-            {"tag1", "tag2"},
-            {"tag2", "tag3"},
-        ]
-        for i, list_ in enumerate(indices):
-            set_tags = set(
-                self.displayer._scrub_checklist_input(list_, TAGS))
-            self.assertEqual(set_tags, exp[i])
-
-    @mock.patch("certbot.display.util.input_with_timeout")
-    def test_directory_select(self, mock_input):
-        args = ["msg", "/var/www/html", "--flag", True]
-        user_input = "/var/www/html"
-        mock_input.return_value = user_input
-
-        returned = self.displayer.directory_select(*args)
-        self.assertEqual(returned, (display_util.OK, user_input))
-
-    def test_directory_select_noninteractive(self):
-        default = "/var/www/html"
-        code, input_ = self._force_noninteractive(
-            self.displayer.directory_select, "msg", default=default)
-
-        self.assertEqual(code, display_util.OK)
-        self.assertEqual(input_, default)
-
-    def _force_noninteractive(self, func, *args, **kwargs):
-        skipped_interaction = self.displayer.skipped_interaction
-
-        with mock.patch("certbot.display.util.sys.stdin") as mock_stdin:
-            mock_stdin.isatty.return_value = False
-            with mock.patch("certbot.display.util.logger") as mock_logger:
-                result = func(*args, **kwargs)
-
-        if skipped_interaction:
-            self.assertIs(mock_logger.warning.called, False)
-        else:
-            self.assertEqual(mock_logger.warning.call_count, 1)
-
-        return result
-
-    def test_scrub_checklist_input_invalid(self):
-        # pylint: disable=protected-access
-        indices = [
-            ["0"],
-            ["4"],
-            ["tag1"],
-            ["1", "tag1"],
-            ["2", "o"]
-        ]
-        for list_ in indices:
-            self.assertEqual(
-                self.displayer._scrub_checklist_input(list_, TAGS), [])
-
-    def test_print_menu(self):
-        # pylint: disable=protected-access
-        # This is purely cosmetic... just make sure there aren't any exceptions
-        self.displayer._print_menu("msg", CHOICES)
-        self.displayer._print_menu("msg", TAGS)
-
-    def test_wrap_lines(self):
-        # pylint: disable=protected-access
-        msg = ("This is just a weak test{0}"
-               "This function is only meant to be for easy viewing{0}"
-               "Test a really really really really really really really really "
-               "really really really really long line...".format('\n'))
-        text = display_util._wrap_lines(msg)
-
-        self.assertEqual(text.count('\n'), 3)
-
-    def test_get_valid_int_ans_valid(self):
-        # pylint: disable=protected-access
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        with mock.patch(input_with_timeout, return_value="1"):
-            self.assertEqual(
-                self.displayer._get_valid_int_ans(1), (display_util.OK, 1))
-        ans = "2"
-        with mock.patch(input_with_timeout, return_value=ans):
-            self.assertEqual(
-                self.displayer._get_valid_int_ans(3),
-                (display_util.OK, int(ans)))
-
-    def test_get_valid_int_ans_invalid(self):
-        # pylint: disable=protected-access
-        answers = [
-            ["0", "c"],
-            ["4", "one", "C"],
-            ["c"],
-        ]
-        input_with_timeout = "certbot.display.util.input_with_timeout"
-        for ans in answers:
-            with mock.patch(input_with_timeout, side_effect=ans):
-                self.assertEqual(
-                    self.displayer._get_valid_int_ans(3),
-                    (display_util.CANCEL, -1))
-
-    def test_methods_take_force_interactive(self):
-        # Every IDisplay method implemented by FileDisplay must take
-        # force_interactive to prevent workflow regressions.
-        for name in interfaces.IDisplay.names():
-            arg_spec = inspect.getfullargspec(getattr(self.displayer, name))
-            self.assertIn("force_interactive", arg_spec.args)
-
-
-class NoninteractiveDisplayTest(unittest.TestCase):
-    """Test non-interactive display. These tests are pretty easy!"""
-    def setUp(self):
-        self.mock_stdout = mock.MagicMock()
-        self.displayer = display_util.NoninteractiveDisplay(self.mock_stdout)
-
-    @mock.patch("certbot.display.util.logger")
-    def test_notification_no_pause(self, mock_logger):
-        self.displayer.notification("message", 10)
-        string = self.mock_stdout.write.call_args[0][0]
-
-        self.assertIn("message", string)
-        mock_logger.debug.assert_called_with("Notifying user: %s", "message")
-
-    def test_notification_decoration(self):
-        from certbot.compat import os
-        self.displayer.notification("message", pause=False, decorate=False)
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertEqual(string, "message" + os.linesep)
-
-        self.displayer.notification("message2", pause=False)
-        string = self.mock_stdout.write.call_args[0][0]
-        self.assertTrue("- - - " in string and ("message2" + os.linesep) in string)
-
-    def test_input(self):
-        d = "an incomputable value"
-        ret = self.displayer.input("message", default=d)
-        self.assertEqual(ret, (display_util.OK, d))
-        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.input, "message")
-
-    def test_menu(self):
-        ret = self.displayer.menu("message", CHOICES, default=1)
-        self.assertEqual(ret, (display_util.OK, 1))
-        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.menu, "message", CHOICES)
-
-    def test_yesno(self):
-        d = False
-        ret = self.displayer.yesno("message", default=d)
-        self.assertEqual(ret, d)
-        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.yesno, "message")
-
-    def test_checklist(self):
-        d = [1, 3]
-        ret = self.displayer.checklist("message", TAGS, default=d)
-        self.assertEqual(ret, (display_util.OK, d))
-        self.assertRaises(errors.MissingCommandlineFlag, self.displayer.checklist, "message", TAGS)
-
-    def test_directory_select(self):
-        default = "/var/www/html"
-        expected = (display_util.OK, default)
-        actual = self.displayer.directory_select("msg", default)
-        self.assertEqual(expected, actual)
-
-        self.assertRaises(
-            errors.MissingCommandlineFlag, self.displayer.directory_select, "msg")
-
-    def test_methods_take_kwargs(self):
-        # Every IDisplay method implemented by NoninteractiveDisplay
-        # should take **kwargs because every method of FileDisplay must
-        # take force_interactive which doesn't apply to
-        # NoninteractiveDisplay.
-
-        # Use pylint code for disable to keep on single line under line length limit
-        for name in interfaces.IDisplay.names():  # pylint: disable=E1120
-            method = getattr(self.displayer, name)
-            # asserts method accepts arbitrary keyword arguments
-            result = inspect.getfullargspec(method).varkw
-            self.assertIsNotNone(result)
-
-
 class SeparateListInputTest(unittest.TestCase):
     """Test Module functions."""
     def setUp(self):
@@ -435,20 +163,6 @@ class SeparateListInputTest(unittest.TestCase):
             self.assertEqual(act, self.exp)
 
 
-class PlaceParensTest(unittest.TestCase):
-    @classmethod
-    def _call(cls, label):  # pylint: disable=protected-access
-        from certbot.display.util import _parens_around_char
-        return _parens_around_char(label)
-
-    def test_single_letter(self):
-        self.assertEqual("(a)", self._call("a"))
-
-    def test_multiple(self):
-        self.assertEqual("(L)abel", self._call("Label"))
-        self.assertEqual("(y)es please", self._call("yes please"))
-
-
 class SummarizeDomainListTest(unittest.TestCase):
     @classmethod
     def _call(cls, domains):
@@ -470,17 +184,5 @@ class SummarizeDomainListTest(unittest.TestCase):
         self.assertEqual("", self._call([]))
 
 
-class NotifyTest(unittest.TestCase):
-    """Test the notify function """
-
-    @test_util.patch_get_utility()
-    def test_notify(self, mock_util):
-        from certbot.display.util import notify
-        notify("Hello World")
-        mock_util().notification.assert_called_with(
-            "Hello World", pause=False, decorate=False, wrap=False
-        )
-
-
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot/tests/eff_test.py

@@ -42,7 +42,7 @@ class PrepareSubscriptionTest(SubscriptionTest):
         from certbot._internal.eff import prepare_subscription
         prepare_subscription(self.config, self.account)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch("certbot._internal.eff.display_util.notify")
     def test_failure(self, mock_notify, mock_get_utility):
         self.config.email = None
@@ -53,21 +53,21 @@ class PrepareSubscriptionTest(SubscriptionTest):
         self.assertIn(expected_part, actual)
         self.assertIsNone(self.account.meta.register_to_eff)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_will_not_subscribe_with_no_prompt(self, mock_get_utility):
         self.config.eff_email = False
         self._call()
         self._assert_no_get_utility_calls(mock_get_utility)
         self.assertIsNone(self.account.meta.register_to_eff)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_will_subscribe_with_no_prompt(self, mock_get_utility):
         self.config.eff_email = True
         self._call()
         self._assert_no_get_utility_calls(mock_get_utility)
         self.assertEqual(self.account.meta.register_to_eff, self.config.email)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_will_not_subscribe_with_prompt(self, mock_get_utility):
         mock_get_utility().yesno.return_value = False
         self._call()
@@ -75,7 +75,7 @@ class PrepareSubscriptionTest(SubscriptionTest):
         self._assert_correct_yesno_call(mock_get_utility)
         self.assertIsNone(self.account.meta.register_to_eff)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_will_subscribe_with_prompt(self, mock_get_utility):
         mock_get_utility().yesno.return_value = True
         self._call()
@@ -176,7 +176,7 @@ class SubscribeTest(unittest.TestCase):
         self.assertTrue(self.mock_notify.called)
         return self.mock_notify.call_args[0][0]
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_subscribe(self, mock_get_utility):
         self._call()
         self.assertIs(mock_get_utility.called, False)

certbot/tests/main_test.py

@@ -182,7 +182,7 @@ class CertonlyTest(unittest.TestCase):
     """Tests for certbot._internal.main.certonly."""
 
     def setUp(self):
-        self.get_utility_patch = test_util.patch_get_utility()
+        self.get_utility_patch = test_util.patch_display_util()
         self.mock_get_utility = self.get_utility_patch.start()
 
     def tearDown(self):
@@ -203,16 +203,15 @@ class CertonlyTest(unittest.TestCase):
     @mock.patch('certbot._internal.main._find_cert')
     @mock.patch('certbot._internal.main._get_and_save_cert')
     @mock.patch('certbot._internal.main._report_new_cert')
-    def test_no_reinstall_text_pause(self, unused_report, mock_auth,
-        mock_find_cert):
+    def test_no_reinstall_text_pause(self, unused_report, mock_auth, mock_find_cert):
         mock_notification = self.mock_get_utility().notification
         mock_notification.side_effect = self._assert_no_pause
         mock_auth.return_value = mock.Mock()
         mock_find_cert.return_value = False, None
         self._call('certonly --webroot -d example.com'.split())
 
-    def _assert_no_pause(self, message, pause=True):  # pylint: disable=unused-argument
-        self.assertIs(pause, False)
+    def _assert_no_pause(self, *args, **kwargs):  # pylint: disable=unused-argument
+        self.assertIs(kwargs.get("pause"), False)
 
     @mock.patch('certbot._internal.main._report_next_steps')
     @mock.patch('certbot._internal.cert_manager.lineage_for_certname')
@@ -432,7 +431,7 @@ class RevokeTest(test_util.TempDirTestCase):
 
     @mock.patch('certbot._internal.main._delete_if_appropriate')
     @mock.patch('certbot._internal.cert_manager.delete')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_revocation_with_prompt(self, mock_get_utility,
             mock_delete, mock_delete_if_appropriate):
         mock_get_utility().yesno.return_value = False
@@ -452,12 +451,12 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase):
             self._call(self.config)
         mock_delete.assert_not_called()
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_delete_flag_opt_out(self, unused_mock_get_utility):
         self.config.delete_after_revoke = False
         self._test_delete_opt_out_common()
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_delete_prompt_opt_out(self, mock_get_utility):
         util_mock = mock_get_utility()
         util_mock.yesno.return_value = False
@@ -469,7 +468,7 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.cert_manager.match_and_check_overlaps')
     @mock.patch('certbot._internal.storage.full_archive_path')
     @mock.patch('certbot._internal.cert_manager.cert_path_to_lineage')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_overlapping_archive_dirs(self, mock_get_utility,
             mock_cert_path_to_lineage, mock_archive,
             mock_match_and_check_overlaps, mock_delete,
@@ -489,7 +488,7 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.storage.full_archive_path')
     @mock.patch('certbot._internal.cert_manager.delete')
     @mock.patch('certbot._internal.cert_manager.cert_path_to_lineage')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_cert_path_only(self, mock_get_utility,
             mock_cert_path_to_lineage, mock_delete, mock_archive,
             mock_overlapping_archive_dirs, mock_renewal_file_for_certname):
@@ -507,7 +506,7 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.storage.full_archive_path')
     @mock.patch('certbot._internal.cert_manager.cert_path_to_lineage')
     @mock.patch('certbot._internal.cert_manager.delete')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_noninteractive_deletion(self, mock_get_utility, mock_delete,
             mock_cert_path_to_lineage, mock_full_archive_dir,
             mock_match_and_check_overlaps, mock_renewal_file_for_certname):
@@ -527,7 +526,7 @@ class DeleteIfAppropriateTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.storage.full_archive_path')
     @mock.patch('certbot._internal.cert_manager.cert_path_to_lineage')
     @mock.patch('certbot._internal.cert_manager.delete')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_opt_in_deletion(self, mock_get_utility, mock_delete,
             mock_cert_path_to_lineage, mock_full_archive_dir,
             mock_match_and_check_overlaps, mock_renewal_file_for_certname):
@@ -563,7 +562,7 @@ class DetermineAccountTest(test_util.ConfigTestCase):
         # pylint: disable=protected-access
         from certbot._internal.main import _determine_account
         with mock.patch('certbot._internal.main.account.AccountFileStorage') as mock_storage, \
-             test_util.patch_get_utility():
+             test_util.patch_display_util():
             mock_storage.return_value = self.account_storage
             return _determine_account(self.config)
 
@@ -661,7 +660,7 @@ class MainTest(test_util.ConfigTestCase):
                 return ret, stdout, stderr, client
 
     def _call_no_clientmock(self, args, stdout=None):
-        "Run the client with output streams mocked out"
+        """Run the client with output streams mocked out"""
         args = self.standard_args + args
 
         toy_stdout = stdout if stdout else io.StringIO()
@@ -896,7 +895,7 @@ class MainTest(test_util.ConfigTestCase):
         plugins = mock_disco.PluginsRegistry.find_all()
 
         stdout = io.StringIO()
-        with test_util.patch_get_utility_with_stdout(stdout=stdout):
+        with test_util.patch_display_util_with_stdout(stdout=stdout):
             _, stdout, _, _ = self._call(['plugins'], stdout)
 
         plugins.visible.assert_called_once_with()
@@ -917,7 +916,7 @@ class MainTest(test_util.ConfigTestCase):
 
         stdout = io.StringIO()
         with mock.patch('certbot.util.set_up_core_dir') as mock_set_up_core_dir:
-            with test_util.patch_get_utility_with_stdout(stdout=stdout):
+            with test_util.patch_display_util_with_stdout(stdout=stdout):
                 mock_set_up_core_dir.side_effect = throw_error
                 _, stdout, _, _ = self._call(['plugins'], stdout)
 
@@ -933,7 +932,7 @@ class MainTest(test_util.ConfigTestCase):
         plugins = mock_disco.PluginsRegistry.find_all()
 
         stdout = io.StringIO()
-        with test_util.patch_get_utility_with_stdout(stdout=stdout):
+        with test_util.patch_display_util_with_stdout(stdout=stdout):
             _, stdout, _, _ = self._call(['plugins', '--init'], stdout)
 
         plugins.visible.assert_called_once_with()
@@ -951,7 +950,7 @@ class MainTest(test_util.ConfigTestCase):
         plugins = mock_disco.PluginsRegistry.find_all()
 
         stdout = io.StringIO()
-        with test_util.patch_get_utility_with_stdout(stdout=stdout):
+        with test_util.patch_display_util_with_stdout(stdout=stdout):
             _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'], stdout)
 
         plugins.visible.assert_called_once_with()
@@ -1040,8 +1039,7 @@ class MainTest(test_util.ConfigTestCase):
                 self._call(args)
 
     @mock.patch('certbot._internal.main._report_new_cert')
-    @test_util.patch_get_utility()
-    def test_certonly_dry_run_new_request_success(self, mock_get_utility, mock_report):
+    def test_certonly_dry_run_new_request_success(self, mock_report):
         mock_client = mock.MagicMock()
         mock_client.obtain_and_enroll_certificate.return_value = None
         self._certonly_new_request_common(mock_client, ['--dry-run'])
@@ -1049,15 +1047,12 @@ class MainTest(test_util.ConfigTestCase):
             mock_client.obtain_and_enroll_certificate.call_count, 1)
         self.assertEqual(mock_report.call_count, 1)
         self.assertIs(mock_report.call_args[0][0].dry_run, True)
-        # Asserts we don't suggest donating after a successful dry run
-        self.assertEqual(mock_get_utility().add_message.call_count, 0)
 
     @mock.patch('certbot._internal.main._report_new_cert')
     @mock.patch('certbot._internal.main.util.atexit_register')
     @mock.patch('certbot._internal.eff.handle_subscription')
     @mock.patch('certbot.crypto_util.notAfter')
-    @test_util.patch_get_utility()
-    def test_certonly_new_request_success(self, unused_mock_get_utility, mock_notAfter,
+    def test_certonly_new_request_success(self, mock_notAfter,
                                           mock_subscription, mock_register, mock_report):
         cert_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/foo.bar'))
         key_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/baz.qux'))
@@ -1114,9 +1109,9 @@ class MainTest(test_util.ConfigTestCase):
                 mock_fdc.return_value = (mock_lineage, None)
                 with mock.patch('certbot._internal.main._init_le_client') as mock_init:
                     mock_init.return_value = mock_client
-                    with test_util.patch_get_utility() as mock_get_utility:
+                    with mock.patch('certbot._internal.display.obj.get_display') as mock_display:
                         if not quiet_mode:
-                            mock_get_utility().notification.side_effect = write_msg
+                            mock_display().notification.side_effect = write_msg
                         with mock.patch('certbot._internal.main.renewal.crypto_util') \
                             as mock_crypto_util:
                             mock_crypto_util.notAfter.return_value = expiry_date
@@ -1156,7 +1151,7 @@ class MainTest(test_util.ConfigTestCase):
                 with open(os.path.join(self.config.logs_dir, "letsencrypt.log")) as lf:
                     self.assertIn(log_out, lf.read())
 
-        return mock_lineage, mock_get_utility, stdout
+        return mock_lineage, mock_display, stdout
 
     @mock.patch('certbot._internal.main._report_new_cert')
     @mock.patch('certbot._internal.main.util.atexit_register')
@@ -1182,9 +1177,9 @@ class MainTest(test_util.ConfigTestCase):
         self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'],
                                   log_out="Auto-renewal forced")
 
-        _, get_utility, _ = self._test_renewal_common(False, ['-tvv', '--debug', '--keep'],
+        _, mock_displayer, _ = self._test_renewal_common(False, ['-tvv', '--debug', '--keep'],
                                   should_renew=False)
-        self.assertIn('not yet due', get_utility().notification.call_args[0][0])
+        self.assertIn('not yet due', mock_displayer().notification.call_args[0][0])
 
     def _dump_log(self):
         print("Logs:")
@@ -1392,7 +1387,7 @@ class MainTest(test_util.ConfigTestCase):
                   .format(sys.executable)])
         self.assertIn('No hooks were run.', stdout.getvalue())
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.main._find_lineage_for_domains_and_certname')
     @mock.patch('certbot._internal.main._init_le_client')
     @mock.patch('certbot._internal.main._report_new_cert')
@@ -1421,17 +1416,16 @@ class MainTest(test_util.ConfigTestCase):
         mock_client.save_certificate.return_value = cert_path, None, full_path
         with mock.patch('certbot._internal.main._init_le_client') as mock_init:
             mock_init.return_value = mock_client
-            with test_util.patch_get_utility() as mock_get_utility:
-                chain_path = os.path.normpath(os.path.join(
-                    self.config.config_dir,
-                    'live/example.com/chain.pem'))
-                args = ('-a standalone certonly --csr {0} --cert-path {1} '
-                        '--chain-path {2} --fullchain-path {3}').format(
-                            CSR, cert_path, chain_path, full_path).split()
-                if extra_args:
-                    args += extra_args
-                with mock.patch('certbot._internal.main.crypto_util'):
-                    self._call(args)
+            chain_path = os.path.normpath(os.path.join(
+                self.config.config_dir,
+                'live/example.com/chain.pem'))
+            args = ('-a standalone certonly --csr {0} --cert-path {1} '
+                    '--chain-path {2} --fullchain-path {3}').format(
+                        CSR, cert_path, chain_path, full_path).split()
+            if extra_args:
+                args += extra_args
+            with mock.patch('certbot._internal.main.crypto_util'):
+                self._call(args)
 
         if '--dry-run' in args:
             self.assertIs(mock_client.save_certificate.called, False)
@@ -1439,13 +1433,11 @@ class MainTest(test_util.ConfigTestCase):
             mock_client.save_certificate.assert_called_once_with(
                 certr, chain, cert_path, chain_path, full_path)
 
-        return mock_get_utility
-
     @mock.patch('certbot._internal.main._csr_report_new_cert')
     @mock.patch('certbot._internal.main.util.atexit_register')
     @mock.patch('certbot._internal.eff.handle_subscription')
     def test_certonly_csr(self, mock_subscription, mock_register, mock_csr_report):
-        _ = self._test_certonly_csr_common()
+        self._test_certonly_csr_common()
         self.assertEqual(mock_csr_report.call_count, 1)
         self.assertIn('cert_512.pem', mock_csr_report.call_args[0][1])
         self.assertIsNone(mock_csr_report.call_args[0][2])
@@ -1455,7 +1447,7 @@ class MainTest(test_util.ConfigTestCase):
 
     @mock.patch('certbot._internal.main._csr_report_new_cert')
     def test_certonly_csr_dry_run(self, mock_csr_report):
-        _ = self._test_certonly_csr_common(['--dry-run'])
+        self._test_certonly_csr_common(['--dry-run'])
         self.assertEqual(mock_csr_report.call_count, 1)
         self.assertIs(mock_csr_report.call_args[0][0].dry_run, True)
 
@@ -1552,7 +1544,7 @@ class UnregisterTest(unittest.TestCase):
             '_determine_account': mock.patch('certbot._internal.main._determine_account'),
             'account': mock.patch('certbot._internal.main.account'),
             'client': mock.patch('certbot._internal.main.client'),
-            'get_utility': test_util.patch_get_utility()}
+            'get_utility': test_util.patch_display_util()}
         self.mocks = {k: v.start() for k, v in self.patchers.items()}
 
     def tearDown(self):
@@ -1633,7 +1625,7 @@ class EnhanceTest(test_util.ConfigTestCase):
 
     def setUp(self):
         super().setUp()
-        self.get_utility_patch = test_util.patch_get_utility()
+        self.get_utility_patch = test_util.patch_display_util()
         self.mock_get_utility = self.get_utility_patch.start()
         self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement)
 
@@ -1743,7 +1735,7 @@ class EnhanceTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.main.display_ops.choose_values')
     @mock.patch('certbot._internal.main.plug_sel.pick_installer')
     @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_enhancement_enable(self, _, _rec, mock_inst, mock_choose, mock_lineage):
         mock_inst.return_value = self.mockinstaller
         mock_choose.return_value = ["example.com", "another.tld"]
@@ -1757,7 +1749,7 @@ class EnhanceTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.main.display_ops.choose_values')
     @mock.patch('certbot._internal.main.plug_sel.pick_installer')
     @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_enhancement_enable_not_supported(self, _, _rec, mock_inst, mock_choose, mock_lineage):
         mock_inst.return_value = null.Installer(self.config, "null")
         mock_choose.return_value = ["example.com", "another.tld"]
@@ -1978,7 +1970,7 @@ class UpdateAccountTest(test_util.ConfigTestCase):
             'determine_account': mock.patch('certbot._internal.main._determine_account'),
             'notify': mock.patch('certbot._internal.main.display_util.notify'),
             'prepare_sub': mock.patch('certbot._internal.eff.prepare_subscription'),
-            'util': test_util.patch_get_utility()
+            'util': test_util.patch_display_util()
         }
         self.mocks = { k: patches[k].start() for k in patches }
         for patch in patches.values():

certbot/tests/plugins/dns_common_test.py

@@ -42,7 +42,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen
 
         self.auth = DNSAuthenticatorTest._FakeDNSAuthenticator(self.config, "fake")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform(self, unused_mock_get_utility):
         self.auth.perform([self.achall])
 
@@ -55,7 +55,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen
 
         self.auth._cleanup.assert_called_once_with(dns_test_common.DOMAIN, mock.ANY, mock.ANY)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_prompt(self, mock_get_utility):
         mock_display = mock_get_utility()
         mock_display.input.side_effect = ((display_util.OK, "",),
@@ -64,14 +64,14 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen
         self.auth._configure("other_key", "")
         self.assertEqual(self.auth.config.fake_other_key, "value")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_prompt_canceled(self, mock_get_utility):
         mock_display = mock_get_utility()
         mock_display.input.side_effect = ((display_util.CANCEL, "c",),)
 
         self.assertRaises(errors.PluginError, self.auth._configure, "other_key", "")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_prompt_file(self, mock_get_utility):
         path = os.path.join(self.tempdir, 'file.ini')
         open(path, "wb").close()
@@ -85,7 +85,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen
         self.auth._configure_file("file_path", "")
         self.assertEqual(self.auth.config.fake_file_path, path)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_prompt_file_canceled(self, mock_get_utility):
         mock_display = mock_get_utility()
         mock_display.directory_select.side_effect = ((display_util.CANCEL, "c",),)
@@ -101,7 +101,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen
 
         self.assertEqual(credentials.conf("test"), "value")
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_prompt_credentials(self, mock_get_utility):
         bad_path = os.path.join(self.tempdir, 'bad-file.ini')
         dns_test_common.write({"fake_other": "other_value"}, bad_path)

certbot/tests/plugins/enhancements_test.py

@@ -19,7 +19,7 @@ class EnhancementTest(test_util.ConfigTestCase):
         self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement)
 
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_enhancement_enabled_enhancements(self, _):
         FAKEINDEX = [
             {

certbot/tests/plugins/manual_test.py

@@ -21,9 +21,9 @@ class AuthenticatorTest(test_util.TempDirTestCase):
 
     def setUp(self):
         super().setUp()
-        get_utility_patch = test_util.patch_get_utility()
-        self.mock_get_utility = get_utility_patch.start()
-        self.addCleanup(get_utility_patch.stop)
+        get_display_patch = test_util.patch_display_util()
+        self.mock_get_display = get_display_patch.start()
+        self.addCleanup(get_display_patch.stop)
 
         self.http_achall = acme_util.HTTP01_A
         self.dns_achall = acme_util.DNS01_A
@@ -95,8 +95,8 @@ class AuthenticatorTest(test_util.TempDirTestCase):
             http_expected)
 
         # Successful hook output should be sent to notify
-        self.assertEqual(self.mock_get_utility().notification.call_count, len(self.achalls))
-        for i, (args, _) in enumerate(self.mock_get_utility().notification.call_args_list):
+        self.assertEqual(self.mock_get_display().notification.call_count, len(self.achalls))
+        for i, (args, _) in enumerate(self.mock_get_display().notification.call_args_list):
             needle = textwrap.indent(self.auth.env[self.achalls[i]]['CERTBOT_AUTH_OUTPUT'], ' ')
             self.assertIn(needle, args[0])
 
@@ -105,8 +105,8 @@ class AuthenticatorTest(test_util.TempDirTestCase):
             self.auth.perform(self.achalls),
             [achall.response(achall.account_key) for achall in self.achalls])
 
-        self.assertEqual(self.mock_get_utility().notification.call_count, len(self.achalls))
-        for i, (args, kwargs) in enumerate(self.mock_get_utility().notification.call_args_list):
+        self.assertEqual(self.mock_get_display().notification.call_count, len(self.achalls))
+        for i, (args, kwargs) in enumerate(self.mock_get_display().notification.call_args_list):
             achall = self.achalls[i]
             self.assertIn(achall.validation(achall.account_key), args[0])
             self.assertIs(kwargs['wrap'], False)

certbot/tests/plugins/selection_test.py

@@ -1,21 +1,21 @@
 """Tests for letsencrypt.plugins.selection"""
 import sys
-import unittest
 from typing import List
+import unittest
 
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock
-import zope.component
 
 from certbot import errors
 from certbot import interfaces
+from certbot._internal.display import obj as display_obj
 from certbot._internal.plugins.disco import PluginsRegistry
-from certbot.compat import os
 from certbot.display import util as display_util
 from certbot.tests import util as test_util
 
+try:
+    import mock
+except ImportError:  # pragma: no cover
+    from unittest import mock
+
 
 class ConveniencePickPluginTest(unittest.TestCase):
     """Tests for certbot._internal.plugins.selection.pick_*."""
@@ -118,8 +118,8 @@ class ChoosePluginTest(unittest.TestCase):
     """Tests for certbot._internal.plugins.selection.choose_plugin."""
 
     def setUp(self):
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout,
-                                                               False))
+        display_obj.set_display(display_obj.FileDisplay(sys.stdout, False))
+
         self.mock_apache = mock.Mock(
             description_with_name="a", misconfigured=True)
         self.mock_apache.name = "apache"
@@ -135,14 +135,14 @@ class ChoosePluginTest(unittest.TestCase):
         from certbot._internal.plugins.selection import choose_plugin
         return choose_plugin(self.plugins, "Question?")
 
-    @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util")
+    @test_util.patch_display_util()
     def test_selection(self, mock_util):
         mock_util().menu.side_effect = [(display_util.OK, 0),
                                         (display_util.OK, 1)]
         self.assertEqual(self.mock_stand, self._call())
         self.assertEqual(mock_util().notification.call_count, 1)
 
-    @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util")
+    @test_util.patch_display_util()
     def test_more_info(self, mock_util):
         mock_util().menu.side_effect = [
             (display_util.OK, 1),
@@ -150,7 +150,7 @@ class ChoosePluginTest(unittest.TestCase):
 
         self.assertEqual(self.mock_stand, self._call())
 
-    @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util")
+    @test_util.patch_display_util()
     def test_no_choice(self, mock_util):
         mock_util().menu.return_value = (display_util.CANCEL, 0)
         self.assertIsNone(self._call())

certbot/tests/plugins/standalone_test.py

@@ -1,14 +1,12 @@
 """Tests for certbot._internal.plugins.standalone."""
 import errno
 import socket
+from typing import Dict
+from typing import Set
+from typing import Tuple
 import unittest
-from typing import Dict, Set, Tuple
 
 import josepy as jose
-try:
-    import mock
-except ImportError: # pragma: no cover
-    from unittest import mock
 import OpenSSL.crypto  # pylint: disable=unused-import
 
 from acme import challenges
@@ -18,6 +16,12 @@ from certbot import errors
 from certbot.tests import acme_util
 from certbot.tests import util as test_util
 
+try:
+    import mock
+except ImportError: # pragma: no cover
+    from unittest import mock
+
+
 
 class ServerManagerTest(unittest.TestCase):
     """Tests for certbot._internal.plugins.standalone.ServerManager."""
@@ -101,7 +105,7 @@ class AuthenticatorTest(unittest.TestCase):
         expected = [achall.response(achall.account_key) for achall in achalls]
         self.assertEqual(response, expected)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform_eaddrinuse_retry(self, mock_get_utility):
         mock_utility = mock_get_utility()
         encountered_errno = errno.EADDRINUSE
@@ -113,7 +117,7 @@ class AuthenticatorTest(unittest.TestCase):
         self.test_perform()
         self._assert_correct_yesno_call(mock_yesno)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform_eaddrinuse_no_retry(self, mock_get_utility):
         mock_utility = mock_get_utility()
         mock_yesno = mock_utility.yesno

certbot/tests/plugins/webroot_test.py

@@ -69,7 +69,7 @@ class AuthenticatorTest(unittest.TestCase):
     def test_prepare(self):
         self.auth.prepare()  # shouldn't raise any exceptions
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_webroot_from_list(self, mock_get_utility):
         self.config.webroot_path = []
         self.config.webroot_map = {"otherthing.com": self.path}
@@ -86,7 +86,7 @@ class AuthenticatorTest(unittest.TestCase):
         self.assertEqual(self.config.webroot_map[self.achall.domain],
                          self.path)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_webroot_from_list_help_and_cancel(self, mock_get_utility):
         self.config.webroot_path = []
         self.config.webroot_map = {"otherthing.com": self.path}
@@ -101,7 +101,7 @@ class AuthenticatorTest(unittest.TestCase):
                 webroot in call[0][1]
                 for webroot in self.config.webroot_map.values()))
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_new_webroot(self, mock_get_utility):
         self.config.webroot_path = []
         self.config.webroot_map = {"something.com": self.path}
@@ -116,7 +116,7 @@ class AuthenticatorTest(unittest.TestCase):
 
         self.assertEqual(self.config.webroot_map[self.achall.domain], self.path)
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_new_webroot_empty_map_cancel(self, mock_get_utility):
         self.config.webroot_path = []
         self.config.webroot_map = {}
@@ -154,7 +154,7 @@ class AuthenticatorTest(unittest.TestCase):
         mock_ownership.side_effect = OSError(errno.EACCES, "msg")
         self.auth.perform([self.achall])  # exception caught and logged
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_perform_new_webroot_not_in_map(self, mock_get_utility):
         new_webroot = tempfile.mkdtemp()
         self.config.webroot_path = []

certbot/tests/renewal_test.py

@@ -100,7 +100,7 @@ class RenewalTest(test_util.ConfigTestCase):
 
         assert self.config.elliptic_curve == 'secp256r1'
 
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     @mock.patch('certbot._internal.renewal.cli.set_by_cli')
     def test_remove_deprecated_config_elements(self, mock_set_by_cli, unused_mock_get_utility):
         mock_set_by_cli.return_value = False

certbot/tests/renewupdater_test.py

@@ -26,7 +26,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase):
     @mock.patch('certbot._internal.main._get_and_save_cert')
     @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins')
     @mock.patch('certbot._internal.plugins.selection.get_unprepared_installer')
-    @test_util.patch_get_utility()
+    @test_util.patch_display_util()
     def test_server_updates(self, _, mock_geti, mock_select, mock_getsave):
         mock_getsave.return_value = mock.MagicMock()
         mock_generic_updater = self.generic_updater

letsencrypt-auto-source/certbot-auto.asc

@@ -1,11 +0,0 @@
------BEGIN PGP SIGNATURE-----
-
-iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmBsmUkACgkQTRfJlc2X
-dfI7Bwf9FkNrf1HEh2G3uk1p+qLMd/s5kcVV2udK2FkRELee5nHlLZx2YmHA/8ID
-gqsk8EsyRZNMX374nGrPm0syykdEsyVtMJTbHCEr+Ms3l54ZgE3HV6ywnhWSlAFo
-Za50kdzhodBVTS5AEADbCKLKObVAWwO3fFKtKyv/iY29ykpHK0KSHCKRII3iQU7l
-dnR6u35Z0wgfEmDxsH27K6uo0YepZaEL70qHHFk93MhCh9Z15rO17gRpsVzz7Z1j
-YClI6h2K/VOfZtbkoQvoks7s+xd75Kjr3GNH+cznkJx8gNWSZLfkc1XX4Bjdm4GG
-IWz3Ezy8tFg6PtITb7y+aIg75kWx4w==
-=zEy4
------END PGP SIGNATURE-----

pytest.ini

@@ -13,6 +13,10 @@
 # The current warnings being ignored are:
 # 1) The warning raised when importing certbot.tests.util and the external mock
 #    library is installed.
+# 2) An ImportWarning is raised with older versions of setuptools and
+#    zope.interface. See
+#    https://github.com/zopefoundation/zope.interface/issues/68 for more info.
 filterwarnings =
     error
     ignore:The external mock module:PendingDeprecationWarning
+    ignore:.*zope. missing __init__:ImportWarning

tests/modification-check.py

@@ -10,11 +10,6 @@ import os
 # taken from our v1.14.0 tag which was the last release we intended to make
 # changes to certbot-auto.
 #
-# certbot-auto, letsencrypt-auto, and letsencrypt-auto-source/certbot-auto.asc
-# can be removed from this dict after coordinating with tech ops to ensure we
-# get the behavior we want from https://dl.eff.org. See
-# https://github.com/certbot/certbot/issues/8742 for more info.
-#
 # Deleting letsencrypt-auto-source/letsencrypt-auto and
 # letsencrypt-auto-source/letsencrypt-auto.sig can be done once we're
 # comfortable breaking any certbot-auto scripts that haven't already updated to
@@ -22,14 +17,8 @@ import os
 # https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna for
 # more info.
 EXPECTED_FILES = {
-    'certbot-auto':
-        'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2',
-    'letsencrypt-auto':
-        'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2',
     os.path.join('letsencrypt-auto-source', 'letsencrypt-auto'):
         'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2',
-    os.path.join('letsencrypt-auto-source', 'certbot-auto.asc'):
-        '0558ba7bd816732b38c092e8fedb6033dad01f263e290ec6b946263aaf6625a8',
     os.path.join('letsencrypt-auto-source', 'letsencrypt-auto.sig'):
         '61c036aabf75da350b0633da1b2bef0260303921ecda993455ea5e6d3af3b2fe',
 }

tools/dev_constraints.txt

@@ -1,131 +0,0 @@
-# Specifies extra Python package versions during our tests with the oldest
-# supported versions of our dependencies. Some dev package versions specified
-# here may be overridden by higher level constraints files during tests (eg.
-# tools/oldest_constraints.txt).
-alabaster==0.7.10
-apacheconfig==0.3.2
-apipkg==1.4
-appnope==0.1.0
-asn1crypto==0.22.0
-astroid==2.3.3
-attrs==17.3.0
-azure-devops==6.0.0b2
-Babel==2.5.1
-backcall==0.2.0
-backports.functools-lru-cache==1.5
-backports.shutil-get-terminal-size==1.0.0
-backports.ssl-match-hostname==3.7.0.1
-bcrypt==3.1.6
-boto3==1.17.4
-botocore==1.20.4
-cached-property==1.5.1
-cloudflare==2.3.1
-configparser==3.7.4
-contextlib2==0.6.0.post1
-coverage==4.5.4
-decorator==4.4.1
-deprecated==1.2.10
-dns-lexicon==3.3.17
-dnspython==2.1.0
-docker==4.3.1
-docker-compose==1.26.2
-docker-pycreds==0.4.0
-dockerpty==0.4.1
-docopt==0.6.2
-docutils==0.15.2
-execnet==1.5.0
-functools32==3.2.3.post2
-future==0.16.0
-futures==3.3.0
-filelock==3.0.12
-google-api-python-client==1.5.5
-httplib2==0.18.1
-imagesize==0.7.1
-importlib-metadata==0.23
-ipdb==0.12.3
-ipython==7.9.0
-ipython-genutils==0.2.0
-isodate==0.6.0
-isort==4.3.21
-jedi==0.17.1
-Jinja2==2.9.6
-jmespath==0.9.4
-josepy==1.1.0
-jsonpickle==2.0.0
-jsonschema==2.6.0
-lazy-object-proxy==1.4.3
-logger==1.4
-logilab-common==1.4.1
-MarkupSafe==1.1.1
-mccabe==0.6.1
-more-itertools==5.0.0
-msrest==0.6.18
-mypy==0.812
-mypy-extensions==0.4.3
-ndg-httpsclient==0.3.2
-google-auth==1.32.1
-oauthlib==3.1.0
-packaging==19.2
-paramiko==2.4.2
-parso==0.7.0
-pathlib2==2.3.5
-pexpect==4.7.0
-pickleshare==0.7.5
-pip==20.2.4
-pkginfo==1.4.2
-pluggy==0.13.0
-ply==3.4
-prompt-toolkit==2.0.10
-ptyprocess==0.6.0
-py==1.8.0
-pyasn1==0.1.9
-pyasn1-modules==0.0.10
-PyGithub==1.52
-Pygments==2.2.0
-pyjwt==1.7.1
-pylint==2.4.3
-pynacl==1.3.0
-# If pynsist version is upgraded, our NSIS template windows-installer/template.nsi
-# must be upgraded if necessary using the new built-in one from pynsist.
-pynsist==2.7
-pytest==3.2.5
-pytest-cov==2.5.1
-pytest-forked==0.2
-pytest-xdist==1.22.5
-pytest-sugar==0.9.2
-pytest-rerunfailures==4.2
-python-dateutil==2.8.1
-python-digitalocean==1.15.0
-python-dotenv==0.14.0
-pywin32==300
-PyYAML==5.3.1
-repoze.sphinx.autointerface==0.8
-requests-file==1.4.2
-requests-oauthlib==1.3.0
-requests-toolbelt==0.8.0
-rsa==3.4.2
-s3transfer==0.3.1
-scandir==1.10.0
-simplegeneric==0.8.1
-singledispatch==3.4.0.3
-snowballstemmer==1.2.1
-Sphinx==1.7.5
-sphinx-rtd-theme==0.2.4
-sphinxcontrib-websupport==1.0.1
-texttable==0.9.1
-tldextract==2.2.0
-toml==0.10.0
-tox==3.14.0
-tqdm==4.19.4
-traitlets==4.3.3
-twine==1.11.0
-typed-ast==1.4.1
-typing==3.6.4
-typing-extensions==3.7.4.3
-uritemplate==3.0.0
-virtualenv==16.6.2
-wcwidth==0.1.8
-websocket-client==0.56.0
-wheel==0.35.1
-wrapt==1.11.2
-zipp==0.6.0

tools/merge_requirements.py

@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-"""Merges multiple Python requirements files into one file.
-
-Requirements files specified later take precedence over earlier ones.
-Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are
-currently supported.
-
-"""
-import sys
-
-
-def process_entries(entries):
-    """
-    Ignore empty lines, comments and editable requirements
-
-    :param list entries: List of entries
-
-    :returns: mapping from a project to its version specifier
-    :rtype: dict
-    """
-    data = {}
-    for e in entries:
-        e = e.strip()
-        if e and not e.startswith('#') and not e.startswith('-e'):
-            # Support for <= was added as part of
-            # https://github.com/certbot/certbot/pull/8460 because we weren't
-            # able to pin a package to an exact version. Normally, this
-            # functionality shouldn't be needed so we could remove it in the
-            # future. If you do so, make sure to update other places in this
-            # file related to this behavior such as this file's docstring.
-            for comparison in ('==', '<=',):
-                parts = e.split(comparison)
-                if len(parts) == 2:
-                    project_name = parts[0]
-                    version = parts[1]
-                    data[project_name] = comparison + version
-                    break
-            else:
-                raise ValueError("Unexpected syntax '{0}'".format(e))
-    return data
-
-def read_file(file_path):
-    """Reads in a Python requirements file.
-
-    :param str file_path: path to requirements file
-
-    :returns: list of entries in the file
-    :rtype: list
-
-    """
-    with open(file_path) as file_h:
-        return file_h.readlines()
-
-def output_requirements(requirements):
-    """Prepare print requirements to stdout.
-
-    :param dict requirements: mapping from a project to its version
-        specifier
-
-    """
-    return '\n'.join('{0}{1}'.format(key, value)
-                     for key, value in sorted(requirements.items()))
-
-
-def main(*paths):
-    """Merges multiple requirements files together and prints the result.
-
-    Requirement files specified later in the list take precedence over earlier
-    files. Files are read from file paths passed from the command line arguments.
-
-    If no command line arguments are defined, data is read from stdin instead.
-
-    :param tuple paths: paths to requirements files provided on command line
-
-    """
-    data = {}
-    if paths:
-        for path in paths:
-            data.update(process_entries(read_file(path)))
-    else:
-        # Need to check if interactive to avoid blocking if nothing is piped
-        if not sys.stdin.isatty():
-            stdin_data = []
-            for line in sys.stdin:
-                stdin_data.append(line)
-            data.update(process_entries(stdin_data))
-
-    return output_requirements(data)
-
-
-if __name__ == '__main__':
-    print(main(*sys.argv[1:]))  # pylint: disable=star-args

tools/oldest_constraints.txt

@@ -1,77 +1,100 @@
-# This file contains the oldest versions of our dependencies we're trying to
-# support. Usually these version numbers are taken from the packages of our
-# dependencies available in popular LTS Linux distros. Keeping compatibility
-# with those versions makes it much easier for OS maintainers to update their
-# Certbot packages.
-#
-# When updating these dependencies, we should try to only update them to the
-# oldest version of the package that is found in a non-EOL'd version of
-# CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories
-# using a version of Python we support. If the distro is EOL'd or using a
-# version of Python we don't support, it can be ignored.
-
-# CentOS/RHEL 7 EPEL constraints
-# Some of these constraints may be stricter than necessary because they
-# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL.
-cffi==1.9.1
-chardet==2.2.1
-ipaddress==1.0.16
-mock==1.0.1
-ndg-httpsclient==0.3.2
-ply==3.4
-pyOpenSSL==17.3.0
-pyasn1==0.1.9
-pycparser==2.14
-pyRFC3339==1.0
-python-augeas==0.5.0
-google-auth==1.32.1
-urllib3==1.10.2
-zope.component==4.1.0
-zope.event==4.0.3
-zope.interface==4.0.5
-
-# Debian Jessie Backports constraints
-# Debian Jessie has reached end of life so these dependencies can probably be
-# updated as needed or desired.
-colorama==0.3.2
-enum34==1.0.3
-html5lib==0.999
-pbr==1.8.0
-pytz==2012rc0
-
-# Debian Buster constraints
-google-api-python-client==1.5.5
-pyparsing==2.2.0
-
-# Our setup.py constraints
+# This file was generated by tools/pinning/oldest/repin.sh and can be updated using
+# that script.
 apacheconfig==0.3.2
-cloudflare==1.5.1
-python-digitalocean==1.11
-requests==2.6.0
-
-# Ubuntu Xenial constraints
-# Ubuntu Xenial only has versions of Python which we do not support available
-# so these dependencies can probably be updated as needed or desired.
-ConfigArgParse==0.10.0
-funcsigs==0.4
-zope.hookable==4.0.4
-
-# Ubuntu Bionic constraints.
-cryptography==2.1.4
-distro==1.0.1
-httplib2==0.9.2
-idna==2.6
-setuptools==39.0.1
-six==1.11.0
-
-# Ubuntu Focal constraints
 asn1crypto==0.24.0
-configobj==5.0.6
-parsedatetime==2.4
-
-# Plugin constraints
-# These aren't necessarily the oldest versions we need to support
-# Tracking at https://github.com/certbot/certbot/issues/6473
+astroid==2.5.6; python_version >= "3.6" and python_version < "4.0"
+atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0"
+attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 boto3==1.4.7
 botocore==1.7.41
-dns-lexicon==3.1.0
+cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+cffi==1.9.1
+chardet==2.2.1
+cloudflare==1.5.1
+colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and python_version >= "3.6" and sys_platform == "win32" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.5.0"
+configargparse==0.10.0
+configobj==5.0.6
+coverage==5.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
+cryptography==2.1.4
+cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
+distro==1.0.1
+dns-lexicon==3.2.1
+dnspython==2.1.0; python_version >= "3.6"
+docker-compose==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+docker-pycreds==0.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+docker==3.7.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+docutils==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+filelock==3.0.12; python_version >= "3.6"
+funcsigs==0.4
+future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
+google-api-python-client==1.5.5
+httplib2==0.9.2
+idna==2.6
+importlib-metadata==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8" or python_version < "3.8" and python_version >= "3.6"
+iniconfig==1.1.1; python_version >= "3.6"
+ipaddress==1.0.16
+isort==5.8.0; python_version >= "3.6" and python_version < "4.0"
+jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
+josepy==1.8.0; python_version >= "3.6"
+jsonschema==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0"
+logger==1.4; python_version >= "3.6"
+mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0"
+mock==1.0.1
+mypy-extensions==0.4.3; python_version >= "3.6"
+mypy==0.910; python_version >= "3.6"
+ndg-httpsclient==0.3.2
+oauth2client==4.0.0
+packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+paramiko==2.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+parsedatetime==2.4
+pbr==1.8.0
+pip==20.2.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+ply==3.4
+py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pyasn1-modules==0.0.10; python_version >= "3.6"
+pyasn1==0.1.9
+pycparser==2.14
+pylint==2.8.3; python_version >= "3.6" and python_version < "4.0"
+pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+pyopenssl==17.3.0
+pyparsing==2.2.0
+pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+pyrfc3339==1.0
+pytest-cov==2.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pytest-xdist==2.3.0; python_version >= "3.6" or python_version >= "3.6"
+pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+python-augeas==0.5.0
+python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
+python-digitalocean==1.11
+pytz==2012c
+pywin32==301; sys_platform == "win32" and python_version >= "3.6" or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
+pyyaml==3.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6"
+requests-file==1.5.1; python_version >= "3.6"
+requests-toolbelt==0.9.1; python_version >= "3.6"
+requests==2.14.2
+rsa==4.7.2; python_version >= "3.6" and python_version < "4"
+s3transfer==0.1.13; python_version >= "3.6"
+setuptools==39.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
+six==1.11.0
+texttable==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+tldextract==3.1.0; python_version >= "3.6"
+toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0"
+typed-ast==1.4.3; python_version >= "3.6" and python_version < "3.8" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
+typing-extensions==3.10.0.0; python_version >= "3.6" or python_version < "3.8" and python_version >= "3.6"
+uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+urllib3==1.10.2
+websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+wheel==0.33.6; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
+wrapt==1.12.1; python_version >= "3.6" and python_version < "4.0"
+zipp==3.4.1; python_version < "3.8" and python_version >= "3.6"
+zope.component==4.1.0
+zope.event==4.0.3
+zope.hookable==4.0.4
+zope.interface==4.0.5

tools/pinning/DESIGN.md

@@ -0,0 +1,125 @@
+# Certbot dependency pinning
+
+As described in the developer guide, we try to pin Certbot's dependencies to
+well tested versions in almost all cases. Pinning Python dependencies across
+different environments like this is actually quite tricky though and the files
+under this directory make a somewhat reasonable, best effort approach to solve
+this problem.
+
+## Python packaging background
+
+### Sdists and wheels
+
+Python projects are most commonly distributed on [PyPI](https://pypi.org/) as
+either a [source
+distribution](https://packaging.python.org/glossary/#term-Source-Distribution-or-sdist)
+or a [wheel](https://packaging.python.org/glossary/#term-Wheel). Wheels don't
+present a problem for us pinning dependencies because they offer a well defined
+format where the dependencies of a package can be easily parsed.
+
+Source distributions or "sdists" are a problem though because they can contain
+arbitrary Python code that must be run to determine the dependencies of the
+package. This code could theoretically do anything, but it most commonly
+inspects the environment it is running in and changes the project's
+dependencies based the environment. As of writing this, we even do this in some
+of Certbot's packages as you can see
+[here](https://github.com/certbot/certbot/blob/8b610239bfcf7aac06f6e36d09a5abba3ba87047/certbot-dns-cloudflare/setup.py#L15-L27).
+This is a problem because it means the environment an sdist was run in affects
+the dependencies it declares making it difficult for us to determine Certbot's
+dependencies for an arbitrary environment. Luckily, this is becoming less and
+less of an issue with the increasing use of wheels, however, as of writing
+this, some of Certbot's dependencies are still only available as sdists on some
+platforms.
+
+### Environment markers and pyproject.toml
+
+Two other things have helped reduce the problems caused by sdists and are
+relevant here. The first is the usage of [environment
+markers](https://www.python.org/dev/peps/pep-0496/) which allows a package to
+consistently declare its conditional dependencies with a static string
+specifying the conditions where a dependency is required instead of dynamically
+generating the list of required dependencies at runtime. This static string
+keeps the package's declaration of its dependencies consistent across
+environments.
+
+The other relatively recent change in Python packaging is the adoption of
+[pyproject.toml files](https://www.python.org/dev/peps/pep-0518/) which allows
+sdists to define their packages using a static file instead of a setup.py
+file, which has historically been the norm.
+Using a static file instead of arbitrary Python code makes it
+much easier for package declarations to be reliably interpreted. The
+introduction of pyproject.toml also allows for the use of build systems other
+than setuptools which becomes relevant in the next section of this doc.
+
+## Our pinning system
+
+### Overview
+
+The files inside `tools/pinning` are used to generate Certbot's pinning files.
+The files under `oldest` are used to generate the constraints file used for our
+"oldest" tests while `current` is used to generate the constraints used
+everywhere else. `common` includes shared files that are used for both sets of
+pinnings.
+
+Under `current` and `oldest`, there are two files as of writing this. One is a
+pyproject.toml file for use with [Poetry](https://python-poetry.org/) while
+the other is a script that can be run to regenerate pinnings. The
+pyproject.toml file defines a Python package that depends on everything we want
+to pin. This file largely just depends on our own local packages, however,
+extra dependencies can be declared to further constrain package versions or to
+declare additional dependencies.
+
+The reason we use Poetry is that it is somewhat unique among Python packaging
+tools in that when locking dependencies, it makes a best effort approach to do
+this for all environments rather than just the current environment. This
+includes recursively resolving dependencies declared through environment
+markers that are not relevant for the current platform. It also includes
+checking all wheels and sdists of a package for dependencies when picking a
+specific version of a package from PyPI. You can see this in action through
+the inclusion of dependencies like pywin32 which we only have a dependency on
+for Windows.
+
+### Potential problems
+
+As of writing this, I'm aware of two potential problems with this pinning
+system. The first is largely described earlier in the doc which is the problem
+of sdists that use code to dynamically declare its dependencies. It's simply
+not feasible to ensure this arbitrary Python code declares its dependencies in
+the same way across all environments. Luckily, this is a largely a theoretical
+problem and I'm aware of no issues with our current dependencies.
+
+The second problem with this approach is that build dependencies are not
+tracked and pinned. Unfortunately, [this seems to be a largely unsolved problem
+in Python right
+now](https://discuss.python.org/t/pinning-build-dependencies/8363). Our tooling
+ensures that when installing build dependencies when using our pinning files
+that versions from the pinning files are used (see
+https://github.com/certbot/certbot/pull/8443 for more info about that), but I'm
+not aware of any tool that automates the process of tracking and pinning down
+build dependencies. For now, if we find any unpinned build dependencies, we can
+declare a dependency on them in pyproject.toml. If a build dependency isn't
+included in the constraints file, pip will use the latest version available on
+PyPI.
+
+## Theoretical future work
+
+I think the system described above should work pretty well and I think it's
+much better than the system we had before where how to update things like our
+"oldest" pinnings was an open question. If we wanted to improve on this in the
+future though, I think things to consider would be:
+
+1. We could require that wheels are used for all of our dependencies. If a
+   wheel is not available for one of our dependencies, we could try to work
+   with upstream to change that or build it and host it locally for ourselves.
+   (If we do the latter, how to pin build dependencies when building the wheel
+   remains an open question.)
+2. We could only really try to pin our dependencies for certain environments.
+   This would be done by doing something like installing our packages in each
+   environment we care about and saving the output of a command like `pip
+   freeze`. With our use of snaps and Docker, this may be somewhat reasonable
+   because we could base them all on a common system like Ubuntu LTS, however,
+   it's not entirely trivial because we still have problems such as supporting
+   multiple CPU architectures and pinning dependencies for Windows. Alternative
+   development and test environments also wouldn't be fully supported.
+3. We could help build better tooling that solves some of the problems with
+   this approach or adopts it when it becomes available.

tools/pinning/common/export-pinned-dependencies.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# This script accepts a directory containing a pyproject.toml file configured
+# for use with poetry and generates and prints the pinned dependencies of that
+# file. Any dependencies on acme or those referencing certbot will be removed
+# from the output. The exported requirements are printed to stdout.
+#
+# For example, if a directory containing a pyproject.toml file for poetry is at
+# ../current, you could activate Certbot's developer environment and then run a
+# command like the following to generate requirements.txt for that environment:
+#   ./export-pinned-dependencies.sh ../current > requirements.txt
+set -euo pipefail
+
+# If this script wasn't given a command line argument, print usage and exit.
+if [ -z ${1+x} ]; then
+    echo "Usage:" >&2
+    echo "$0 PYPROJECT_TOML_DIRECTORY" >&2
+    exit 1
+fi
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+WORK_DIR="$1"
+
+if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then
+    echo "Please install poetry 1.2+." >&2
+    echo "You may need to recreate Certbot's virtual environment and activate it." >&2
+    exit 1
+fi
+
+# Old eggs can cause outdated dependency information to be used by poetry so we
+# delete them before generating the lock file. See
+# https://github.com/python-poetry/poetry/issues/4103 for more info.
+rm -rf ${REPO_ROOT}/*.egg-info
+
+cd "${WORK_DIR}"
+
+if [ -f poetry.lock ]; then
+    rm poetry.lock
+fi
+
+poetry lock >&2
+trap 'rm poetry.lock' EXIT
+
+# We need to remove local packages from the output.
+poetry export --without-hashes | sed '/^acme @/d; /certbot/d;'

tools/pinning/current/pyproject.toml

@@ -12,28 +12,28 @@ python = "^3.6"
 # Any local packages that have dependencies on other local packages must be
 # listed below before the package it depends on. For instance, certbot depends
 # on acme so certbot must be listed before acme.
-certbot-ci = {path = "../../certbot-ci"}
-certbot-compatibility-test = {path = "../../certbot-compatibility-test"}
-certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]}
-certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]}
-certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]}
-certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]}
-certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]}
-certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]}
-certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]}
-certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]}
-certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]}
-certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]}
-certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]}
-certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]}
-certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]}
-certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]}
-certbot-nginx = {path = "../../certbot-nginx"}
-certbot-apache = {path = "../../certbot-apache", extras = ["dev"]}
-certbot = {path = "../../certbot", extras = ["all"]}
-acme = {path = "../../acme", extras = ["docs", "test"]}
-letstest = {path = "../../letstest"}
-windows-installer = {path = "../../windows-installer"}
+certbot-ci = {path = "../../../certbot-ci"}
+certbot-compatibility-test = {path = "../../../certbot-compatibility-test"}
+certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare", extras = ["docs"]}
+certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns", extras = ["docs"]}
+certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean", extras = ["docs"]}
+certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple", extras = ["docs"]}
+certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy", extras = ["docs"]}
+certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn", extras = ["docs"]}
+certbot-dns-google = {path = "../../../certbot-dns-google", extras = ["docs"]}
+certbot-dns-linode = {path = "../../../certbot-dns-linode", extras = ["docs"]}
+certbot-dns-luadns = {path = "../../../certbot-dns-luadns", extras = ["docs"]}
+certbot-dns-nsone = {path = "../../../certbot-dns-nsone", extras = ["docs"]}
+certbot-dns-ovh = {path = "../../../certbot-dns-ovh", extras = ["docs"]}
+certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136", extras = ["docs"]}
+certbot-dns-route53 = {path = "../../../certbot-dns-route53", extras = ["docs"]}
+certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud", extras = ["docs"]}
+certbot-nginx = {path = "../../../certbot-nginx"}
+certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]}
+certbot = {path = "../../../certbot", extras = ["all"]}
+acme = {path = "../../../acme", extras = ["docs", "test"]}
+letstest = {path = "../../../letstest"}
+windows-installer = {path = "../../../windows-installer"}
 
 # Extra dependencies
 # awscli is just listed here as a performance optimization. As of writing this,

tools/pinning/current/repin.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# This script accepts no arguments and automates the process of updating
+# Certbot's dependencies including automatically updating the correct file.
+# Dependencies can be pinned to older versions by modifying pyproject.toml in
+# the same directory as this file.
+set -euo pipefail
+
+WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+COMMON_DIR="$(dirname "${WORK_DIR}")/common"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")"
+REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt"
+
+PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}")
+cat << EOF > "$REQUIREMENTS_FILE"
+# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using
+# that script.
+#
+# It is normally used as constraints to pip, however, it has the name
+# requirements.txt so that is scanned by GitHub. See
+# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems
+# for more info.
+EOF
+echo "${PINNINGS}" >> "${REQUIREMENTS_FILE}"

tools/pinning/oldest/pyproject.toml

@@ -0,0 +1,145 @@
+[tool.poetry]
+name = "certbot-pinner"
+version = "0.1.0"
+description = "A simple project for pinning Certbot's dependencies using Poetry."
+authors = ["Certbot Project"]
+license = "Apache License 2.0"
+
+[tool.poetry.dependencies]
+# The Python version here should be kept in sync with the one used in our
+# oldest tests in tox.ini.
+python = "3.6"
+
+# Local dependencies
+# Any local packages that have dependencies on other local packages must be
+# listed below before the package it depends on. For instance, certbot depends
+# on acme so certbot must be listed before acme.
+certbot-ci = {path = "../../../certbot-ci"}
+certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare"}
+certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns"}
+certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean"}
+certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple"}
+certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy"}
+certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn"}
+certbot-dns-google = {path = "../../../certbot-dns-google"}
+certbot-dns-linode = {path = "../../../certbot-dns-linode"}
+certbot-dns-luadns = {path = "../../../certbot-dns-luadns"}
+certbot-dns-nsone = {path = "../../../certbot-dns-nsone"}
+certbot-dns-ovh = {path = "../../../certbot-dns-ovh"}
+certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136"}
+certbot-dns-route53 = {path = "../../../certbot-dns-route53"}
+certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud"}
+certbot-nginx = {path = "../../../certbot-nginx"}
+certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]}
+certbot = {path = "../../../certbot", extras = ["test"]}
+acme = {path = "../../../acme", extras = ["test"]}
+
+# Oldest dependencies
+# We specify the oldest versions our dependencies that we're trying to keep
+# support for below. Usually these version numbers are taken from the packages
+# of our dependencies available in popular LTS Linux distros. Keeping
+# compatibility with those versions makes it much easier for OS maintainers to
+# update their Certbot packages.
+#
+# When updating these dependencies, we should ideally try to only update them
+# to the oldest version of the dependency that is found in a non-EOL'd version
+# of CentOS, Debian, or Ubuntu that has Certbot packages in their OS
+# repositories using a version of Python we support. If the distro is EOL'd or
+# using a version of Python we don't support, it can be ignored. If the
+# dependency being updated is a direct dependency of one of our own packages,
+# the minimum required version of that dependency should be updated in our
+# setup.py files as well to communicate this information to our users.
+
+# CentOS/RHEL 7 EPEL dependencies
+# Some of these dependencies may be stricter than necessary because they
+# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL.
+cffi = "1.9.1"
+chardet = "2.2.1"
+ipaddress = "1.0.16"
+mock = "1.0.1"
+ndg-httpsclient = "0.3.2"
+ply = "3.4"
+pyOpenSSL = "17.3.0"
+pyasn1 = "0.1.9"
+pycparser = "2.14"
+pyRFC3339 = "1.0"
+python-augeas = "0.5.0"
+oauth2client = "4.0.0"
+requests = "2.14.2"
+urllib3 = "1.10.2"
+# Package names containing "." need to be quoted.
+"zope.component" = "4.1.0"
+"zope.event" = "4.0.3"
+"zope.interface" = "4.0.5"
+
+# Debian Jessie Backports dependencies
+# Debian Jessie has reached end of life so these dependencies can probably be
+# updated as needed or desired.
+pbr = "1.8.0"
+pytz = "2012rc0"
+
+# Debian Buster dependencies
+google-api-python-client = "1.5.5"
+pyparsing = "2.2.0"
+
+# Our setup.py dependencies
+apacheconfig = "0.3.2"
+cloudflare = "1.5.1"
+python-digitalocean = "1.11"
+
+# Ubuntu Xenial dependencies
+# Ubuntu Xenial only has versions of Python which we do not support available
+# so these dependencies can probably be updated as needed or desired.
+ConfigArgParse = "0.10.0"
+funcsigs = "0.4"
+# Package names containing "." need to be quoted.
+"zope.hookable" = "4.0.4"
+
+# Ubuntu Bionic dependencies.
+cryptography = "2.1.4"
+distro = "1.0.1"
+httplib2 = "0.9.2"
+idna = "2.6"
+setuptools = "39.0.1"
+six = "1.11.0"
+
+# Ubuntu Focal dependencies
+asn1crypto = "0.24.0"
+configobj = "5.0.6"
+parsedatetime = "2.4"
+
+# Plugin dependencies
+# These aren't necessarily the oldest versions we need to support
+# Tracking at https://github.com/certbot/certbot/issues/6473
+boto3 = "1.4.7"
+botocore = "1.7.41"
+dns-lexicon = "3.2.1"
+
+# Build dependencies
+# Since there doesn't appear to
+# doesn't appear to be a good way to automatically track down and pin build
+# dependencies in Python (see
+# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list any
+# build dependencies here to ensure they're pinned for extra stability.
+
+# cython is a build dependency of pyyaml
+cython = "*"
+
+# Other dependencies
+# We add any dependencies that must be specified in this file for any another
+# reason below.
+
+# pip's new dependency resolver fails on local packages that depend on each
+# other when those packages are requested with extras such as 'certbot[dev]' so
+# let's pin it back for now. See https://github.com/pypa/pip/issues/9204.
+pip = "20.2.4"
+
+# wheel 0.34.1+ does not support the version of setuptools pinned above (and
+# wheel 0.34.0 is buggy).
+wheel = "<0.34.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tools/pinning/oldest/repin.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script accepts no arguments and automates the process of updating
+# Certbot's dependencies including automatically updating the correct file.
+# Dependencies can be pinned to older versions by modifying pyproject.toml in
+# the same directory as this file.
+set -euo pipefail
+
+WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+COMMON_DIR="$(dirname "${WORK_DIR}")/common"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")"
+CONSTRAINTS_FILE="$REPO_ROOT/tools/oldest_constraints.txt"
+
+PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}")
+cat << EOF > "$CONSTRAINTS_FILE"
+# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using
+# that script.
+EOF
+echo "${PINNINGS}" >> "${CONSTRAINTS_FILE}"