diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml
index ebadcb2dc..6d6fcfff2 100644
--- a/.azure-pipelines/templates/installer-tests.yml
+++ b/.azure-pipelines/templates/installer-tests.yml
@@ -49,6 +49,7 @@ jobs:
         displayName: Retrieve Windows installer
       - script: |
           py -3 -m venv venv
+          venv\Scripts\python -m pip install --upgrade pip setuptools
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py
index 9737fbe7a..208405a34 100644
--- a/certbot-ci/windows_installer_integration_tests/test_main.py
+++ b/certbot-ci/windows_installer_integration_tests/test_main.py
@@ -12,9 +12,7 @@ import shutil
 import warnings
 
 import pytest
-with warnings.catch_warnings():
-    warnings.simplefilter("ignore")
-    import pkg_resources
+import pkg_resources
 
 SCHEDULED_TASK_NAME = 'Certbot Renew and Auto-Update Task'
 GITHUB_FAKE_API_PORT = 8009