Configurable installer path, prepare azure pipelines

.azure-pipelines/templates/installer-tests.yml

@@ -33,22 +33,24 @@ jobs:
     pool:
       vmImage: $(imageName)
     steps:
+      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
+        displayName: Get Python
+      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
+        displayName: Install Python
       - task: DownloadPipelineArtifact@2
         inputs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
-      - script: $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe /S
-        displayName: Install Certbot
-      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
-        displayName: Get Python
-      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
-        displayName: Install Python
       - script: |
           py -3 -m venv venv
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
+      - script: |
+          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
+        displayName: Run windows installer integration tests
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run integration tests
+        displayName: Run certbot integration tests

certbot-ci/windows_installer_integration_tests/conftest.py

@@ -0,0 +1,24 @@
+"""
+General conftest for pytest execution of all integration tests lying
+in the window_installer_integration tests package.
+As stated by pytest documentation, conftest module is used to set on
+for a directory a specific configuration using built-in pytest hooks.
+
+See https://docs.pytest.org/en/latest/reference.html#hook-reference
+"""
+from __future__ import print_function
+import os
+
+ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+
+
+def pytest_addoption(parser):
+    """
+    Standard pytest hook to add options to the pytest parser.
+    :param parser: current pytest parser that will be used on the CLI
+    """
+    parser.addoption('--installer-path',
+                     default=os.path.join(ROOT_PATH, 'windows-installer', 'build',
+                                          'nsis', 'certbot-beta-installer-win32.exe'),
+                     help='set the path of the windows installer to use, default to '
+                          'CERTBOT_ROOT_PATH\\windows-installer\\build\\nsis\\certbot-beta-installer-win32.exe')

certbot-ci/windows_installer_integration_tests/test_main.py

@@ -2,11 +2,10 @@ import os
 import time
 import unittest
 import subprocess
-import sys
 
 
 @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')
-def test_it():
+def test_it(request):
     try:
         subprocess.check_call(['certbot', '--version'])
     except (subprocess.CalledProcessError, OSError):
@@ -14,14 +13,9 @@ def test_it():
     else:
         raise AssertionError('Expect certbot to not be available in the PATH.')
 
-    root_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-
     try:
-        # Build the installer
-        subprocess.check_call([sys.executable, os.path.join(root_path, 'windows-installer', 'construct.py')])
-
         # Install certbot
-        subprocess.check_call([os.path.join(root_path, 'windows-installer', 'build', 'nsis', 'certbot-beta-installer-win32.exe'), '/S'])
+        subprocess.check_call([request.config.installer_path, '/S'])
 
         # Assert certbot is installed and runnable
         output = subprocess.check_output(['certbot', '--version'], universal_newlines=True)