Integrate DNS snap tests into certbot-ci

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -151,8 +151,11 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot DNS plugins snaps
       - script: |
-          sudo tools/pip_install.py pytest
-          sudo -E pytest tests/certbot_dns_snap_test.py
+          python3 -m venv venv
+          venv/Scripts/python tools/pip_install.py -e certbot-ci
+        displayName: Prepare Certbot-CI
+      - script: |
+          python3 -m venv venv
+          venv/Scripts/python tools/pip_install.py -e certbot-ci
+          sudo -E venv/Scripts/python -m pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap
         displayName: Test DNS plugins snaps
-        env:
-          SNAP_FOLDER: $(Build.SourcesDirectory)/snap

certbot-ci/snap_integration_tests/conftest.py

@@ -0,0 +1,40 @@
+"""
+General conftest for pytest execution of all integration tests lying
+in the window_installer_integration tests package.
+As stated by pytest documentation, conftest module is used to set on
+for a directory a specific configuration using built-in pytest hooks.
+
+See https://docs.pytest.org/en/latest/reference.html#hook-reference
+"""
+import glob
+import os
+
+
+def pytest_addoption(parser):
+    """
+    Standard pytest hook to add options to the pytest parser.
+    :param parser: current pytest parser that will be used on the CLI
+    """
+    parser.addoption('--snap-folder', required=True,
+                     help='set the folder path where snaps to test are located')
+    parser.addoption('--allow-persistent-changes', action='store_true',
+                     help='needs to be set, and confirm that the test will make persistent changes on this machine')
+
+
+def pytest_configure(config):
+    """
+    Standard pytest hook used to add a configuration logic for each node of a pytest run.
+    :param config: the current pytest configuration
+    """
+    if not config.option.allow_persistent_changes:
+        raise RuntimeError('This integration test would install Certbot snap on your machine. '
+                           'Please run it again with the `--allow-persistent-changes` flag set to acknowledge.')
+
+
+def pytest_generate_tests(metafunc):
+    """
+    Generate (multiple) parametrized calls to a test function.
+    """
+    if "dns_snap_path" in metafunc.fixturenames:
+        snap_dns_path_list = glob.glob(os.path.join(metafunc.config.snap_folder, 'certbot-dns-*_*.snap'))
+        metafunc.parametrize("snap_dns_path", snap_dns_path_list)

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -6,33 +6,12 @@ import os
 import re
 
 
-def _get_snap_directory():
-    snap_directory = os.environ.get('SNAP_FOLDER')
-    if not snap_directory:
-        raise ValueError('Error, SNAP_FOLDER environment variable is not set.')
-
-    return snap_directory
-
-
-def _list_dns_snaps_paths():
-    snap_directory = _get_snap_directory()
-    if not snap_directory:
-        raise ValueError('Error, SNAP_FOLDER environment variable is not set.')
-
-    return glob.glob(os.path.join(snap_directory, 'certbot-dns-*_*.snap'))
-
-
-def _extract_plugin_name(dns_snap_path):
-    return re.match(r'^certbot-(dns-\w+)_.*\.snap$', os.path.basename(dns_snap_path)).group(1)
-
-
 @pytest.fixture(autouse=True, scope="module")
-def install_certbot_snap():
+def install_certbot_snap(request):
     with pytest.raises(Exception):
         subprocess.check_call(['certbot', '--version'])
     try:
-        snap_directory = _get_snap_directory()
-        snap_path = glob.glob(os.path.join(snap_directory, 'certbot_*.snap'))[0]
+        snap_path = glob.glob(os.path.join(request.config.snap_folder, 'certbot_*.snap'))[0]
         subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path])
         subprocess.check_call(['certbot', '--version'])
         yield
@@ -40,13 +19,13 @@ def install_certbot_snap():
         subprocess.call(['snap', 'remove', 'certbot'])
 
 
-@pytest.mark.parametrize('dns_snap_path', _list_dns_snaps_paths())
-def test_it(dns_snap_path):
+def test_dns_plugin_install(dns_snap_path):
     """
     Test that each DNS plugin Certbot snap available in SNAP_FOLDER
     can be installed and is usable with the Certbot snap.
     """
-    plugin_name = _extract_plugin_name(dns_snap_path)
+    plugin_name = re.match(r'^certbot-(dns-\w+)_.*\.snap$',
+                           os.path.basename(dns_snap_path)).group(1)
     snap_name = 'certbot-{0}'.format(plugin_name)
     assert plugin_name not in subprocess.check_output(['certbot', 'plugins', '--prepare'],
                                                       universal_newlines=True)