Build the DNS plugins snaps (#8129)

Fixes #8041

This PR makes Azure Pipeline build the DNS plugins snaps for the 3 architectures during the CI.

It leverages the existing logic for building the Certbot snap in order to deploy a QEMU environment with Docker, and leverages the local PyPI index to speed up the build when installing `cffi` and `cryptography`.

All DNS plugins snaps are constructed in one unique docker container, in order to save the time required to install the system dependencies upon first start of `snapcraft`, and so speed up significantly the build.

Finally, all `amd64` DNS plugins snaps are built within 6 minutes. For `arm64` and `armhf`, it is around 40 mins: this is quite fast in fact, considering that 14 DNS plugins snaps are built.

However, this is still an extremely heavy task to make the full 3 architectures builds, even for Azure Pipelines and its 10 parallel jobs capability. That is why I make the `arm64` and `armhf` builds be skipped for the `full-test-suite`, and let them run only for `nightly` and `release`. This means however that these builds will not be done for the release branches. If this is a problem, I can put a more elaborate suspend condition to triggers the builds in this case.

All snaps are stored in the pipeline artifacts storage, making them available for publication during a `release` pipeline.

The PR is set as Draft for now, because I use temporarily `pr_test-suite` to validate the packaging jobs when commits are pushed. Once the PR is ready, I will revert it back to the normal configuration (run the standard tests).

* Configure a script to build DNS snaps

* Focus on packaging

* Trigger all architectures

* Add extra index

* Prepare conditional suspend

* Set final suspend logic

* Set final suspend value

* Loop for publication

* Use python3

* Clean before build

* Add a test

* Add test job in Azure

* Preserve env

* Apply normal config for pipelines

* Skip QEMU jobs only for test branches

* Makes snap run tests depends also on the Certbot snap build

* Update .azure-pipelines/templates/jobs/packaging-jobs.yml

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* Update .azure-pipelines/templates/stages/deploy-stage.yml

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* More accurate way to get the plugin snap name

* Integrate DNS snap tests into certbot-ci

* Fixes

* Update certbot-ci/snap_integration_tests/conftest.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* Update certbot-ci/snap_integration_tests/conftest.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* Clean an _init_.py file

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -63,15 +63,17 @@ jobs:
       matrix:
         amd64:
           ARCH: amd64
-        arm64:
-          ARCH: arm64
-        armhf:
-          ARCH: armhf
+        # Do not run the QEMU jobs for test branches
+        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+          arm64:
+            ARCH: arm64
+          armhf:
+            ARCH: armhf
     pool:
       vmImage: ubuntu-18.04
     steps:
       - script: |
-          snap/local/build.sh ${ARCH}
+          tools/snap/build.sh ${ARCH}
           mv *.snap $(Build.ArtifactStagingDirectory)
         displayName: Build Certbot snap
       - task: PublishPipelineArtifact@1
@@ -79,6 +81,29 @@ jobs:
           path: $(Build.ArtifactStagingDirectory)
           artifact: snap-$(arch)
         displayName: Store snap artifact
+  - job: snap_dns_build
+    strategy:
+      matrix:
+        amd64:
+          ARCH: amd64
+        # Do not run the QEMU jobs for test branches
+        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
+          arm64:
+            ARCH: arm64
+          armhf:
+            ARCH: armhf
+    pool:
+      vmImage: ubuntu-18.04
+    steps:
+      - script: |
+          tools/snap/build_dns.sh ${ARCH} ALL
+          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
+        displayName: Build Certbot DNS snaps
+      - task: PublishPipelineArtifact@1
+        inputs:
+          path: $(Build.ArtifactStagingDirectory)
+          artifact: dns-snap-$(arch)
+        displayName: Store snaps artifacts
   - job: snap_run
     dependsOn: snap_build
     pool:
@@ -100,3 +125,35 @@ jobs:
       - script: |
           python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
+  - job: snap_dns_run
+    dependsOn:
+      - snap_build
+      - snap_dns_build
+    pool:
+      vmImage: ubuntu-18.04
+    steps:
+      - script: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends snapd
+        displayName: Install dependencies
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.8
+          addToPath: true
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifact: snap-amd64
+          path: $(Build.SourcesDirectory)/snap
+        displayName: Retrieve Certbot snap
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifact: dns-snap-amd64
+          path: $(Build.SourcesDirectory)/snap
+        displayName: Retrieve Certbot DNS plugins snaps
+      - script: |
+          python3 -m venv venv
+          venv/bin/python tools/pip_install.py -e certbot-ci
+        displayName: Prepare Certbot-CI
+      - script: |
+          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap
+        displayName: Test DNS plugins snaps

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -32,6 +32,11 @@ stages:
               artifact: snap-$(arch)
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve Certbot snap
+          - task: DownloadPipelineArtifact@2
+            inputs:
+              artifact: dns-snap-$(arch)
+              path: $(Build.SourcesDirectory)/snap
+            displayName: Retrieve DNS plugins snaps
           - task: DownloadSecureFile@1
             name: snapcraftCfg
             inputs:
@@ -39,5 +44,7 @@ stages:
           - bash: |
               mkdir -p .snapcraft
               ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
-              snapcraft upload --release=edge snap/*.snap
+              for SNAP_FILE in snap/*.snap; do
+                snapcraft upload --release=edge "${SNAP_FILE}"
+              done
             displayName: Publish to Snap store

certbot-ci/snap_integration_tests/conftest.py

@@ -0,0 +1,40 @@
+"""
+General conftest for pytest execution of all integration tests lying
+in the snap_installer_integration tests package.
+As stated by pytest documentation, conftest module is used to set on
+for a directory a specific configuration using built-in pytest hooks.
+
+See https://docs.pytest.org/en/latest/reference.html#hook-reference
+"""
+import glob
+import os
+
+
+def pytest_addoption(parser):
+    """
+    Standard pytest hook to add options to the pytest parser.
+    :param parser: current pytest parser that will be used on the CLI
+    """
+    parser.addoption('--snap-folder', required=True,
+                     help='set the folder path where snaps to test are located')
+    parser.addoption('--allow-persistent-changes', action='store_true',
+                     help='needs to be set, and confirm that the test will make persistent changes on this machine')
+
+
+def pytest_configure(config):
+    """
+    Standard pytest hook used to add a configuration logic for each node of a pytest run.
+    :param config: the current pytest configuration
+    """
+    if not config.option.allow_persistent_changes:
+        raise RuntimeError('This integration test would install the Certbot snap on your machine. '
+                           'Please run it again with the `--allow-persistent-changes` flag set to acknowledge.')
+
+
+def pytest_generate_tests(metafunc):
+    """
+    Generate (multiple) parametrized calls to a test function.
+    """
+    if "dns_snap_path" in metafunc.fixturenames:
+        snap_dns_path_list = glob.glob(os.path.join(metafunc.config.getoption('snap_folder'), 'certbot-dns-*_*.snap'))
+        metafunc.parametrize("dns_snap_path", snap_dns_path_list)

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import pytest
+import subprocess
+import glob
+import os
+import re
+
+
+@pytest.fixture(autouse=True, scope="module")
+def install_certbot_snap(request):
+    with pytest.raises(Exception):
+        subprocess.check_call(['certbot', '--version'])
+    try:
+        snap_path = glob.glob(os.path.join(request.config.getoption("snap_folder"),
+                                           'certbot_*.snap'))[0]
+        subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path])
+        subprocess.check_call(['certbot', '--version'])
+        yield
+    finally:
+        subprocess.call(['snap', 'remove', 'certbot'])
+
+
+def test_dns_plugin_install(dns_snap_path):
+    """
+    Test that each DNS plugin Certbot snap can be installed
+    and is usable with the Certbot snap.
+    """
+    plugin_name = re.match(r'^certbot-(dns-\w+)_.*\.snap$',
+                           os.path.basename(dns_snap_path)).group(1)
+    snap_name = 'certbot-{0}'.format(plugin_name)
+    assert plugin_name not in subprocess.check_output(['certbot', 'plugins', '--prepare'],
+                                                      universal_newlines=True)
+
+    try:
+        subprocess.check_call(['snap', 'install', '--dangerous', dns_snap_path])
+        subprocess.check_call(['snap', 'set', 'certbot', 'trust-plugin-with-root=ok'])
+        subprocess.check_call(['snap', 'connect', 'certbot:plugin', snap_name])
+
+        assert plugin_name in subprocess.check_output(['certbot', 'plugins', '--prepare'],
+                                                      universal_newlines=True)
+    finally:
+        subprocess.call(['snap', 'remove', 'plugin_name'])

tools/snap/build.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-# Cross-compile the Certbot snap from local sources for the specified architecture,
-# and install it if this architecture is also the the current machine one.
+# Cross-compile the Certbot snap from local sources for the specified architecture.
 # This script is designed for CI tests purpose.
 # Usage: build.sh [amd64,arm64,armhf]
 set -ex

tools/snap/build_dns.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Cross-compile the specified Certbot DNS plugins snaps from local sources for the specified architecture.
+# This script is designed for CI tests purpose.
+# Usage: build.sh [amd64,arm64,armhf] [DNS_PLUGIN1,DNS_PLUGIN2 or ALL]
+set -ex
+
+SNAP_ARCH=$1
+DNS_PLUGINS=$2
+
+if [[ -z "${SNAP_ARCH}" ]]; then
+    echo "You need to specify the target architecture"
+    exit 1
+fi
+
+if [[ -z "${DNS_PLUGINS}" ]]; then
+    echo "You need to specify the DNS plugins"
+    exit 1
+fi
+
+if [[ "${DNS_PLUGINS}" = "ALL" ]]; then
+    DNS_PLUGINS=$(find . -maxdepth 1 -type d -name "certbot-dns-*" -exec basename {} \; | paste -sd "," -)
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")"
+
+# shellcheck source=common.sh
+source "${DIR}/common.sh"
+
+RegisterQemuHandlers
+ResolveArch "${SNAP_ARCH}"
+
+pushd "${DIR}/packages"
+"${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 &
+HTTP_SERVER_PID="$!"
+popd
+
+function cleanup() {
+    kill "${HTTP_SERVER_PID}"
+}
+
+trap cleanup EXIT
+
+SCRIPT=$(mktemp /tmp/script.XXXXXX.sh)
+chmod +x "${SCRIPT}"
+
+SNAP_CONSTRAINTS=$(mktemp /tmp/snap-constraints.XXXXXX.txt)
+python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > "${SNAP_CONSTRAINTS}"
+
+cat << "EOF" >> "${SCRIPT}"
+#!/bin/bash
+set -ex
+IFS=","
+for DNS_PLUGIN in ${DNS_PLUGINS}; do
+  pushd "${DNS_PLUGIN}"
+  cp /snap-constraints.txt .
+  snapcraft clean
+  snapcraft
+  popd
+done
+EOF
+
+docker run \
+  --rm \
+  --net=host \
+  -v "${CERTBOT_DIR}:/certbot" \
+  -v "${SCRIPT}:/script.sh" \
+  -v "${SNAP_CONSTRAINTS}:/snap-constraints.txt" \
+  -w "/certbot" \
+  -e "DNS_PLUGINS=${DNS_PLUGINS}" \
+  -e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \
+  "adferrand/snapcraft:${DOCKER_ARCH}-stable" \
+  /script.sh