Setup pipeline to use remote builds

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -59,50 +59,54 @@ jobs:
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
         displayName: Run certbot integration tests
   - job: snap_build
-    strategy:
-      matrix:
-        amd64:
-          ARCH: amd64
-        # Do not run the QEMU jobs for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm64:
-            ARCH: arm64
-          armhf:
-            ARCH: armhf
     pool:
       vmImage: ubuntu-18.04
+    variables:
+      - group: certbot-common
     steps:
       - script: |
-          tools/snap/build.sh ${ARCH}
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends snapd
+        displayName: Install dependencies
+      - task: DownloadSecureFile@1
+        name: snapcraftCfg
+        inputs:
+          secureFile: snapcraft.cfg
+      - script: |
+          mkdir -p .snapcraft
+          ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
+          tools/snap/build_remote.sh amd64,arm64,armhf
           mv *.snap $(Build.ArtifactStagingDirectory)
         displayName: Build Certbot snap
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          artifact: snap-$(arch)
-        displayName: Store snap artifact
+          artifact: snaps
+        displayName: Store snaps artifacts
   - job: snap_dns_build
-    strategy:
-      matrix:
-        amd64:
-          ARCH: amd64
-        # Do not run the QEMU jobs for test branches
-        ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
-          arm64:
-            ARCH: arm64
-          armhf:
-            ARCH: armhf
     pool:
       vmImage: ubuntu-18.04
+    variables:
+      - group: certbot-common
     steps:
       - script: |
-          tools/snap/build_dns.sh ${ARCH} ALL
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends snapd
+        displayName: Install dependencies
+      - task: DownloadSecureFile@1
+        name: snapcraftCfg
+        inputs:
+          secureFile: snapcraft.cfg
+      - script: |
+          mkdir -p .snapcraft
+          ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
+          tools/snap/build_dns_remote.sh amd64,arm64,armhf ALL
           mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
         displayName: Build Certbot DNS snaps
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
-          artifact: dns-snap-$(arch)
+          artifact: dns-snaps
         displayName: Store snaps artifacts
   - job: snap_run
     dependsOn: snap_build
@@ -116,11 +120,11 @@ jobs:
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snap-amd64
+          artifact: snaps
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snap
       - script: |
-          sudo snap install --dangerous --classic snap/*.snap
+          sudo snap install --dangerous --classic snap/*_amd64.snap
         displayName: Install Certbot snap
       - script: |
           python -m tox -e integration-external,apacheconftest-external-with-pebble
@@ -142,12 +146,12 @@ jobs:
           addToPath: true
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: snap-amd64
+          artifact: snaps
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snap
       - task: DownloadPipelineArtifact@2
         inputs:
-          artifact: dns-snap-amd64
+          artifact: dns-snaps
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot DNS plugins snaps
       - script: |
@@ -155,5 +159,5 @@ jobs:
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
-          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap
+          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -9,14 +9,6 @@ stages:
       # prevent automated deploys from breaking. Remembering to do this is
       # also tracked by https://github.com/certbot/certbot/issues/7931.
       - job: publish_snap
-        strategy:
-          matrix:
-            amd64:
-              ARCH: amd64
-            arm64:
-              ARCH: arm64
-            armhf:
-              ARCH: armhf
         pool:
           vmImage: ubuntu-18.04
         variables:
@@ -29,12 +21,12 @@ stages:
             displayName: Install dependencies
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: snap-$(arch)
+              artifact: snaps
               path: $(Build.SourcesDirectory)/snap
-            displayName: Retrieve Certbot snap
+            displayName: Retrieve Certbot snaps
           - task: DownloadPipelineArtifact@2
             inputs:
-              artifact: dns-snap-$(arch)
+              artifact: dns-snaps
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve DNS plugins snaps
           - task: DownloadSecureFile@1

certbot-ci/snap_integration_tests/conftest.py

@@ -17,6 +17,8 @@ def pytest_addoption(parser):
     """
     parser.addoption('--snap-folder', required=True,
                      help='set the folder path where snaps to test are located')
+    parser.addoption('--snap-arch', default='amd64',
+                    help='set the architecture do test (default: amd64)')
     parser.addoption('--allow-persistent-changes', action='store_true',
                      help='needs to be set, and confirm that the test will make persistent changes on this machine')
 
@@ -36,5 +38,8 @@ def pytest_generate_tests(metafunc):
     Generate (multiple) parametrized calls to a test function.
     """
     if "dns_snap_path" in metafunc.fixturenames:
-        snap_dns_path_list = glob.glob(os.path.join(metafunc.config.getoption('snap_folder'), 'certbot-dns-*_*.snap'))
+        snap_arch = metafunc.config.getoption('snap_arch')
+        snap_folder = metafunc.config.getoption('snap_folder')
+        snap_dns_path_list = glob.glob(os.path.join(snap_folder, 
+                                                    'certbot-dns-*_{0}.snap'.format(snap_arch)))
         metafunc.parametrize("dns_snap_path", snap_dns_path_list)

certbot-ci/snap_integration_tests/dns_tests/test_main.py

@@ -11,8 +11,9 @@ def install_certbot_snap(request):
     with pytest.raises(Exception):
         subprocess.check_call(['certbot', '--version'])
     try:
-        snap_path = glob.glob(os.path.join(request.config.getoption("snap_folder"),
-                                           'certbot_*.snap'))[0]
+        snap_folder = request.config.getoption("snap_folder")
+        snap_arch = request.config.getoption("snap_arch")
+        snap_path = glob.glob(os.path.join(snap_folder, 'certbot_*_{0}.snap'.format(snap_arch)))[0]
         subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path])
         subprocess.check_call(['certbot', '--version'])
         yield

tools/snap/build_dns_remote.sh

@@ -1,22 +1,26 @@
 #!/bin/bash
 set -ex
 
-DNS_PLUGINS=$1
-SNAP_ARCHS=$2
-
-if [[ -z "${DNS_PLUGINS}" ]]; then
-    echo "You need to specify the DNS plugins"
-    exit 1
-fi
+SNAP_ARCHS=$1
+DNS_PLUGINS=$2
 
 if [[ -z "${SNAP_ARCHS}" ]]; then
     echo "You need to specify at least one target architecture"
     exit 1
 fi
 
+if [[ -z "${DNS_PLUGINS}" ]]; then
+    echo "You need to specify the DNS plugins"
+    exit 1
+fi
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")"
 
+if [[ "${DNS_PLUGINS}" = "ALL" ]]; then
+    DNS_PLUGINS=$(find . -maxdepth 1 -type d -name "${CERTBOT_DIR}/certbot-dns-*" -exec basename {} \; | paste -sd "," -)
+fi
+
 trap popd EXIT
 
 IFS=","