Reorganize the pipeline

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -58,7 +58,7 @@ jobs:
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
         displayName: Run certbot integration tests
-  - job: snap_build
+  - job: snaps_build
     pool:
       vmImage: ubuntu-18.04
     steps:
@@ -67,6 +67,10 @@ jobs:
           sudo apt-get install -y --no-install-recommends snapd
           sudo snap install --classic snapcraft
         displayName: Install dependencies
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.8
+          addToPath: true
       - task: DownloadSecureFile@1
         name: credentials
         inputs:
@@ -76,42 +80,17 @@ jobs:
           git config --global user.name "$(Build.RequestedFor)"
           mkdir -p ~/.local/share/snapcraft/provider/launchpad
           cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
-          tools/snap/build_remote.sh amd64,arm64,armhf
+          python3 tools/snap/build_remote.py ALL --archs amd64 arm64 armhf
           mv *.snap $(Build.ArtifactStagingDirectory)
-        displayName: Build Certbot snap
+          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
+        displayName: Build Certbot + DNS plugins snaps
       - task: PublishPipelineArtifact@1
         inputs:
           path: $(Build.ArtifactStagingDirectory)
           artifact: snaps
         displayName: Store snaps artifacts
-  - job: snap_dns_build
-    pool:
-      vmImage: ubuntu-18.04
-    steps:
-      - script: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends snapd
-          sudo snap install --classic snapcraft
-        displayName: Install dependencies
-      - task: DownloadSecureFile@1
-        name: credentials
-        inputs:
-          secureFile: credentials
-      - script: |
-          git config --global user.email "$(Build.RequestedForEmail)"
-          git config --global user.name "$(Build.RequestedFor)"
-          mkdir -p ~/.local/share/snapcraft/provider/launchpad
-          cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
-          tools/snap/build_dns_remote.sh amd64,arm64,armhf ALL
-          mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
-        displayName: Build Certbot DNS snaps
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          artifact: dns-snaps
-        displayName: Store snaps artifacts
   - job: snap_run
-    dependsOn: snap_build
+    dependsOn: snaps_build
     pool:
       vmImage: ubuntu-18.04
     steps:
@@ -132,9 +111,7 @@ jobs:
           python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
-    dependsOn:
-      - snap_build
-      - snap_dns_build
+    dependsOn: snaps_build
     pool:
       vmImage: ubuntu-18.04
     steps:
@@ -151,11 +128,6 @@ jobs:
           artifact: snaps
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snap
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: dns-snaps
-          path: $(Build.SourcesDirectory)/snap
-        displayName: Retrieve Certbot DNS plugins snaps
       - script: |
           python3 -m venv venv
           venv/bin/python tools/pip_install.py -e certbot-ci

.azure-pipelines/templates/stages/deploy-stage.yml

@@ -24,11 +24,6 @@ stages:
               artifact: snaps
               path: $(Build.SourcesDirectory)/snap
             displayName: Retrieve Certbot snaps
-          - task: DownloadPipelineArtifact@2
-            inputs:
-              artifact: dns-snaps
-              path: $(Build.SourcesDirectory)/snap
-            displayName: Retrieve DNS plugins snaps
           - task: DownloadSecureFile@1
             name: snapcraftCfg
             inputs:

tools/snap/build_remote.py

@@ -69,7 +69,7 @@ def _dump_results(targets, archs, results):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('targets', nargs='+', choices=['certbot', *PLUGINS],
+    parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS],
                         help='the list of snaps to build')
     parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default='amd64',
                         help='the architectures for which snaps are built')
@@ -78,6 +78,14 @@ def main():
     archs = set(args.archs)
     targets = set(args.targets)
 
+    if 'ALL' in targets:
+        targets.remove('ALL')
+        targets.update(['certbot', 'DNS_PLUGINS'])
+
+    if 'DNS_PLUGINS' in targets:
+        targets.remove('DNS_PLUGINS')
+        targets.update(PLUGINS)
+
     pool = multiprocessing.Pool(processes=len(targets))
     async_results = [pool.apply_async(_build_snap, (target, archs)) for target in targets]