From 5473f173ae7ab096957382a8f242cff25cf2c933 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 14 May 2026 17:58:00 -0700 Subject: [PATCH 1/4] Migrate release pipeline from azure to github actions --- .github/workflows/create_changelog.yml | 4 +- .github/workflows/notify_release.yml | 40 ++++++++ .github/workflows/release.yml | 133 +++++++++++++++++++++++++ tools/notify_mattermost.py | 98 ------------------ tools/release_message.py | 66 ++++++++++++ 5 files changed, 241 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/notify_release.yml create mode 100644 .github/workflows/release.yml delete mode 100755 tools/notify_mattermost.py create mode 100755 tools/release_message.py diff --git a/.github/workflows/create_changelog.yml b/.github/workflows/create_changelog.yml index 507f5c074..efa447955 100644 --- a/.github/workflows/create_changelog.yml +++ b/.github/workflows/create_changelog.yml @@ -9,7 +9,7 @@ jobs: name: Changelog runs-on: ubuntu-latest steps: - # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py + # If we change the output filename from `release_notes.md`, it should also be changed in .github/workflows/release.yml - name: checkout uses: actions/checkout@v6.0.2 with: @@ -22,6 +22,6 @@ jobs: - name: Publish changelog uses: actions/upload-artifact@v7.0.0 with: - # If we change the artifact's name, it should also be changed in tools/create_github_release.py + # If we change the artifact's name, it should also be changed in .github/workflows/release.yml name: changelog path: "${{ runner.temp }}/release_notes.md" diff --git a/.github/workflows/notify_release.yml b/.github/workflows/notify_release.yml new file mode 100644 index 000000000..9ca930207 --- /dev/null +++ b/.github/workflows/notify_release.yml @@ -0,0 +1,40 @@ +name: Notify release workflow finished + +on: + workflow_call: + inputs: + success: + type: string + secrets: + MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK: + required: true + +permissions: + actions: read + contents: read + +jobs: + notify_mattermost: + name: Notify mattermost + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + - name: Generate message text + shell: bash + id: message + env: + SUCCESS: "${{ inputs.success }}" + run: |- + AUTHOR_NAME="$(git log -1 --pretty=format:'%an')" + MESSAGE=$(tools/release_message.py "${AUTHOR_NAME}" "${SUCCESS}") + echo "result<> $GITHUB_OUTPUT + echo "$MESSAGE" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Send to mattermost + uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK }} + TEXT: "${{ steps.message.outputs.result }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..a6659e0df --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,133 @@ +# Release pipeline to run our full test suite, build artifacts, and deploy them +# for GitHub release tags. +name: Release +run-name: Release Certbot ${{ github.ref_name }} +on: + push: + tags: + - v* + +permissions: + contents: read + +jobs: + # While many of these jobs could be grouped in a separate workflow, the github actions UI + # is much nicer if they are instead listed explicitly here. + ########################### + #### testing jobs ### + ########################### + standard_tests_jobs: + name: Standard tests + uses: "./.github/workflows/standard_tests_jobs.yml" + extended_tests_jobs: + name: Extended tests + uses: "./.github/workflows/extended_tests_jobs.yml" + secrets: + AWS_TEST_FARM_PEM: "${{ secrets.AWS_TEST_FARM_PEM }}" + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + ########################### + #### packaging jobs ### + ########################### + docker_packaging_jobs: + name: Docker packaging + uses: "./.github/workflows/docker_packaging_jobs.yml" + with: + dockerTag: "${{ github.ref_name }}" + snap_packaging_jobs: + name: Snap packaging + uses: "./.github/workflows/snap_packaging_jobs.yml" + with: + snapBuildTimeout: 19800 + secrets: + LAUNCHPAD_CREDENTIALS: "${{ secrets.LAUNCHPAD_CREDENTIALS }}" + create_changelog: + name: Create changelog + uses: "./.github/workflows/create_changelog.yml" + ############################ + #### deploy jobs ### + ############################ + docker_deploy_jobs: + name: Deploy docker images + needs: + - standard_tests_jobs + - extended_tests_jobs + - docker_packaging_jobs + uses: "./.github/workflows/deploy_docker_images.yml" + secrets: + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}" + with: + dockerTag: "${{ github.ref_name }}" + snap_deploy_jobs: + name: Deploy snaps + needs: + - standard_tests_jobs + - extended_tests_jobs + - snap_packaging_jobs + uses: "./.github/workflows/deploy_snaps.yml" + secrets: + SNAPCRAFTCFG: "${{ secrets.SNAPCRAFTCFG }}" + with: + snapReleaseChannel: beta + create_github_release: + name: Create GitHub release + needs: + - standard_tests_jobs + - extended_tests_jobs + - docker_packaging_jobs + - snap_packaging_jobs + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + - name: Download changelog + uses: actions/download-artifact@v8.0.1 + with: + name: changelog + path: "${{ github.workspace }}" + - name: GitHub release + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + run: |- + gh release create "$TAG" "${GITHUB_WORKSPACE}/packages/"{*.tar.gz,SHA256SUMS*} \ + --title "Certbot ${TAG#v}" \ + --notes-file "$GITHUB_WORKSPACE/release_notes.md" + + ########################### + #### notify ### + ########################### + notify_success: + name: Notify success + needs: + - docker_deploy_jobs + - snap_deploy_jobs + - create_github_release + uses: "./.github/workflows/notify_release.yml" + permissions: + actions: read + contents: read + with: + success: 'True' + secrets: + MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK: "${{ secrets.MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK }}" + notify_failure: + name: Notify failure + needs: + - docker_deploy_jobs + - snap_deploy_jobs + - create_github_release + if: ${{ failure() }} + uses: "./.github/workflows/notify_release.yml" + permissions: + actions: read + contents: read + with: + success: 'False' + secrets: + MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK: "${{ secrets.MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK }}" diff --git a/tools/notify_mattermost.py b/tools/notify_mattermost.py deleted file mode 100755 index 320f90c64..000000000 --- a/tools/notify_mattermost.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -""" -Script to notify the person doing the release that the Azure run was successful. - -Run: - -python tools/notify_mattermost.py GITHUB_AUTHOR_NAME MATTERMOST_WEBHOOK_URL -""" -import os -import random -import requests -import sys - -repo_name = os.environ['BUILD_REPOSITORY_ID'] -build_id = os.environ['BUILD_BUILDID'] - -def get_greeting(): - fun_greetings = [ - 'Hey', - 'Paging', - 'Hi', - 'Pinging', - ] - return random.choice(fun_greetings) - -def get_message(): - fun_success_messages = [ - 'the certbot release is ready to come out of the oven!', - "it's release-finishing go time!", - 'all certbot release systems are set for launch!', - ] - - # https://learn.microsoft.com/en-us/rest/api/azure/devops/build/timeline/get?view=azure-devops-rest-7.1 - timeline_url = f'https://dev.azure.com/{repo_name}/_apis/build/builds/{build_id}/timeline/?api-version=7.1' - response = requests.get(timeline_url) - response.raise_for_status() - data = response.json() - deploy_record = next((rec for rec in data['records'] if rec['name'] == 'Deploy'), None) - if deploy_record is None: - raise RuntimeError('Unable to find the record for the Deploy stage') - deploy_result = deploy_record['result'] - if deploy_result in ['succeeded', 'succeededWithIssues']: - message = random.choice(fun_success_messages) - elif deploy_result in ['skipped', 'failed', 'abandoned']: - message = "the release pipeline has failed." - else: - raise RuntimeError('Unexpected stage result {0}'.format(deploy_result)) - return message - -def get_mattermost_url(): - # This should be a mattermost webhook url that posts to a specific channel, - # created by certbotbot, with a file containing the url saved in azure pipelines secret - # files, under pipelines > library. The secret file will need to be given permission to - # be used by the specific pipeline, in this case 'release.' - url_path = sys.argv[2] - with open(url_path, 'r') as file: - url = file.read().rstrip() - return url - -def get_headers(): - headers = { - 'Content-Type': 'application/json', - } - return headers - -def get_content(): - build_url = f'https://dev.azure.com/{repo_name}/_build/results?buildId={build_id}&view=results' - - # We use github author here because it's what we have access to. If the name sometimes - # changes, add any name it might be. Check the git log. - requested_for = sys.argv[1].rstrip() - # This is a map of team member github author names to opensource mattermost username - usernames_map = { - 'Will Greenberg': 'willg', - 'Erica Portnoy': 'erica', - 'Brad Warren': 'brad', - 'ohemorange': 'erica', - } - - if requested_for in usernames_map: - text_body = f'{get_greeting()} @{usernames_map[requested_for]}, {get_message()}\n{build_url}' - else: - text_body = (f"{get_greeting()} {requested_for}, {get_message()}\nIf you'd like to get @ mentioned for " - "releases you do in the future, please modify tools/notify_mattermost.py with your " - f"git author name.\n{build_url}") - - content = { - 'text': text_body, - } - return content - -response = requests.request( - method='POST', - url=get_mattermost_url(), - headers=get_headers(), - json=get_content(), -) -response.raise_for_status() diff --git a/tools/release_message.py b/tools/release_message.py new file mode 100755 index 000000000..5eced3e5f --- /dev/null +++ b/tools/release_message.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +""" +Script to generate a message notifying the person doing the release +of the result of the workflow run. + +Run: + +python tools/notify_mattermost.py GITHUB_AUTHOR_NAME SUCCESS + +where SUCCESS is True or False +""" +import os +import random +import sys + +server_url = os.environ['GITHUB_SERVER_URL'] +repo_name = os.environ['GITHUB_REPOSITORY'] +run_id = os.environ['GITHUB_RUN_ID'] + +def get_greeting(): + fun_greetings = [ + 'Hey', + 'Paging', + 'Hi', + 'Pinging', + ] + return random.choice(fun_greetings) + +def get_message(success: bool): + fun_success_messages = [ + 'the certbot release is ready to come out of the oven!', + "it's release-finishing go time!", + 'all certbot release systems are set for launch!', + ] + + if success: + message = random.choice(fun_success_messages) + else: + message = "the release pipeline has failed." + return message + +def get_content(requested_for: str, success: bool): + build_url = f'{server_url}/{repo_name}/actions/runs/{run_id}' + + # We use github author here because it's what we have access to. If the name sometimes + # changes, add any name it might be. Check the git log. + # This is a map of team member github author names to opensource mattermost username + usernames_map = { + 'Will Greenberg': 'willg', + 'Erica Portnoy': 'erica', + 'Brad Warren': 'brad', + 'ohemorange': 'erica', + } + + if requested_for in usernames_map: + text_body = f'{get_greeting()} @{usernames_map[requested_for]}, {get_message(success)}\n{build_url}' + else: + text_body = (f"{get_greeting()} {requested_for}, {get_message(success)}\nIf you'd like to get @ mentioned for " + "releases you do in the future, please modify tools/notify_mattermost.py with your " + f"git author name.\n{build_url}") + return text_body + +random.seed() +requested_for: str = sys.argv[1].rstrip() +success: bool = eval(sys.argv[2].rstrip()) +print(get_content(requested_for, success)) From 10aa343d3ab018db31a219207f1dd6e790af3eca Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 18 May 2026 11:11:01 -0700 Subject: [PATCH 2/4] remove .azure-pipelines --- .azure-pipelines/INSTALL.md | 119 ------------- .azure-pipelines/release.yml | 17 -- .../templates/jobs/common-deploy-jobs.yml | 125 -------------- .../templates/jobs/extended-tests-jobs.yml | 57 ------- .../templates/jobs/packaging-jobs.yml | 159 ------------------ .../templates/jobs/standard-tests-jobs.yml | 55 ------ .../templates/stages/changelog-stage.yml | 19 --- .../templates/stages/notify-stage.yml | 20 --- .../templates/stages/release-deploy-stage.yml | 29 ---- .../stages/test-and-package-stage.yml | 6 - .../templates/steps/sphinx-steps.yml | 24 --- .../templates/steps/tox-steps.yml | 77 --------- 12 files changed, 707 deletions(-) delete mode 100644 .azure-pipelines/INSTALL.md delete mode 100644 .azure-pipelines/release.yml delete mode 100644 .azure-pipelines/templates/jobs/common-deploy-jobs.yml delete mode 100644 .azure-pipelines/templates/jobs/extended-tests-jobs.yml delete mode 100644 .azure-pipelines/templates/jobs/packaging-jobs.yml delete mode 100644 .azure-pipelines/templates/jobs/standard-tests-jobs.yml delete mode 100644 .azure-pipelines/templates/stages/changelog-stage.yml delete mode 100644 .azure-pipelines/templates/stages/notify-stage.yml delete mode 100644 .azure-pipelines/templates/stages/release-deploy-stage.yml delete mode 100644 .azure-pipelines/templates/stages/test-and-package-stage.yml delete mode 100644 .azure-pipelines/templates/steps/sphinx-steps.yml delete mode 100644 .azure-pipelines/templates/steps/tox-steps.yml diff --git a/.azure-pipelines/INSTALL.md b/.azure-pipelines/INSTALL.md deleted file mode 100644 index 2901fb4e3..000000000 --- a/.azure-pipelines/INSTALL.md +++ /dev/null @@ -1,119 +0,0 @@ -# Configuring Azure Pipelines with Certbot - -Let's begin. All pipelines are defined in `.azure-pipelines`. Currently there are two: -* `.azure-pipelines/main.yml` is the main one, executed on PRs for main, and pushes to main, -* `.azure-pipelines/advanced.yml` add installer testing on top of the main pipeline, and is executed for `test-*` branches, release branches, and nightly run for main. - -Several templates are defined in `.azure-pipelines/templates`. These YAML files aggregate common jobs configuration that can be reused in several pipelines. - -Unlike Travis, where CodeCov is working without any action required, CodeCov supports Azure Pipelines -using the coverage-bash utility (not python-coverage for now) only if you provide the Codecov repo token -using the `CODECOV_TOKEN` environment variable. So `CODECOV_TOKEN` needs to be set as a secured -environment variable to allow the main pipeline to publish coverage reports to CodeCov. - -This INSTALL.md file explains how to configure Azure Pipelines with Certbot in order to execute the CI/CD logic defined in `.azure-pipelines` folder with it. -During this installation step, warnings describing user access and legal comitments will be displayed like this: -``` -!!! ACCESS REQUIRED !!! -``` - -This document suppose that the Azure DevOps organization is named _certbot_, and the Azure DevOps project is also _certbot_. - -## Useful links - -* https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema -* https://www.azuredevopslabs.com/labs/azuredevops/github-integration/ -* https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/python?view=azure-devops - -## Prerequisites - -### Having a GitHub account - -Use your GitHub user for a normal GitHub account, or a user that has administrative rights to the GitHub organization if relevant. - -### Having an Azure DevOps account -- Go to https://dev.azure.com/, click "Start free with GitHub" -- Login to GitHub - -``` -!!! ACCESS REQUIRED !!! -Personal user data (email + profile info, in read-only) -``` - -- Microsoft will create a Live account using the email referenced for the GitHub account. This account is also linked to GitHub account (meaning you can log it using GitHub authentication) -- Proceed with account registration (birth date, country), add details about name and email contact - -``` -!!! ACCESS REQUIRED !!! -Microsoft proposes to send commercial links to this mail -Azure DevOps terms of service need to be accepted -``` - -_Logged to Azure DevOps, account is ready._ - -### Installing Azure Pipelines to GitHub - -- On GitHub, go to Marketplace -- Select Azure Pipeline, and "Set up a plan" -- Select Free, then "Install it for free" -- Click "Complete order and begin installation" - -``` -!!! ACCESS !!! -Azure Pipeline needs RW on code, RO on metadata, RW on checks, commit statuses, deployments, issues, pull requests. -RW access here is required to allow update of the pipelines YAML files from Azure DevOps interface, and to -update the status of builds and PRs on GitHub side when Azure Pipelines are triggered. -Note however that no admin access is defined here: this means that Azure Pipelines cannot do anything with -protected branches, like main, and cannot modify the security context around this on GitHub. -Access can be defined for all or only selected repositories, which is nice. -``` - -- Redirected to Azure DevOps, select the account created in _Having an Azure DevOps account_ section. -- Select the organization, and click "Create a new project" (let's name it the same than the targeted github repo) -- The Visibility is public, to profit from 10 parallel jobs - -``` -!!! ACCESS !!! -Azure Pipelines needs access to the GitHub account (in term of being able to check it is valid), and the Resources shared between the GitHub account and Azure Pipelines. -``` - -_Done. We can move to pipelines configuration._ - -## Import an existing pipelines from `.azure-pipelines` folder - -- On Azure DevOps, go to your organization (eg. _certbot_) then your project (eg. _certbot_) -- Click "Pipelines" tab -- Click "New pipeline" -- Where is your code?: select "__Use the classic editor__" - -__Warning: Do not choose the GitHub option in Where is your code? section. Indeed, this option will trigger an OAuth -grant permissions from Azure Pipelines to GitHub in order to setup a GitHub OAuth Application. The permissions asked -then are way too large (admin level on almost everything), while the classic approach does not add any more -permissions, and works perfectly well.__ - -- Select GitHub in "Select your repository section", choose certbot/certbot in Repository, main in default branch. -- Click on YAML option for "Select a template" -- Choose a name for the pipeline (eg. test-pipeline), and browse to the actual pipeline YAML definition in the - "YAML file path" input (eg. `.azure-pipelines/test-pipeline.yml`) -- Click "Save & queue", choose the main branch to build the first pipeline, and click "Save and run" button. - -_Done. Pipeline is operational. Repeat to add more pipelines from existing YAML files in `.azure-pipelines`._ - -## Add a secret variable to a pipeline (like `CODECOV_TOKEN`) - -__NB: Following steps suppose that you already setup the YAML pipeline file to -consume the secret variable that these steps will create as an environment variable. -For a variable named `CODECOV_TOKEN` consuming the variable `codecov_token`, -in the YAML file this setup would take the form of the following: -``` -steps: - - script: ./do_something_that_consumes_CODECOV_TOKEN # Eg. `codecov -F windows` - env: - CODECOV_TOKEN: $(codecov_token) -``` - -To set up a variable that is shared between pipelines, follow the instructions -at -https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups. -When adding variables to a group, don't forget to tick "Keep this value secret" -if it shouldn't be shared publcily. diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml deleted file mode 100644 index 9e28382d9..000000000 --- a/.azure-pipelines/release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Release pipeline to run our full test suite, build artifacts, and deploy them -# for GitHub release tags. -trigger: - tags: - include: - - v* -pr: none - -variables: - dockerTag: ${{variables['Build.SourceBranchName']}} - snapBuildTimeout: 19800 - -stages: - - template: templates/stages/test-and-package-stage.yml - - template: templates/stages/changelog-stage.yml - - template: templates/stages/release-deploy-stage.yml - - template: templates/stages/notify-stage.yml diff --git a/.azure-pipelines/templates/jobs/common-deploy-jobs.yml b/.azure-pipelines/templates/jobs/common-deploy-jobs.yml deleted file mode 100644 index 3f5d15dbf..000000000 --- a/.azure-pipelines/templates/jobs/common-deploy-jobs.yml +++ /dev/null @@ -1,125 +0,0 @@ -# As (somewhat) described at -# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates?view=azure-devops#context, -# each template only has access to the parameters passed into it. To help make -# use of this design, we define snapReleaseChannel without a default value -# which requires the user of this template to define it as described at -# https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/parameters-name?view=azure-pipelines#remarks. -# This makes the user of this template be explicit while allowing them to -# define their own parameters with defaults that make sense for that context. -parameters: -- name: snapReleaseChannel - type: string - values: - - edge - - beta - -jobs: - # This job relies on credentials used to publish the Certbot snaps. This - # credential file was created by running: - # - # snapcraft logout - # snapcraft export-login --channels=beta,edge snapcraft.cfg - # (provide the shared snapcraft credentials when prompted) - # - # Then the file was added as a secure file in Azure pipelines - # with the name snapcraft.cfg by following the instructions at - # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops - # including authorizing the file for use in the "nightly" and "release" - # pipelines as described at - # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops#q-how-do-i-authorize-a-secure-file-for-use-in-a-specific-pipeline. - # - # This file has a maximum lifetime of one year. Revoking these credentials - # can be done by changing the password of the account used to generate the - # credentials. See - # https://forum.snapcraft.io/t/revoking-exported-credentials/19031 for more - # info. - - job: publish_snap - pool: - vmImage: ubuntu-22.04 - variables: - - group: certbot-common - strategy: - matrix: - amd64: - SNAP_ARCH: amd64 - arm32v6: - SNAP_ARCH: armhf - arm64v8: - SNAP_ARCH: arm64 - steps: - - bash: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends snapd - sudo snap install --classic snapcraft - displayName: Install dependencies - - task: DownloadPipelineArtifact@2 - inputs: - artifact: snaps_$(SNAP_ARCH) - path: $(Build.SourcesDirectory)/snap - displayName: Retrieve Certbot snaps - - task: DownloadSecureFile@1 - name: snapcraftCfg - inputs: - secureFile: snapcraft.cfg - - bash: | - set -e - export SNAPCRAFT_STORE_CREDENTIALS=$(cat "$(snapcraftCfg.secureFilePath)") - for SNAP_FILE in snap/*.snap; do - tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}" - done - displayName: Publish to Snap store - # The credentials used in the following jobs are for the shared - # certbotbot account on Docker Hub. The credentials are stored - # in a service account which was created by following the - # instructions at - # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-docreg. - # The name given to this service account must match the value - # given to containerRegistry below. The authentication used when - # creating this service account was a personal access token - # rather than a password to bypass 2FA. When Brad set this up, - # Azure Pipelines failed to verify the credentials with an error - # like "access is forbidden with a JWT issued from a personal - # access token", but after saving them without verification, the - # access token worked when the pipeline actually ran. "Grant - # access to all pipelines" should also be checked on the service - # account. The access token can be deleted on Docker Hub if - # these credentials need to be revoked. - - job: publish_docker_by_arch - pool: - vmImage: ubuntu-22.04 - strategy: - matrix: - arm32v6: - DOCKER_ARCH: arm32v6 - arm64v8: - DOCKER_ARCH: arm64v8 - amd64: - DOCKER_ARCH: amd64 - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: docker_$(DOCKER_ARCH) - path: $(Build.SourcesDirectory) - displayName: Retrieve Docker images - - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar - displayName: Load Docker images - - task: Docker@2 - inputs: - command: login - containerRegistry: docker-hub - displayName: Login to Docker Hub - - bash: set -e && tools/docker/deploy_images.sh $(dockerTag) $DOCKER_ARCH - displayName: Deploy the Docker images by architecture - - job: publish_docker_multiarch - dependsOn: publish_docker_by_arch - pool: - vmImage: ubuntu-22.04 - steps: - - task: Docker@2 - inputs: - command: login - containerRegistry: docker-hub - displayName: Login to Docker Hub - - bash: set -e && tools/docker/deploy_manifests.sh $(dockerTag) all - displayName: Deploy the Docker multiarch manifests diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml deleted file mode 100644 index 10d0de934..000000000 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ /dev/null @@ -1,57 +0,0 @@ -jobs: - - job: extended_test - variables: - - name: IMAGE_NAME - value: ubuntu-22.04 - - name: PYTHON_VERSION - value: 3.14 - - group: certbot-common - strategy: - matrix: - linux-py311: - PYTHON_VERSION: 3.11 - TOXENV: py311 - linux-py312: - PYTHON_VERSION: 3.12 - TOXENV: py312 - linux-py313: - PYTHON_VERSION: 3.13 - TOXENV: py313 - linux-isolated: - TOXENV: 'isolated-acme,isolated-certbot,isolated-apache,isolated-cloudflare,isolated-digitalocean,isolated-dnsimple,isolated-dnsmadeeasy,isolated-gehirn,isolated-google,isolated-linode,isolated-luadns,isolated-nsone,isolated-ovh,isolated-rfc2136,isolated-route53,isolated-sakuracloud,isolated-nginx' - linux-integration-certbot-oldest: - PYTHON_VERSION: 3.10 - TOXENV: integration-certbot-oldest - linux-integration-nginx-oldest: - PYTHON_VERSION: 3.10 - TOXENV: integration-nginx-oldest - linux-py310-integration: - PYTHON_VERSION: 3.10 - TOXENV: integration - linux-py311-integration: - PYTHON_VERSION: 3.11 - TOXENV: integration - linux-py312-integration: - PYTHON_VERSION: 3.12 - TOXENV: integration - linux-py313-integration: - PYTHON_VERSION: 3.13 - TOXENV: integration - # python 3.14 integration tests are not run here because they're run as - # part of the standard test suite - nginx-compat: - TOXENV: nginx_compat - linux-integration-rfc2136: - IMAGE_NAME: ubuntu-22.04 - PYTHON_VERSION: 3.12 - TOXENV: integration-dns-rfc2136 - le-modification: - IMAGE_NAME: ubuntu-22.04 - TOXENV: modification - farmtest-apache2: - PYTHON_VERSION: 3.12 - TOXENV: test-farm-apache2 - pool: - vmImage: $(IMAGE_NAME) - steps: - - template: ../steps/tox-steps.yml diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml deleted file mode 100644 index be2979f57..000000000 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ /dev/null @@ -1,159 +0,0 @@ -jobs: - - job: docker_build - pool: - vmImage: ubuntu-24.04 - strategy: - matrix: - arm32v6: - DOCKER_ARCH: arm32v6 - arm64v8: - DOCKER_ARCH: arm64v8 - amd64: - DOCKER_ARCH: amd64 - # The default timeout of 60 minutes is a little low for compiling - # cryptography on ARM architectures. - timeoutInMinutes: 180 - steps: - - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH - displayName: Build the Docker images - # We don't filter for the Docker Hub organization to continue to allow - # easy testing of these scripts on forks. - - bash: | - set -e - DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}') - docker save --output images.tar $DOCKER_IMAGES - displayName: Save the Docker images - # If the name of the tar file or artifact changes, the deploy stage will - # also need to be updated. - - bash: set -e && mv images.tar $(Build.ArtifactStagingDirectory) - displayName: Prepare Docker artifact - - task: PublishPipelineArtifact@1 - inputs: - path: $(Build.ArtifactStagingDirectory) - artifact: docker_$(DOCKER_ARCH) - displayName: Store Docker artifact - - job: docker_test - dependsOn: docker_build - pool: - vmImage: ubuntu-22.04 - strategy: - matrix: - arm32v6: - DOCKER_ARCH: arm32v6 - arm64v8: - DOCKER_ARCH: arm64v8 - amd64: - DOCKER_ARCH: amd64 - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: docker_$(DOCKER_ARCH) - path: $(Build.SourcesDirectory) - displayName: Retrieve Docker images - - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar - displayName: Load Docker images - - bash: | - set -e && tools/docker/test.sh $(dockerTag) $DOCKER_ARCH - displayName: Run integration tests for Docker images - - job: snaps_build - pool: - vmImage: ubuntu-22.04 - strategy: - matrix: - amd64: - SNAP_ARCH: amd64 - armhf: - SNAP_ARCH: armhf - arm64: - SNAP_ARCH: arm64 - timeoutInMinutes: 0 - steps: - - script: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends snapd - sudo snap install --classic snapcraft - displayName: Install dependencies - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.12 - addToPath: true - - task: DownloadSecureFile@1 - name: credentials - inputs: - secureFile: launchpad-credentials - - script: | - set -e - git config --global user.email "$(Build.RequestedForEmail)" - git config --global user.name "$(Build.RequestedFor)" - mkdir -p ~/.local/share/snapcraft/provider/launchpad - cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout $(snapBuildTimeout) - displayName: Build snaps - - script: | - set -e - mv *.snap $(Build.ArtifactStagingDirectory) - mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory) - displayName: Prepare artifacts - - task: PublishPipelineArtifact@1 - inputs: - path: $(Build.ArtifactStagingDirectory) - artifact: snaps_$(SNAP_ARCH) - displayName: Store snaps artifacts - - job: snap_run - dependsOn: snaps_build - pool: - vmImage: ubuntu-22.04 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.12 - addToPath: true - - script: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends nginx-light snapd - python3 -m venv venv - venv/bin/python tools/pip_install.py -U tox - displayName: Install dependencies - - task: DownloadPipelineArtifact@2 - inputs: - artifact: snaps_amd64 - path: $(Build.SourcesDirectory)/snap - displayName: Retrieve Certbot snaps - - script: | - set -e - sudo snap install --dangerous --classic snap/certbot_*.snap - displayName: Install Certbot snap - - script: | - set -e - venv/bin/python -m tox run -e integration-external,apacheconftest-external-with-pebble - displayName: Run tox - - job: snap_dns_run - dependsOn: snaps_build - pool: - vmImage: ubuntu-22.04 - steps: - - script: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends snapd - displayName: Install dependencies - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.12 - addToPath: true - - task: DownloadPipelineArtifact@2 - inputs: - artifact: snaps_amd64 - path: $(Build.SourcesDirectory)/snap - displayName: Retrieve Certbot snaps - - script: | - set -e - python3 -m venv venv - venv/bin/python tools/pip_install.py -e certbot-ci - displayName: Prepare Certbot-CI - - script: | - set -e - sudo -E venv/bin/pytest certbot-ci/src/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64 - displayName: Test DNS plugins snaps diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml deleted file mode 100644 index 0902f04cd..000000000 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ /dev/null @@ -1,55 +0,0 @@ -jobs: - - job: test - variables: - PYTHON_VERSION: 3.14 - strategy: - matrix: - macos-cover: - IMAGE_NAME: macOS-15 - TOXENV: cover - # As of pip 23.1.0, builds started failing on macOS unless this flag was set. - # See https://github.com/certbot/certbot/pull/9717#issuecomment-1610861794. - PIP_USE_PEP517: "true" - linux-oldest: - IMAGE_NAME: ubuntu-22.04 - PYTHON_VERSION: 3.10 - TOXENV: oldest - linux-py310: - # linux unit tests with the oldest python we support - IMAGE_NAME: ubuntu-22.04 - PYTHON_VERSION: 3.10 - TOXENV: py310 - linux-cover: - # linux unit+cover tests with the newest python we support - IMAGE_NAME: ubuntu-22.04 - TOXENV: cover - linux-lint: - IMAGE_NAME: ubuntu-22.04 - TOXENV: lint-posix - linux-mypy: - IMAGE_NAME: ubuntu-22.04 - TOXENV: mypy - linux-integration: - IMAGE_NAME: ubuntu-22.04 - TOXENV: integration - apache-compat: - IMAGE_NAME: ubuntu-22.04 - TOXENV: apache_compat - apacheconftest: - IMAGE_NAME: ubuntu-22.04 - TOXENV: apacheconftest-with-pebble - nginxroundtrip: - IMAGE_NAME: ubuntu-22.04 - TOXENV: nginxroundtrip - validate-changelog: - IMAGE_NAME: ubuntu-22.04 - TOXENV: validate-changelog - pool: - vmImage: $(IMAGE_NAME) - steps: - - template: ../steps/tox-steps.yml - - job: test_sphinx_builds - pool: - vmImage: ubuntu-22.04 - steps: - - template: ../steps/sphinx-steps.yml diff --git a/.azure-pipelines/templates/stages/changelog-stage.yml b/.azure-pipelines/templates/stages/changelog-stage.yml deleted file mode 100644 index a4d7e569c..000000000 --- a/.azure-pipelines/templates/stages/changelog-stage.yml +++ /dev/null @@ -1,19 +0,0 @@ -stages: - - stage: Changelog - jobs: - - job: prepare - pool: - vmImage: ubuntu-latest - steps: - # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py - - bash: | - set -e - CERTBOT_VERSION="$(cd certbot/src && python -c "import certbot; print(certbot.__version__)" && cd ~-)" - "${BUILD_REPOSITORY_LOCALPATH}/tools/extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md" - displayName: Prepare changelog - - task: PublishPipelineArtifact@1 - inputs: - path: $(Build.ArtifactStagingDirectory) - # If we change the artifact's name, it should also be changed in tools/create_github_release.py - artifact: changelog - displayName: Publish changelog diff --git a/.azure-pipelines/templates/stages/notify-stage.yml b/.azure-pipelines/templates/stages/notify-stage.yml deleted file mode 100644 index e1ea4871f..000000000 --- a/.azure-pipelines/templates/stages/notify-stage.yml +++ /dev/null @@ -1,20 +0,0 @@ -stages: - - stage: Notify - condition: succeededOrFailed() - jobs: - - job: notify_release_finished - pool: - vmImage: ubuntu-latest - steps: - - task: DownloadSecureFile@1 - name: webhook_url - inputs: - secureFile: mattermost_webhook_url_release_notify - - bash: | - set -e - python -m venv venv - source venv/bin/activate - tools/pip_install.py requests - AUTHOR_NAME="$(git log -1 --pretty=format:'%an')" - tools/notify_mattermost.py "${AUTHOR_NAME}" $(webhook_url.secureFilePath) - displayName: Send mattermost message diff --git a/.azure-pipelines/templates/stages/release-deploy-stage.yml b/.azure-pipelines/templates/stages/release-deploy-stage.yml deleted file mode 100644 index 866d6994d..000000000 --- a/.azure-pipelines/templates/stages/release-deploy-stage.yml +++ /dev/null @@ -1,29 +0,0 @@ -stages: - # tools/notify_mattermost.py depends on this stage being named 'Deploy' - # If you change the name here, please remember to update that script as well - - stage: Deploy - jobs: - - template: ../jobs/common-deploy-jobs.yml - parameters: - snapReleaseChannel: beta - - job: create_github_release - pool: - vmImage: ubuntu-22.04 - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: changelog - path: '$(Pipeline.Workspace)' - - task: GitHubRelease@1 - inputs: - # this "github-releases" credential is what azure pipelines calls a - # "service connection". it needs to be recreated annually. instructions - # to do so and further information about the token are available at - # https://github.com/EFForg/certbot-misc/wiki/Azure-Pipelines-setup#regenerating-github-release-credentials-for-use-on-azure - # - # as of writing this, the current token will expire on Wed, Feb 25 2026. - gitHubConnection: github-releases - title: ${{ format('Certbot {0}', replace(variables['Build.SourceBranchName'], 'v', '')) }} - releaseNotesFilePath: '$(Pipeline.Workspace)/release_notes.md' - assets: '$(Build.SourcesDirectory)/packages/{*.tar.gz,SHA256SUMS*}' - addChangeLog: false diff --git a/.azure-pipelines/templates/stages/test-and-package-stage.yml b/.azure-pipelines/templates/stages/test-and-package-stage.yml deleted file mode 100644 index 26010644d..000000000 --- a/.azure-pipelines/templates/stages/test-and-package-stage.yml +++ /dev/null @@ -1,6 +0,0 @@ -stages: - - stage: TestAndPackage - jobs: - - template: ../jobs/standard-tests-jobs.yml - - template: ../jobs/extended-tests-jobs.yml - - template: ../jobs/packaging-jobs.yml diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml deleted file mode 100644 index c34debe4c..000000000 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ /dev/null @@ -1,24 +0,0 @@ -steps: - - bash: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends libaugeas-dev - FINAL_STATUS=0 - declare -a FAILED_BUILDS - tools/venv.py - source venv/bin/activate - for doc_path in */docs - do - echo "" - echo "##[group]Building $doc_path" - if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then - FINAL_STATUS=1 - FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" - fi - echo "##[endgroup]" - done - if [[ $FINAL_STATUS -ne 0 ]]; then - echo "##[error]The following builds failed: ${FAILED_BUILDS[*]}" - exit 1 - fi - displayName: Build Sphinx Documentation diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml deleted file mode 100644 index 2466cce58..000000000 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ /dev/null @@ -1,77 +0,0 @@ -# This does not include the dependencies needed to build cryptography. See -# https://cryptography.io/en/latest/installation/ -steps: - # We run brew update because we've seen attempts to install an older version - # of a package fail. See - # https://github.com/actions/virtual-environments/issues/3165. - # - # We untap homebrew/core and homebrew/cask and unset HOMEBREW_NO_INSTALL_FROM_API (which - # is set by the CI macOS env) because GitHub has been having issues, making these jobs - # fail on git clones: https://github.com/orgs/Homebrew/discussions/4612. - - bash: | - set -e - unset HOMEBREW_NO_INSTALL_FROM_API - brew untap homebrew/core homebrew/cask - brew update - brew install augeas - condition: startswith(variables['IMAGE_NAME'], 'macOS') - displayName: Install MacOS dependencies - - bash: | - set -e - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - libaugeas-dev \ - nginx-light - sudo systemctl stop nginx - sudo sysctl net.ipv4.ip_unprivileged_port_start=0 - condition: startswith(variables['IMAGE_NAME'], 'ubuntu') - displayName: Install Linux dependencies - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - addToPath: true - - bash: | - set -e - python3 tools/pip_install.py tox - displayName: Install runtime dependencies - - task: DownloadSecureFile@1 - name: testFarmPem - inputs: - secureFile: azure-test-farm.pem - condition: contains(variables['TOXENV'], 'test-farm') - - bash: | - set -e - export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`" - [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}" - env - python3 -m tox run - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_EC2_PEM_FILE: $(testFarmPem.secureFilePath) - displayName: Run tox - # For now, let's omit `set -e` and avoid the script exiting with a nonzero - # status code to prevent problems here from causing build failures. If - # this turns out to work well, we can change this. - - bash: | - python3 tools/pip_install.py -I coverage - case "$AGENT_OS" in - Darwin) - CODECOV_URL="https://uploader.codecov.io/latest/macos/codecov" - ;; - Linux) - CODECOV_URL="https://uploader.codecov.io/latest/linux/codecov" - ;; - Windows_NT) - CODECOV_URL="https://uploader.codecov.io/latest/windows/codecov.exe" - ;; - *) - echo "Unexpected OS" - exit 0 - esac - curl --retry 3 -o codecov "$CODECOV_URL" - chmod +x codecov - coverage xml - ./codecov || echo "Uploading coverage data failed" - condition: and(eq(variables['uploadCoverage'], true), or(startsWith(variables['TOXENV'], 'cover'), startsWith(variables['TOXENV'], 'integration'))) - displayName: Upload coverage data From 2ea33021aa2599390f1eb622e1c86d9e14bdc95e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 18 May 2026 11:16:04 -0700 Subject: [PATCH 3/4] add comments explaining about mirroring changes in nightly and release --- .github/workflows/nightly.yml | 3 ++- .github/workflows/release.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f3873580b..a47d3b499 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -7,7 +7,8 @@ on: jobs: # While many of these jobs could be grouped in a separate workflow, the github actions UI - # is much nicer if they are instead listed explicitly here. + # is much nicer if they are instead listed explicitly here. As a result, changes made here + # may need to be mirrored in .github/workflows/release.yml. ########################### #### testing jobs ### ########################### diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6659e0df..6f2f4c332 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,8 @@ permissions: jobs: # While many of these jobs could be grouped in a separate workflow, the github actions UI - # is much nicer if they are instead listed explicitly here. + # is much nicer if they are instead listed explicitly here. As a result, changes made here + # may need to be mirrored in .github/workflows/nightly.yml. ########################### #### testing jobs ### ########################### From 85fdacd49ec6ad8b1edb69e77bf7d84fdf07890e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 19 May 2026 12:30:53 -0700 Subject: [PATCH 4/4] Upgrade action-mattermost-notify action to use node 24 --- .github/workflows/notify_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notify_release.yml b/.github/workflows/notify_release.yml index 9ca930207..95b83d480 100644 --- a/.github/workflows/notify_release.yml +++ b/.github/workflows/notify_release.yml @@ -34,7 +34,7 @@ jobs: echo "$MESSAGE" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - name: Send to mattermost - uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a + uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e with: MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_PUBLIC_CERTBOT_CHANNEL_WEBHOOK }} TEXT: "${{ steps.message.outputs.result }}"