From da3ffa20410fe175c0940cc84ca29d8a7b43829d Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Mon, 23 Mar 2026 14:29:48 -0700 Subject: [PATCH] migrate advanced tests to github actions credentials for test farms are not currently working. credentials for launchpad may or may not be working. pass inputs to action filename must match key name move secret to one action move env to top use env add quotes set AWS_EC2_PEM_FILE in caller env mkdir dirname use workspace remove quotes use different subs syntax --- .azure-pipelines/advanced-test.yml | 15 -- .github/actions/run_tox/action.yml | 30 ++- .github/actions/setup_tox/action.yml | 17 +- .github/workflows/extended_tests_jobs.yml | 64 +++++++ .github/workflows/full-test-suite.yml | 19 ++ .github/workflows/packaging_jobs.yml | 188 +++++++++++++++++++ .github/workflows/test_and_package_stage.yml | 33 ++++ 7 files changed, 338 insertions(+), 28 deletions(-) delete mode 100644 .azure-pipelines/advanced-test.yml create mode 100644 .github/workflows/extended_tests_jobs.yml create mode 100644 .github/workflows/full-test-suite.yml create mode 100644 .github/workflows/packaging_jobs.yml create mode 100644 .github/workflows/test_and_package_stage.yml diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml deleted file mode 100644 index 9915881ce..000000000 --- a/.azure-pipelines/advanced-test.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Advanced pipeline for running our full test suite on demand. -trigger: - # When changing these triggers, please ensure the documentation under - # "Running tests in CI" is still correct. - - test-* -pr: none - -variables: - # We don't publish our Docker images in this pipeline, but when building them - # for testing, let's use the nightly tag. - dockerTag: nightly - snapBuildTimeout: 5400 - -stages: - - template: templates/stages/test-and-package-stage.yml diff --git a/.github/actions/run_tox/action.yml b/.github/actions/run_tox/action.yml index 1e365230f..91b400ad1 100644 --- a/.github/actions/run_tox/action.yml +++ b/.github/actions/run_tox/action.yml @@ -1,14 +1,34 @@ name: run_tox + +inputs: + AWS_ACCESS_KEY_ID: + description: 'access key ID for AWS' + AWS_SECRET_ACCESS_KEY: + description: 'access key for AWS' + AWS_TEST_FARM_PEM: + description: 'contents of AWS PEM file to be placed in $AWS_EC2_PEM_FILE from environment' + PIP_USE_PEP517: + description: 'a pip flag' + TOXENV: + description: 'the tox environment to run' + runs: using: composite steps: + - name: Create test farm pem file + if: contains(matrix.TOXENV, 'test-farm') + env: + PEM_CONTENTS: "${{ inputs.AWS_TEST_FARM_PEM }}" + run: |- + set -e + echo ${PEM_CONTENTS} >> $AWS_EC2_PEM_FILE + shell: bash - name: Run tox env: - AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" - AWS_EC2_PEM: "github-test-farm.pem" - PIP_USE_PEP517: "${{ matrix.PIP_USE_PEP517 }}" - TOXENV: "${{ matrix.TOXENV }}" + AWS_ACCESS_KEY_ID: "${{ inputs.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ inputs.AWS_SECRET_ACCESS_KEY }}" + PIP_USE_PEP517: "${{ inputs.PIP_USE_PEP517 }}" + TOXENV: "${{ inputs.TOXENV }}" run: |- set -e export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`" diff --git a/.github/actions/setup_tox/action.yml b/.github/actions/setup_tox/action.yml index a662f9ccd..6ff581593 100644 --- a/.github/actions/setup_tox/action.yml +++ b/.github/actions/setup_tox/action.yml @@ -1,9 +1,16 @@ name: setup_tox + +inputs: + AWS_TEST_FARM_PEM: + description: 'Contents of keyfile for AWS' + AWS_EC2_PEM_FILE: + description: 'Location of keyfile for AWS' + runs: using: composite steps: - name: Install MacOS dependencies - if: startsWith(matrix.IMAGE_NAME, 'macOS') + if: runner.os == 'macOS' run: |- set -e unset HOMEBREW_NO_INSTALL_FROM_API @@ -12,7 +19,7 @@ runs: brew install augeas shell: bash - name: Install Linux dependencies - if: startsWith(matrix.IMAGE_NAME, 'ubuntu') + if: runner.os == 'Linux' run: |- set -e sudo apt-get update @@ -30,9 +37,3 @@ runs: set -e python3 tools/pip_install.py tox shell: bash - - name: Create test farm pem file - if: contains(matrix.TOXENV, 'test-farm') - env: - PEM_CONTENTS: "${{ secrets.AWS_TEST_FARM_PEM }}" - run: 'echo ${PEM_CONTENTS} >> github-test-farm.pem' - shell: bash diff --git a/.github/workflows/extended_tests_jobs.yml b/.github/workflows/extended_tests_jobs.yml new file mode 100644 index 000000000..808028100 --- /dev/null +++ b/.github/workflows/extended_tests_jobs.yml @@ -0,0 +1,64 @@ +# Environment variables defined in a calling workflow are not accessible to this reusable workflow. Refer to the documentation for further details on this limitation. +name: extended_tests_jobs +on: + workflow_call: + +jobs: + test: + name: extended_test ${{ matrix.TOXENV }} ${{ matrix.PYTHON_VERSION }} + permissions: + contents: read + runs-on: + - 'ubuntu-22.04' + env: + uploadCoverage: ${{ inputs.uploadCoverage }} + strategy: + fail-fast: false + matrix: + PYTHON_VERSION: ['3.14'] + TOXENV: + - isolated-acme,isolated-certbot,isolated-apache,isolated-cloudflare,isolated-digitalocean,isolated-dnsimple,isolated-dnsmadeeasy,isolated-gehirn,isolated-google,isolated-linode,isolated-luadns,isolated-nsone,isolated-ovh,isolated-rfc2136,isolated-route53,isolated-sakuracloud,isolated-nginx + - nginx_compat + - modification + include: + - PYTHON_VERSION: '3.11' + TOXENV: py311 + - PYTHON_VERSION: '3.12' + TOXENV: py312 + - PYTHON_VERSION: '3.13' + TOXENV: py313 + - PYTHON_VERSION: '3.10' + TOXENV: integration-certbot-oldest + - PYTHON_VERSION: '3.10' + TOXENV: integration-nginx-oldest + - PYTHON_VERSION: '3.10' + TOXENV: integration + - PYTHON_VERSION: '3.11' + TOXENV: integration + - PYTHON_VERSION: '3.12' + TOXENV: integration + - PYTHON_VERSION: '3.13' + TOXENV: integration + # python 3.14 integration tests are not run here because they're run as + # part of the standard test suite + - PYTHON_VERSION: '3.12' + TOXENV: integration-dns-rfc2136 + - PYTHON_VERSION: '3.12' + TOXENV: test-farm-apache2 + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + - name: Setup tox + uses: "./.github/actions/setup_tox" + - name: Run tox + uses: "./.github/actions/run_tox" + env: + AWS_EC2_PEM_FILE: ${{ github.workspace }}/GHAKeyPair.pem + with: + AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}" + AWS_TEST_FARM_PEM: "${{ secrets.AWS_TEST_FARM_PEM }}" + PIP_USE_PEP517: "${{ matrix.PIP_USE_PEP517 }}" + TOXENV: "${{ matrix.TOXENV }}" + - name: Upload coverage + uses: "./.github/actions/upload_coverage" diff --git a/.github/workflows/full-test-suite.yml b/.github/workflows/full-test-suite.yml new file mode 100644 index 000000000..1db18ae8e --- /dev/null +++ b/.github/workflows/full-test-suite.yml @@ -0,0 +1,19 @@ +# Advanced pipeline for running our full test suite on demand. +name: certbot/full-test-suite +on: + push: + branches: + # When changing these triggers, please ensure the documentation under + # "Running tests in CI" is still correct. + - test-* + +jobs: + test_and_package_stage: + name: test_and_package_stage + uses: "./.github/workflows/test_and_package_stage.yml" + with: + # We don't publish our Docker images in this pipeline, but when building them + # for testing, let's use the nightly tag. + dockerTag: nightly + snapBuildTimeout: 5400 + secrets: inherit diff --git a/.github/workflows/packaging_jobs.yml b/.github/workflows/packaging_jobs.yml new file mode 100644 index 000000000..bdc552028 --- /dev/null +++ b/.github/workflows/packaging_jobs.yml @@ -0,0 +1,188 @@ +# Environment variables defined in a calling workflow are not accessible to this reusable workflow. Refer to the documentation for further details on this limitation. +name: packaging_jobs +on: + workflow_call: + inputs: + dockerTag: + description: 'docker tag to push to' + type: string + snapBuildTimeout: + description: 'timeout for snap builds' + type: number + +env: + dockerTag: ${{ inputs.dockerTag }} + snapBuildTimeout: ${{ inputs.snapBuildTimeout }} + +jobs: + docker_build: + runs-on: + - ubuntu-24.04 + # The default timeout of 60 minutes is a little low for compiling + # cryptography on ARM architectures. + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + DOCKER_ARCH: + - arm32v6 + - arm64v8 + - amd64 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: Build the Docker images + # We don't filter for the Docker Hub organization to continue to allow + # easy testing of these scripts on forks. + run: set -e && tools/docker/build.sh ${{ env.dockerTag }} ${{ matrix.DOCKER_ARCH }} + shell: bash + - name: Save the Docker images + run: |- + set -e + DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}') + docker save --output images.tar $DOCKER_IMAGES + shell: bash + # If the name of the tar file or artifact changes, the deploy stage will + # also need to be updated. + - name: Prepare Docker artifact + run: set -e && mv images.tar ${{ runner.temp }} + shell: bash + - name: Store Docker artifact + uses: actions/upload-artifact@v4.1.0 + with: + name: docker_${{ matrix.DOCKER_ARCH }} + path: "${{ runner.temp }}" + docker_test: + needs: + - docker_build + runs-on: + - ubuntu-22.04 + strategy: + fail-fast: false + matrix: + DOCKER_ARCH: + - arm32v6 + - arm64v8 + - amd64 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: Retrieve Docker images + uses: actions/download-artifact@v8.0.1 + with: + name: docker_${{ matrix.DOCKER_ARCH }} + github_token: "${{ secrets.GITHUB_TOKEN }}" + path: "${{ github.workspace }}" + repo: "${{ github.repository }}" + - name: Load Docker images + run: set -e && docker load --input ${{ github.workspace }}/images.tar + shell: bash + - name: Run integration tests for Docker images + run: set -e && tools/docker/test.sh ${{ env.dockerTag }} ${{ matrix.DOCKER_ARCH }} + shell: bash + snaps_build: + runs-on: + - ubuntu-22.04 + timeout-minutes: 0 + strategy: + fail-fast: false + matrix: + SNAP_ARCH: + - amd64 + - armhf + - arm64 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: Install dependencies + run: |- + set -e + sudo apt-get update + sudo apt-get install -y --no-install-recommends snapd + sudo snap install --classic snapcraft + - uses: actions/setup-python@v5.0.0 + with: + python-version: '3.12' + - name: Build snaps + env: + SNAPCRAFT_STORE_CREDENTIALS: "${{ secrets.LAUNCHPAD_CREDENTIALS }}" + run: |- + set -e + git config --global user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com" + git config --global user.name "${{ github.actor }}" + python3 tools/snap/build_remote.py ALL --archs ${{ matrix.SNAP_ARCH }} --timeout ${{ env.snapBuildTimeout }} + - name: Prepare artifacts + run: |- + set -e + mv *.snap ${{ runner.temp }} + mv certbot-dns-*/*.snap ${{ runner.temp }} + - name: Store snaps artifacts + uses: actions/upload-artifact@v4.1.0 + with: + name: snaps_${{ matrix.SNAP_ARCH }} + path: "${{ runner.temp }}" + snap_run: + needs: + - snaps_build + runs-on: + - ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - uses: actions/setup-python@v5.0.0 + with: + python-version: '3.12' + - name: Install dependencies + run: |- + set -e + sudo apt-get update + sudo apt-get install -y --no-install-recommends nginx-light snapd + python3 -m venv venv + venv/bin/python tools/pip_install.py -U tox + - name: Retrieve Certbot snaps + uses: actions/download-artifact@v8.0.1 + with: + name: snaps_amd64 + github_token: "${{ secrets.GITHUB_TOKEN }}" + path: "${{ github.workspace }}/snap" + repo: "${{ github.repository }}" + - name: Install Certbot snap + run: |- + set -e + sudo snap install --dangerous --classic snap/certbot_*.snap + - name: Run tox + run: |- + set -e + venv/bin/python -m tox run -e integration-external,apacheconftest-external-with-pebble + snap_dns_run: + needs: + - snaps_build + runs-on: + - ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + - name: Install dependencies + run: |- + set -e + sudo apt-get update + sudo apt-get install -y --no-install-recommends snapd + - uses: actions/setup-python@v5.0.0 + with: + python-version: '3.12' + - name: Retrieve Certbot snaps + uses: actions/download-artifact@v8.0.1 + with: + name: snaps_amd64 + github_token: "${{ secrets.GITHUB_TOKEN }}" + path: "${{ github.workspace }}/snap" + repo: "${{ github.repository }}" + - name: Prepare Certbot-CI + run: |- + set -e + python3 -m venv venv + venv/bin/python tools/pip_install.py -e certbot-ci + - name: Test DNS plugins snaps + run: |- + set -e + sudo -E venv/bin/pytest certbot-ci/src/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder ${{ github.workspace }}/snap --snap-arch amd64 diff --git a/.github/workflows/test_and_package_stage.yml b/.github/workflows/test_and_package_stage.yml new file mode 100644 index 000000000..1e74eceb5 --- /dev/null +++ b/.github/workflows/test_and_package_stage.yml @@ -0,0 +1,33 @@ +# Environment variables defined in a calling workflow are not accessible to this reusable workflow. Refer to the documentation for further details on this limitation. +name: test_and_package_stage +on: + workflow_call: + inputs: + dockerTag: + description: 'docker tag to push to' + type: string + snapBuildTimeout: + description: 'timeout for snap builds' + type: number + +jobs: + standard_tests_jobs: + name: standard_tests_jobs + uses: "./.github/workflows/standard_tests_jobs.yml" + permissions: + contents: read + extended_tests_jobs: + name: extended_tests_jobs + uses: "./.github/workflows/extended_tests_jobs.yml" + permissions: + contents: read + secrets: inherit + packaging_jobs: + name: packaging_jobs + uses: "./.github/workflows/packaging_jobs.yml" + permissions: + contents: read + with: + dockerTag: ${{ inputs.dockerTag }} + snapBuildTimeout: ${{ inputs.snapBuildTimeout }} + secrets: inherit