diff --git a/.azure-pipelines/main.yml b/.azure-pipelines/main.yml index cae4f799c..d3c3a152e 100644 --- a/.azure-pipelines/main.yml +++ b/.azure-pipelines/main.yml @@ -1,8 +1,10 @@ -trigger: none +# We run the test suite on commits to master so codecov gets coverage data +# about the master branch and can use it to track coverage changes. +trigger: + - master pr: - master - '*.x' jobs: - template: templates/jobs/standard-tests-jobs.yml - diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 1e6d7a352..5d0044d19 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -11,6 +11,7 @@ jobs: macos-cover: IMAGE_NAME: macOS-12 TOXENV: cover + UPLOAD_COVERAGE: 1 windows-py37: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.7 @@ -19,10 +20,12 @@ jobs: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.9 TOXENV: cover-win + UPLOAD_COVERAGE: 1 windows-integration-certbot: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.9 TOXENV: integration-certbot + UPLOAD_COVERAGE: 1 linux-oldest-tests-1: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.7 @@ -38,6 +41,7 @@ jobs: linux-cover: IMAGE_NAME: ubuntu-22.04 TOXENV: cover + UPLOAD_COVERAGE: 1 linux-lint: IMAGE_NAME: ubuntu-22.04 TOXENV: lint-posix @@ -49,6 +53,7 @@ jobs: PYTHON_VERSION: 3.8 TOXENV: integration ACME_SERVER: pebble + UPLOAD_COVERAGE: 1 apache-compat: IMAGE_NAME: ubuntu-22.04 TOXENV: apache_compat diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index fbda960a5..a85e1519a 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -55,3 +55,28 @@ steps: AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) AWS_EC2_PEM_FILE: $(testFarmPem.secureFilePath) displayName: Run tox + # For now, let's omit `set -e` and avoid the script exiting with a nonzero + # status code to prevent problems here from causing build failures. If + # this turns out to work well, we can change this. + - bash: | + python3 tools/pip_install.py -I coverage + case "$AGENT_OS" in + Darwin) + CODECOV_URL="https://uploader.codecov.io/latest/macos/codecov" + ;; + Linux) + CODECOV_URL="https://uploader.codecov.io/latest/linux/codecov" + ;; + Windows_NT) + CODECOV_URL="https://uploader.codecov.io/latest/windows/codecov.exe" + ;; + *) + echo "Unexpected OS" + exit 0 + esac + curl --retry 3 -o codecov "$CODECOV_URL" + chmod +x codecov + coverage xml + ./codecov || echo "Uploading coverage data failed" + condition: ne(variables['UPLOAD_COVERAGE'], '') + displayName: Upload coverage data diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000..23d2fbbe9 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,7 @@ +# This disables all reporting from codecov. Let's just set it up to collect +# data for now and then we can play with the settings here. +comment: false +coverage: + status: + project: off + patch: off