Start sending coverage data to codecov (#9544)

* set up codecov

* export coverage data to xml

.azure-pipelines/main.yml

@@ -1,8 +1,10 @@
-trigger: none
+# We run the test suite on commits to master so codecov gets coverage data
+# about the master branch and can use it to track coverage changes.
+trigger:
+  - master
 pr:
   - master
   - '*.x'
 
 jobs:
   - template: templates/jobs/standard-tests-jobs.yml
-

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -11,6 +11,7 @@ jobs:
         macos-cover:
           IMAGE_NAME: macOS-12
           TOXENV: cover
+          UPLOAD_COVERAGE: 1
         windows-py37:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.7
@@ -19,10 +20,12 @@ jobs:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
           TOXENV: cover-win
+          UPLOAD_COVERAGE: 1
         windows-integration-certbot:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
           TOXENV: integration-certbot
+          UPLOAD_COVERAGE: 1
         linux-oldest-tests-1:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.7
@@ -38,6 +41,7 @@ jobs:
         linux-cover:
           IMAGE_NAME: ubuntu-22.04
           TOXENV: cover
+          UPLOAD_COVERAGE: 1
         linux-lint:
           IMAGE_NAME: ubuntu-22.04
           TOXENV: lint-posix
@@ -49,6 +53,7 @@ jobs:
           PYTHON_VERSION: 3.8
           TOXENV: integration
           ACME_SERVER: pebble
+          UPLOAD_COVERAGE: 1
         apache-compat:
           IMAGE_NAME: ubuntu-22.04
           TOXENV: apache_compat

.azure-pipelines/templates/steps/tox-steps.yml

@@ -55,3 +55,28 @@ steps:
       AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
       AWS_EC2_PEM_FILE: $(testFarmPem.secureFilePath)
     displayName: Run tox
+    # For now, let's omit `set -e` and avoid the script exiting with a nonzero
+    # status code to prevent problems here from causing build failures.  If
+    # this turns out to work well, we can change this.
+  - bash: |
+      python3 tools/pip_install.py -I coverage
+      case "$AGENT_OS" in
+        Darwin)
+          CODECOV_URL="https://uploader.codecov.io/latest/macos/codecov"
+          ;;
+        Linux)
+          CODECOV_URL="https://uploader.codecov.io/latest/linux/codecov"
+          ;;
+        Windows_NT)
+          CODECOV_URL="https://uploader.codecov.io/latest/windows/codecov.exe"
+          ;;
+        *)
+          echo "Unexpected OS"
+          exit 0
+      esac
+      curl --retry 3 -o codecov "$CODECOV_URL"
+      chmod +x codecov
+      coverage xml
+      ./codecov || echo "Uploading coverage data failed"
+    condition: ne(variables['UPLOAD_COVERAGE'], '')
+    displayName: Upload coverage data

.github/codecov.yml

@@ -0,0 +1,7 @@
+# This disables all reporting from codecov. Let's just set it up to collect
+# data for now and then we can play with the settings here.
+comment: false
+coverage:
+  status:
+    project: off
+    patch: off