2014-11-27 00:27:26 -05:00
|
|
|
language: python
|
2014-11-28 15:42:47 -05:00
|
|
|
|
2016-01-19 18:47:50 -05:00
|
|
|
cache:
|
|
|
|
|
directories:
|
|
|
|
|
- $HOME/.cache/pip
|
|
|
|
|
|
2017-06-01 12:03:54 -04:00
|
|
|
before_script:
|
2019-05-14 16:56:32 -04:00
|
|
|
- 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
|
|
|
|
|
# On Travis, the fastest parallelization for integration tests has proved to be 4.
|
|
|
|
|
- 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
|
2019-08-16 08:55:45 -04:00
|
|
|
# Use Travis retry feature for farm tests since they are flaky
|
|
|
|
|
- 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
|
2019-01-08 23:45:16 -05:00
|
|
|
- export TOX_TESTENV_PASSENV=TRAVIS
|
2014-11-28 15:42:47 -05:00
|
|
|
|
2019-02-06 15:47:56 -05:00
|
|
|
# Only build pushes to the master branch, PRs, and branches beginning with
|
|
|
|
|
# `test-` or of the form `digit(s).digit(s).x`. This reduces the number of
|
|
|
|
|
# simultaneous Travis runs, which speeds turnaround time on review since there
|
|
|
|
|
# is a cap of on the number of simultaneous runs.
|
|
|
|
|
branches:
|
|
|
|
|
only:
|
2019-07-10 19:30:06 -04:00
|
|
|
# apache-parser-v2 is a temporary branch for doing work related to
|
|
|
|
|
# rewriting the parser in the Apache plugin.
|
|
|
|
|
- apache-parser-v2
|
2019-02-06 15:47:56 -05:00
|
|
|
- master
|
|
|
|
|
- /^\d+\.\d+\.x$/
|
|
|
|
|
- /^test-.*$/
|
|
|
|
|
|
2019-03-06 17:49:43 -05:00
|
|
|
# Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
|
|
|
|
|
not-on-master: ¬-on-master
|
|
|
|
|
if: NOT (type = push AND branch = master)
|
|
|
|
|
|
2019-07-10 19:30:06 -04:00
|
|
|
# Jobs for the extended test suite are executed for cron jobs and pushes to
|
|
|
|
|
# non-development branches. See the explanation for apache-parser-v2 above.
|
2019-03-06 17:49:43 -05:00
|
|
|
extended-test-suite: &extended-test-suite
|
2019-07-10 19:30:06 -04:00
|
|
|
if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
|
2019-03-06 17:49:43 -05:00
|
|
|
|
2015-12-27 06:50:30 -05:00
|
|
|
matrix:
|
|
|
|
|
include:
|
2019-08-27 17:02:08 -04:00
|
|
|
# # Main test suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: ACME_SERVER=pebble TOXENV=integration
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
#
|
|
|
|
|
# # This job is always executed, including on master
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: TOXENV=py27-cover FYI="py27 tests + code coverage"
|
|
|
|
|
#
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: TOXENV=lint
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "3.4"
|
|
|
|
|
# env: TOXENV=mypy
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "3.5"
|
|
|
|
|
# env: TOXENV=mypy
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# # Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# # cryptography we support cannot be compiled against the version of
|
|
|
|
|
# # OpenSSL in Xenial or newer.
|
|
|
|
|
# dist: trusty
|
|
|
|
|
# env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "3.4"
|
|
|
|
|
# env: TOXENV=py34
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "3.7"
|
|
|
|
|
# dist: xenial
|
|
|
|
|
# env: TOXENV=py37
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=apache_compat
|
|
|
|
|
# services: docker
|
|
|
|
|
# before_install:
|
|
|
|
|
# addons:
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=le_auto_xenial
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: TOXENV=apacheconftest-with-pebble
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: TOXENV=nginxroundtrip
|
|
|
|
|
# <<: *not-on-master
|
|
|
|
|
#
|
|
|
|
|
# # Extended test suite on cron jobs and pushes to tested branches other than master
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=nginx_compat
|
|
|
|
|
# services: docker
|
|
|
|
|
# before_install:
|
|
|
|
|
# addons:
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env:
|
|
|
|
|
# - TOXENV=travis-test-farm-apache2
|
|
|
|
|
# - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env:
|
|
|
|
|
# - TOXENV=travis-test-farm-leauto-upgrades
|
|
|
|
|
# - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
# git:
|
|
|
|
|
# depth: false # This is needed to have the history to checkout old versions of certbot-auto.
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env:
|
|
|
|
|
# - TOXENV=travis-test-farm-certonly-standalone
|
|
|
|
|
# - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env:
|
|
|
|
|
# - TOXENV=travis-test-farm-sdists
|
|
|
|
|
# - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.7"
|
|
|
|
|
# env: TOXENV=py37 CERTBOT_NO_PIN=1
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "2.7"
|
|
|
|
|
# env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
2019-05-14 16:56:32 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-05-14 16:56:32 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
|
|
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
|
|
|
|
<<: *extended-test-suite
|
2019-08-27 17:02:08 -04:00
|
|
|
# - python: "3.4"
|
|
|
|
|
# env: TOXENV=py34
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.5"
|
|
|
|
|
# env: TOXENV=py35
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.6"
|
|
|
|
|
# env: TOXENV=py36
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.7"
|
|
|
|
|
# dist: xenial
|
|
|
|
|
# env: TOXENV=py37
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.4"
|
|
|
|
|
# env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.4"
|
|
|
|
|
# env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.5"
|
|
|
|
|
# env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.5"
|
|
|
|
|
# env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.6"
|
|
|
|
|
# env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.6"
|
|
|
|
|
# env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.7"
|
|
|
|
|
# dist: xenial
|
|
|
|
|
# env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - python: "3.7"
|
|
|
|
|
# dist: xenial
|
|
|
|
|
# env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
# sudo: required
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=le_auto_jessie
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=le_auto_centos6
|
|
|
|
|
# services: docker
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - sudo: required
|
|
|
|
|
# env: TOXENV=docker_dev
|
|
|
|
|
# services: docker
|
|
|
|
|
# addons:
|
|
|
|
|
# apt:
|
|
|
|
|
# packages: # don't install nginx and apache
|
|
|
|
|
# - libaugeas0
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - language: generic
|
|
|
|
|
# env: TOXENV=py27
|
|
|
|
|
# os: osx
|
|
|
|
|
# # Using this osx_image is a workaround for
|
|
|
|
|
# # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
|
|
|
|
|
# osx_image: xcode10.2
|
|
|
|
|
# addons:
|
|
|
|
|
# homebrew:
|
|
|
|
|
# packages:
|
|
|
|
|
# - augeas
|
|
|
|
|
# - python2
|
|
|
|
|
# <<: *extended-test-suite
|
|
|
|
|
# - language: generic
|
|
|
|
|
# env: TOXENV=py3
|
|
|
|
|
# os: osx
|
|
|
|
|
# # Using this osx_image is a workaround for
|
|
|
|
|
# # https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
|
|
|
|
|
# osx_image: xcode10.2
|
|
|
|
|
# addons:
|
|
|
|
|
# homebrew:
|
|
|
|
|
# packages:
|
|
|
|
|
# - augeas
|
|
|
|
|
# - python3
|
|
|
|
|
# <<: *extended-test-suite
|
2015-10-10 16:31:35 -04:00
|
|
|
|
2015-11-11 18:46:39 -05:00
|
|
|
# container-based infrastructure
|
|
|
|
|
sudo: false
|
2015-11-04 20:41:48 -05:00
|
|
|
|
2015-07-19 07:22:57 -04:00
|
|
|
addons:
|
2015-09-05 13:23:38 -04:00
|
|
|
apt:
|
2016-02-05 15:26:08 -05:00
|
|
|
packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
|
2015-09-05 13:17:25 -04:00
|
|
|
- python-dev
|
|
|
|
|
- gcc
|
|
|
|
|
- libaugeas0
|
|
|
|
|
- libssl-dev
|
|
|
|
|
- libffi-dev
|
|
|
|
|
- ca-certificates
|
2016-04-14 13:20:23 -04:00
|
|
|
# For certbot-nginx integration testing
|
2015-09-05 13:20:26 -04:00
|
|
|
- nginx-light
|
|
|
|
|
- openssl
|
2015-07-19 07:22:57 -04:00
|
|
|
|
2019-07-02 13:02:00 -04:00
|
|
|
# tools/pip_install.py is used to pin packages to a known working version
|
|
|
|
|
# except in tests where the environment variable CERTBOT_NO_PIN is set.
|
|
|
|
|
# virtualenv is listed here explicitly to make sure it is upgraded when
|
|
|
|
|
# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
|
|
|
|
|
# version of virtualenv.
|
2019-08-16 08:55:45 -04:00
|
|
|
install: 'tools/pip_install.py -U codecov tox virtualenv'
|
|
|
|
|
# Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
|
|
|
|
|
# script command. It is set only to `travis_retry` during farm tests, in
|
|
|
|
|
# order to trigger the Travis retry feature, and compensate the inherent
|
|
|
|
|
# flakiness of these specific tests.
|
|
|
|
|
script: '$TRAVIS_RETRY tox'
|
2015-06-25 04:26:02 -04:00
|
|
|
|
Improve codecov report integration to CI in Certbot (#6934)
So, we observed lately several inconsistencies in how Codecov behave toward the CI pipeline for PRs in Certbot. One example is #6888. The most annoying thing is that the build of PR is **temporary** marked as failed, until all coverage are run.
The correction on the latter is done in two PRs. This is the first part.
TL;DR
This PR separates the Codecov report in two: one for coverage executed on Windows, one for Linux. This is the correct way to do regarding our current CI pipeline. Actions are required by a GitHub administrator of Certbot once this PR is merged.
Complete explanation
So the failure stated in the introduction is essentially due to several things interacting together:
* AppVeyor generates a coverage report for Windows, that have a coverage value a little lower than on Linux (96%)
* Travis generates a coverage report for Linux. Its coverage is higher, and slowly decrease as more specific Windows code is added to Certbot, that cannot be tested on Travis
* Since AppVeyor saw its capacity increasing, it finishes its coverage job before the one from Travis
* Certbot GitHub repo is configured to require the coverage pipeline to succeed (in whatever that means) to success the overall PR build
So here the suite of events:
1) PR is issued. GitHub expect three pipeline to succeed: AppVeyor CI, Travis CI and Codecov (displayed in the PR page)
2) Codecov receive first the report of AppVeyor coverage. It is 96%. It is a failure for now, because coverage in master (AppVeyor+Travis) is 98.6%.
3) GitHub is reported of the failure on Codecov, so fail the PR build
4) Codecov receive then the report of Travis coverage. It is 98%. It merges it with the report from AppVeyor, leading to the 98.6%. The failure becomes a success.
5) GitHub is reported of the success on Codecov, so, nevermind, the PR build is a success finally!
So we have a CI flow that change its mind. Great. This is because of 2) and 4), and we could expect that Codecov should handle that. This is not the case: it is somewhat misleading, because Codecov adverts a lot about its capability to merge reports, including from different CI. But it is about the final state, not about the transient state, while reports are progressively received.
Two things to things that a transient state is existing, with a result that can change:
* first, from Codecov doc itself, explaining that reports should not be trusted during the CI pipeline execution: https://docs.codecov.io/docs/ci-service-relationship#section-checking-ci-status
* second, is an example of transient state of `cryptography` project, this is advert by Codecov to be a reference of the implementation:
As you can see above, build state of `cryptography` is failing after the first report is received, and until all coverage reports from Travis are received.
So, what can we do about it? Thing is, we are aggregating coverage from very two unrelated sources (two different OS systems), and Codecov has something for that. This is flags: https://docs.codecov.io/docs/flags
Flags allow to flag coverage material depending on any logic you apply to the command that uploaded the coverage report (eg. `codecov -F a_flag`). Then, several logics can be applied on it, for instance having in Codecov UI the capability to filter the coverage other a flag, having status of build for each flag and ... having a report for a specific flag.
So:
1) I modified Travis and AppVeyor to send their report under a specific flag: `linux` or `windows`
2) I created a project specific `.codecov.yml` configuration in Certbot repository, to instruct Codecov to push two separate reports on GitHub build: one for Linux, one for Windows. Each report can be validated against its specific coverage from the `master` branch (more on this just after)
With all of this, now the GitHub is succeeding, because each coverage is validated independently.
I think it is the good approach, because it solves the specific issue here, and because it reflects the logic behind: merging coverage from different OS architectures does not make much sense. It would be a long-term problem, because as I said at the beginning, coverages will slowly decrease as more platform specific code is added in Certbot.
Now, it is not finished. Two things need to be done: an administrator action, and a second PR
Administrator action
Certbot GitHub as a a branch protection rule (Settings > Branches > Branch protection rules). It needs to be changed.
Indeed this rule is expecting the full coverage report (named `codecov/project`) to be valid on a PR. It needs to be changed to expect two coverage reports: `codecov/project/linux` and `codecov/project/windows`. The `codecov/project` needs to be removed.
This can be done once this PR is merged, and the specific coverage reports have been generated on master.
Second PR
Once this PR is merged and administrative actions have been done. I will make a new PR modifying `.codecov.yml` with two things:
* disable the faulty full coverage report, that is not required anymore by GitHub branch protection rules
* modify the `linux` and `windows` reports to validate against the relevant coverage calculated from `master` (indeed, in this PR it is a fixed ratio rule, since the coverage to compare on master is the full coverage one, significantly higher)
* Tag reports
* Set per-project codecov configuration
2019-04-09 14:43:26 -04:00
|
|
|
after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
|
2014-12-03 07:32:52 -05:00
|
|
|
|
2019-08-27 17:02:08 -04:00
|
|
|
#notifications:
|
|
|
|
|
# email: false
|
|
|
|
|
# irc:
|
|
|
|
|
# channels:
|
|
|
|
|
# # This is set to a secure variable to prevent forks from sending
|
|
|
|
|
# # notifications. This value was created by installing
|
|
|
|
|
# # https://github.com/travis-ci/travis.rb and running
|
|
|
|
|
# # `travis encrypt "chat.freenode.net#certbot-devel"`.
|
|
|
|
|
# - secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
|
|
|
|
|
# on_cancel: never
|
|
|
|
|
# on_success: never
|
|
|
|
|
# on_failure: always
|
