2014-11-27 00:27:26 -05:00
|
|
|
language: python
|
2019-09-17 18:24:53 -04:00
|
|
|
dist: xenial
|
2014-11-28 15:42:47 -05:00
|
|
|
|
2016-01-19 18:47:50 -05:00
|
|
|
cache:
|
|
|
|
|
directories:
|
|
|
|
|
- $HOME/.cache/pip
|
|
|
|
|
|
2017-06-01 12:03:54 -04:00
|
|
|
before_script:
|
2019-05-14 16:56:32 -04:00
|
|
|
- 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
|
|
|
|
|
# On Travis, the fastest parallelization for integration tests has proved to be 4.
|
|
|
|
|
- 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
|
2019-08-16 08:55:45 -04:00
|
|
|
# Use Travis retry feature for farm tests since they are flaky
|
|
|
|
|
- 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
|
2019-01-08 23:45:16 -05:00
|
|
|
- export TOX_TESTENV_PASSENV=TRAVIS
|
2014-11-28 15:42:47 -05:00
|
|
|
|
2019-02-06 15:47:56 -05:00
|
|
|
# Only build pushes to the master branch, PRs, and branches beginning with
|
|
|
|
|
# `test-` or of the form `digit(s).digit(s).x`. This reduces the number of
|
|
|
|
|
# simultaneous Travis runs, which speeds turnaround time on review since there
|
|
|
|
|
# is a cap of on the number of simultaneous runs.
|
|
|
|
|
branches:
|
|
|
|
|
only:
|
2019-07-10 19:30:06 -04:00
|
|
|
# apache-parser-v2 is a temporary branch for doing work related to
|
|
|
|
|
# rewriting the parser in the Apache plugin.
|
|
|
|
|
- apache-parser-v2
|
2019-02-06 15:47:56 -05:00
|
|
|
- master
|
|
|
|
|
- /^\d+\.\d+\.x$/
|
|
|
|
|
- /^test-.*$/
|
|
|
|
|
|
2019-03-06 17:49:43 -05:00
|
|
|
# Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
|
|
|
|
|
not-on-master: ¬-on-master
|
|
|
|
|
if: NOT (type = push AND branch = master)
|
|
|
|
|
|
2019-07-10 19:30:06 -04:00
|
|
|
# Jobs for the extended test suite are executed for cron jobs and pushes to
|
|
|
|
|
# non-development branches. See the explanation for apache-parser-v2 above.
|
2019-03-06 17:49:43 -05:00
|
|
|
extended-test-suite: &extended-test-suite
|
2019-07-10 19:30:06 -04:00
|
|
|
if: type = cron OR (type = push AND branch NOT IN (apache-parser-v2, master))
|
2019-03-06 17:49:43 -05:00
|
|
|
|
2015-12-27 06:50:30 -05:00
|
|
|
matrix:
|
|
|
|
|
include:
|
2019-03-06 17:49:43 -05:00
|
|
|
# Main test suite
|
2017-12-08 19:35:59 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=pebble TOXENV=integration
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
|
|
|
|
|
|
|
|
|
# This job is always executed, including on master
|
2016-09-30 23:27:04 -04:00
|
|
|
- python: "2.7"
|
2018-11-07 20:16:16 -05:00
|
|
|
env: TOXENV=py27-cover FYI="py27 tests + code coverage"
|
2019-03-06 17:49:43 -05:00
|
|
|
|
Lint certbot code on Python 3, and update Pylint to the latest version (#7551)
Part of #7550
This PR makes appropriate corrections to run pylint on Python 3.
Why not keeping the dependencies unchanged and just run pylint on Python 3?
Because the old version of pylint breaks horribly on Python 3 because of unsupported version of astroid.
Why updating pylint + astroid to the latest version ?
Because this version only fixes some internal errors occuring during the lint of Certbot code, and is also ready to run gracefully on Python 3.8.
Why upgrading mypy ?
Because the old version does not support the new version of astroid required to run pylint correctly.
Why not upgrading mypy to its latest version ?
Because this latest version includes a new typshed version, that adds a lot of new type definitions, and brings dozens of new errors on the Certbot codebase. I would like to fix that in a future PR.
That said so, the work has been to find the correct set of new dependency versions, then configure pylint for sane configuration errors in our situation, disable irrelevant lintings errors, then fixing (or ignoring for good reason) the remaining mypy errors.
I also made PyLint and MyPy checks run correctly on Windows.
* Start configuration
* Reconfigure travis
* Suspend a check specific to python 3. Start fixing code.
* Repair call_args
* Fix return + elif lints
* Reconfigure development to run mainly on python3
* Remove incompatible Python 3.4 jobs
* Suspend pylint in some assertions
* Remove pylint in dev
* Take first mypy that supports typed-ast>=1.4.0 to limit the migration path
* Various return + else lint errors
* Find a set of deps that is working with current mypy version
* Update local oldest requirements
* Remove all current pylint errors
* Rebuild letsencrypt-auto
* Update mypy to fix pylint with new astroid version, and fix mypy issues
* Explain type: ignore
* Reconfigure tox, fix none path
* Simplify pinning
* Remove useless directive
* Remove debugging code
* Remove continue
* Update requirements
* Disable unsubscriptable-object check
* Disable one check, enabling two more
* Plug certbot dev version for oldest requirements
* Remove useless disable directives
* Remove useless no-member disable
* Remove no-else-* checks. Use elif in symetric branches.
* Add back assertion
* Add new line
* Remove unused pylint disable
* Remove other pylint disable
2019-12-10 17:12:50 -05:00
|
|
|
- python: "3.7"
|
2016-09-30 23:27:04 -04:00
|
|
|
env: TOXENV=lint
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2018-04-13 19:10:58 -04:00
|
|
|
- python: "3.5"
|
|
|
|
|
env: TOXENV=mypy
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2016-12-20 19:24:33 -05:00
|
|
|
- python: "2.7"
|
2019-07-26 16:37:16 -04:00
|
|
|
# Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# cryptography we support cannot be compiled against the version of
|
|
|
|
|
# OpenSSL in Xenial or newer.
|
|
|
|
|
dist: trusty
|
2020-01-07 12:57:43 -05:00
|
|
|
env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest'
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2018-02-16 19:20:45 -05:00
|
|
|
- python: "3.4"
|
|
|
|
|
env: TOXENV=py34
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2018-07-06 12:08:40 -04:00
|
|
|
- python: "3.7"
|
|
|
|
|
env: TOXENV=py37
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2019-11-01 13:06:10 -04:00
|
|
|
- python: "3.8"
|
2019-09-16 14:14:26 -04:00
|
|
|
env: TOXENV=py38
|
|
|
|
|
<<: *not-on-master
|
2016-08-10 19:46:53 -04:00
|
|
|
- sudo: required
|
2017-12-08 19:35:59 -05:00
|
|
|
env: TOXENV=apache_compat
|
2016-08-10 19:46:53 -04:00
|
|
|
services: docker
|
|
|
|
|
before_install:
|
|
|
|
|
addons:
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2019-06-26 17:54:08 -04:00
|
|
|
- sudo: required
|
|
|
|
|
env: TOXENV=le_auto_xenial
|
|
|
|
|
services: docker
|
|
|
|
|
<<: *not-on-master
|
2018-05-11 09:12:10 -04:00
|
|
|
- python: "2.7"
|
2019-01-09 15:37:45 -05:00
|
|
|
env: TOXENV=apacheconftest-with-pebble
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2016-09-30 23:27:04 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env: TOXENV=nginxroundtrip
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *not-on-master
|
2016-09-30 23:27:04 -04:00
|
|
|
|
2019-03-06 17:49:43 -05:00
|
|
|
# Extended test suite on cron jobs and pushes to tested branches other than master
|
Move nginx_compat to nightly (#7001)
With the various optimizations already done and upcoming (certbot-ci), the time execution of integration tests have significantly decreased, allowing potentially a complete execution of a Travis PR job to be done within 5min30s.
However, one job is significantly longer that the other ones after this migration: this is nginx_compat, that takes more that 11min to finish. I tried to split the nginx_compat in terms of tested configuration and of tests to execute (auth, install, enhance). Both are not satisfactory:
splitting by configuration may work, but add a significant complexity in the tests
splitting by tests type is supported almost out-of-the-box, but fails to make two fast tests (see https://travis-ci.org/adferrand/certbot/builds/525892885?utm_source=github_status&utm_medium=notification for instance)
Since these tests are designed to check corner cases on the nginx parser, this is mostly useless to execute them on each PR, as the nginx parser is rarely updated.
After some discussion with @bmw, I think that we can just move the nginx_compat from the PR tests to the nightly tests. This PR does that.
2019-06-11 17:54:36 -04:00
|
|
|
- sudo: required
|
|
|
|
|
env: TOXENV=nginx_compat
|
|
|
|
|
services: docker
|
|
|
|
|
before_install:
|
|
|
|
|
addons:
|
|
|
|
|
<<: *extended-test-suite
|
2019-05-17 19:30:20 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env:
|
|
|
|
|
- TOXENV=travis-test-farm-apache2
|
|
|
|
|
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "2.7"
|
|
|
|
|
env:
|
|
|
|
|
- TOXENV=travis-test-farm-leauto-upgrades
|
|
|
|
|
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
git:
|
|
|
|
|
depth: false # This is needed to have the history to checkout old versions of certbot-auto.
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "2.7"
|
|
|
|
|
env:
|
|
|
|
|
- TOXENV=travis-test-farm-certonly-standalone
|
|
|
|
|
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "2.7"
|
|
|
|
|
env:
|
|
|
|
|
- TOXENV=travis-test-farm-sdists
|
|
|
|
|
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
|
|
|
|
|
<<: *extended-test-suite
|
2019-01-17 16:02:35 -05:00
|
|
|
- python: "3.7"
|
|
|
|
|
env: TOXENV=py37 CERTBOT_NO_PIN=1
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "2.7"
|
|
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-05-14 16:56:32 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
|
2019-09-17 18:24:53 -04:00
|
|
|
# Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# cryptography we support cannot be compiled against the version of
|
|
|
|
|
# OpenSSL in Xenial or newer.
|
|
|
|
|
dist: trusty
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
|
2019-09-17 18:24:53 -04:00
|
|
|
# Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# cryptography we support cannot be compiled against the version of
|
|
|
|
|
# OpenSSL in Xenial or newer.
|
|
|
|
|
dist: trusty
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "2.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
|
2019-09-17 18:24:53 -04:00
|
|
|
# Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# cryptography we support cannot be compiled against the version of
|
|
|
|
|
# OpenSSL in Xenial or newer.
|
|
|
|
|
dist: trusty
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-05-14 16:56:32 -04:00
|
|
|
- python: "2.7"
|
|
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
|
2019-09-17 18:24:53 -04:00
|
|
|
# Ubuntu Trusty or older must be used because the oldest version of
|
|
|
|
|
# cryptography we support cannot be compiled against the version of
|
|
|
|
|
# OpenSSL in Xenial or newer.
|
|
|
|
|
dist: trusty
|
2019-05-14 16:56:32 -04:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "3.4"
|
|
|
|
|
env: TOXENV=py34
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "3.5"
|
|
|
|
|
env: TOXENV=py35
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "3.6"
|
|
|
|
|
env: TOXENV=py36
|
|
|
|
|
<<: *extended-test-suite
|
|
|
|
|
- python: "3.7"
|
|
|
|
|
env: TOXENV=py37
|
|
|
|
|
<<: *extended-test-suite
|
2019-12-18 17:00:49 -05:00
|
|
|
- python: "3.8"
|
2019-09-16 14:14:26 -04:00
|
|
|
env: TOXENV=py38
|
|
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.4"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.4"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.5"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.5"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.6"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.6"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- python: "3.7"
|
2019-05-14 16:56:32 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
2019-01-16 15:16:54 -05:00
|
|
|
sudo: required
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-12-18 17:00:49 -05:00
|
|
|
- python: "3.8"
|
2019-09-16 14:14:26 -04:00
|
|
|
env: ACME_SERVER=boulder-v1 TOXENV=integration
|
|
|
|
|
<<: *extended-test-suite
|
2019-12-18 17:00:49 -05:00
|
|
|
- python: "3.8"
|
2019-09-16 14:14:26 -04:00
|
|
|
env: ACME_SERVER=boulder-v2 TOXENV=integration
|
|
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- sudo: required
|
2019-01-16 16:17:37 -05:00
|
|
|
env: TOXENV=le_auto_jessie
|
2019-01-16 15:16:54 -05:00
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- sudo: required
|
|
|
|
|
env: TOXENV=le_auto_centos6
|
|
|
|
|
services: docker
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- sudo: required
|
|
|
|
|
env: TOXENV=docker_dev
|
|
|
|
|
services: docker
|
|
|
|
|
addons:
|
|
|
|
|
apt:
|
|
|
|
|
packages: # don't install nginx and apache
|
|
|
|
|
- libaugeas0
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- language: generic
|
|
|
|
|
env: TOXENV=py27
|
|
|
|
|
os: osx
|
2019-06-19 17:09:30 -04:00
|
|
|
# Using this osx_image is a workaround for
|
|
|
|
|
# https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
|
|
|
|
|
osx_image: xcode10.2
|
2019-01-16 15:16:54 -05:00
|
|
|
addons:
|
|
|
|
|
homebrew:
|
|
|
|
|
packages:
|
|
|
|
|
- augeas
|
|
|
|
|
- python2
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2019-01-16 15:16:54 -05:00
|
|
|
- language: generic
|
|
|
|
|
env: TOXENV=py3
|
|
|
|
|
os: osx
|
2019-06-19 17:09:30 -04:00
|
|
|
# Using this osx_image is a workaround for
|
|
|
|
|
# https://travis-ci.community/t/xcode-8-3-homebrew-outdated-error/3798.
|
|
|
|
|
osx_image: xcode10.2
|
2019-01-16 15:16:54 -05:00
|
|
|
addons:
|
|
|
|
|
homebrew:
|
|
|
|
|
packages:
|
|
|
|
|
- augeas
|
|
|
|
|
- python3
|
2019-03-06 17:49:43 -05:00
|
|
|
<<: *extended-test-suite
|
2015-10-10 16:31:35 -04:00
|
|
|
|
2015-11-11 18:46:39 -05:00
|
|
|
# container-based infrastructure
|
|
|
|
|
sudo: false
|
2015-11-04 20:41:48 -05:00
|
|
|
|
2015-07-19 07:22:57 -04:00
|
|
|
addons:
|
2015-09-05 13:23:38 -04:00
|
|
|
apt:
|
2016-02-05 15:26:08 -05:00
|
|
|
packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
|
2015-09-05 13:17:25 -04:00
|
|
|
- python-dev
|
|
|
|
|
- gcc
|
|
|
|
|
- libaugeas0
|
|
|
|
|
- libssl-dev
|
|
|
|
|
- libffi-dev
|
|
|
|
|
- ca-certificates
|
2016-04-14 13:20:23 -04:00
|
|
|
# For certbot-nginx integration testing
|
2015-09-05 13:20:26 -04:00
|
|
|
- nginx-light
|
|
|
|
|
- openssl
|
2015-07-19 07:22:57 -04:00
|
|
|
|
2019-07-02 13:02:00 -04:00
|
|
|
# tools/pip_install.py is used to pin packages to a known working version
|
|
|
|
|
# except in tests where the environment variable CERTBOT_NO_PIN is set.
|
|
|
|
|
# virtualenv is listed here explicitly to make sure it is upgraded when
|
|
|
|
|
# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
|
|
|
|
|
# version of virtualenv.
|
2019-08-16 08:55:45 -04:00
|
|
|
install: 'tools/pip_install.py -U codecov tox virtualenv'
|
|
|
|
|
# Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
|
|
|
|
|
# script command. It is set only to `travis_retry` during farm tests, in
|
|
|
|
|
# order to trigger the Travis retry feature, and compensate the inherent
|
|
|
|
|
# flakiness of these specific tests.
|
|
|
|
|
script: '$TRAVIS_RETRY tox'
|
2015-06-25 04:26:02 -04:00
|
|
|
|
Improve codecov report integration to CI in Certbot (#6934)
So, we observed lately several inconsistencies in how Codecov behave toward the CI pipeline for PRs in Certbot. One example is #6888. The most annoying thing is that the build of PR is **temporary** marked as failed, until all coverage are run.
The correction on the latter is done in two PRs. This is the first part.
TL;DR
This PR separates the Codecov report in two: one for coverage executed on Windows, one for Linux. This is the correct way to do regarding our current CI pipeline. Actions are required by a GitHub administrator of Certbot once this PR is merged.
Complete explanation
So the failure stated in the introduction is essentially due to several things interacting together:
* AppVeyor generates a coverage report for Windows, that have a coverage value a little lower than on Linux (96%)
* Travis generates a coverage report for Linux. Its coverage is higher, and slowly decrease as more specific Windows code is added to Certbot, that cannot be tested on Travis
* Since AppVeyor saw its capacity increasing, it finishes its coverage job before the one from Travis
* Certbot GitHub repo is configured to require the coverage pipeline to succeed (in whatever that means) to success the overall PR build
So here the suite of events:
1) PR is issued. GitHub expect three pipeline to succeed: AppVeyor CI, Travis CI and Codecov (displayed in the PR page)
2) Codecov receive first the report of AppVeyor coverage. It is 96%. It is a failure for now, because coverage in master (AppVeyor+Travis) is 98.6%.
3) GitHub is reported of the failure on Codecov, so fail the PR build
4) Codecov receive then the report of Travis coverage. It is 98%. It merges it with the report from AppVeyor, leading to the 98.6%. The failure becomes a success.
5) GitHub is reported of the success on Codecov, so, nevermind, the PR build is a success finally!
So we have a CI flow that change its mind. Great. This is because of 2) and 4), and we could expect that Codecov should handle that. This is not the case: it is somewhat misleading, because Codecov adverts a lot about its capability to merge reports, including from different CI. But it is about the final state, not about the transient state, while reports are progressively received.
Two things to things that a transient state is existing, with a result that can change:
* first, from Codecov doc itself, explaining that reports should not be trusted during the CI pipeline execution: https://docs.codecov.io/docs/ci-service-relationship#section-checking-ci-status
* second, is an example of transient state of `cryptography` project, this is advert by Codecov to be a reference of the implementation:
As you can see above, build state of `cryptography` is failing after the first report is received, and until all coverage reports from Travis are received.
So, what can we do about it? Thing is, we are aggregating coverage from very two unrelated sources (two different OS systems), and Codecov has something for that. This is flags: https://docs.codecov.io/docs/flags
Flags allow to flag coverage material depending on any logic you apply to the command that uploaded the coverage report (eg. `codecov -F a_flag`). Then, several logics can be applied on it, for instance having in Codecov UI the capability to filter the coverage other a flag, having status of build for each flag and ... having a report for a specific flag.
So:
1) I modified Travis and AppVeyor to send their report under a specific flag: `linux` or `windows`
2) I created a project specific `.codecov.yml` configuration in Certbot repository, to instruct Codecov to push two separate reports on GitHub build: one for Linux, one for Windows. Each report can be validated against its specific coverage from the `master` branch (more on this just after)
With all of this, now the GitHub is succeeding, because each coverage is validated independently.
I think it is the good approach, because it solves the specific issue here, and because it reflects the logic behind: merging coverage from different OS architectures does not make much sense. It would be a long-term problem, because as I said at the beginning, coverages will slowly decrease as more platform specific code is added in Certbot.
Now, it is not finished. Two things need to be done: an administrator action, and a second PR
Administrator action
Certbot GitHub as a a branch protection rule (Settings > Branches > Branch protection rules). It needs to be changed.
Indeed this rule is expecting the full coverage report (named `codecov/project`) to be valid on a PR. It needs to be changed to expect two coverage reports: `codecov/project/linux` and `codecov/project/windows`. The `codecov/project` needs to be removed.
This can be done once this PR is merged, and the specific coverage reports have been generated on master.
Second PR
Once this PR is merged and administrative actions have been done. I will make a new PR modifying `.codecov.yml` with two things:
* disable the faulty full coverage report, that is not required anymore by GitHub branch protection rules
* modify the `linux` and `windows` reports to validate against the relevant coverage calculated from `master` (indeed, in this PR it is a fixed ratio rule, since the coverage to compare on master is the full coverage one, significantly higher)
* Tag reports
* Set per-project codecov configuration
2019-04-09 14:43:26 -04:00
|
|
|
after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'
|
2014-12-03 07:32:52 -05:00
|
|
|
|
|
|
|
|
notifications:
|
|
|
|
|
email: false
|
2015-03-31 14:33:14 -04:00
|
|
|
irc:
|
|
|
|
|
channels:
|
2019-05-29 02:54:26 -04:00
|
|
|
# This is set to a secure variable to prevent forks from sending
|
|
|
|
|
# notifications. This value was created by installing
|
|
|
|
|
# https://github.com/travis-ci/travis.rb and running
|
|
|
|
|
# `travis encrypt "chat.freenode.net#certbot-devel"`.
|
|
|
|
|
- secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
|
2018-05-23 16:57:22 -04:00
|
|
|
on_cancel: never
|
2015-03-31 14:33:14 -04:00
|
|
|
on_success: never
|
|
|
|
|
on_failure: always
|
