mirror of
https://github.com/certbot/certbot.git
synced 2026-06-04 06:15:36 -04:00
8b684e9b95
You can see the full test suite running at https://travis-ci.com/certbot/certbot/builds/112291892. A few noteworthy things: --fast is included because without, the tests would sometimes reach Travis' 50 minute timeout even with 1 test script per Travis build. The only script that is run at release time which is not being run here is https://github.com/certbot/certbot/blob/master/tests/letstest/scripts/test_tests.sh because that script runs tests on the packages installed by certbot-auto which won't be updated until midway through a release. We check TRAVIS_PULL_REQUEST and error out if it is not false for simplicity which should be fine because these tests are never run on PRs. The reason it's more complex to run test farm tests on PRs is the test farm tests need a named branch to pull from and Travis effectively merges the PR into the target branch before running tests complicating this. I don't think this should block this PRs, but the one final change we may want to make to the current setup is #7071. * Add encrypted private key. * Add test farm tests to tox and travis. * Change magic profile name. * Further split test farm tests. * Build local branch. * more depth
323 lines
10 KiB
INI
323 lines
10 KiB
INI
# Tox (http://tox.testrun.org/) is a tool for running tests in
|
|
# multiple virtualenvs. To use it, "pip install tox" and then run
|
|
# "tox" from this directory.
|
|
|
|
[tox]
|
|
skipsdist = true
|
|
envlist = modification,py3,py27-cover,lint,mypy
|
|
|
|
[base]
|
|
# pip installs the requested packages in editable mode
|
|
pip_install = python {toxinidir}/tools/pip_install_editable.py
|
|
# pip installs the requested packages in editable mode and runs unit tests on
|
|
# them. Each package is installed and tested in the order they are provided
|
|
# before the script moves on to the next package. All dependencies are pinned
|
|
# to a specific version for increased stability for developers.
|
|
install_and_test = python {toxinidir}/tools/install_and_test.py
|
|
dns_packages =
|
|
certbot-dns-cloudflare \
|
|
certbot-dns-cloudxns \
|
|
certbot-dns-digitalocean \
|
|
certbot-dns-dnsimple \
|
|
certbot-dns-dnsmadeeasy \
|
|
certbot-dns-gehirn \
|
|
certbot-dns-google \
|
|
certbot-dns-linode \
|
|
certbot-dns-luadns \
|
|
certbot-dns-nsone \
|
|
certbot-dns-ovh \
|
|
certbot-dns-rfc2136 \
|
|
certbot-dns-route53 \
|
|
certbot-dns-sakuracloud
|
|
all_packages =
|
|
acme[dev] \
|
|
.[dev] \
|
|
certbot-apache \
|
|
{[base]dns_packages} \
|
|
certbot-nginx \
|
|
certbot-postfix \
|
|
letshelp-certbot
|
|
install_packages =
|
|
python {toxinidir}/tools/pip_install_editable.py {[base]all_packages}
|
|
source_paths =
|
|
acme/acme
|
|
certbot
|
|
certbot-apache/certbot_apache
|
|
certbot-compatibility-test/certbot_compatibility_test
|
|
certbot-dns-cloudflare/certbot_dns_cloudflare
|
|
certbot-dns-cloudxns/certbot_dns_cloudxns
|
|
certbot-dns-digitalocean/certbot_dns_digitalocean
|
|
certbot-dns-dnsimple/certbot_dns_dnsimple
|
|
certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy
|
|
certbot-dns-gehirn/certbot_dns_gehirn
|
|
certbot-dns-google/certbot_dns_google
|
|
certbot-dns-linode/certbot_dns_linode
|
|
certbot-dns-luadns/certbot_dns_luadns
|
|
certbot-dns-nsone/certbot_dns_nsone
|
|
certbot-dns-ovh/certbot_dns_ovh
|
|
certbot-dns-rfc2136/certbot_dns_rfc2136
|
|
certbot-dns-route53/certbot_dns_route53
|
|
certbot-dns-sakuracloud/certbot_dns_sakuracloud
|
|
certbot-nginx/certbot_nginx
|
|
certbot-postfix/certbot_postfix
|
|
letshelp-certbot/letshelp_certbot
|
|
tests/lock_test.py
|
|
|
|
[testenv]
|
|
passenv =
|
|
CERTBOT_NO_PIN
|
|
commands =
|
|
{[base]install_and_test} {[base]all_packages}
|
|
python tests/lock_test.py
|
|
setenv =
|
|
PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto}
|
|
PYTHONHASHSEED = 0
|
|
|
|
[testenv:py27-oldest]
|
|
commands =
|
|
{[testenv]commands}
|
|
setenv =
|
|
{[testenv]setenv}
|
|
CERTBOT_OLDEST=1
|
|
|
|
[testenv:py27-acme-oldest]
|
|
commands =
|
|
{[base]install_and_test} acme[dev]
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-apache-oldest]
|
|
commands =
|
|
{[base]install_and_test} certbot-apache
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-certbot-oldest]
|
|
commands =
|
|
{[base]install_and_test} .[dev]
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-dns-oldest]
|
|
commands =
|
|
{[base]install_and_test} {[base]dns_packages}
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-nginx-oldest]
|
|
commands =
|
|
{[base]install_and_test} certbot-nginx
|
|
python tests/lock_test.py
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-postfix-oldest]
|
|
commands =
|
|
{[base]install_and_test} certbot-postfix
|
|
setenv =
|
|
{[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:py27-cover]
|
|
basepython = python2.7
|
|
commands =
|
|
{[base]install_packages}
|
|
python tox.cover.py
|
|
|
|
[testenv:py37-cover]
|
|
basepython = python3.7
|
|
commands =
|
|
{[base]install_packages}
|
|
python tox.cover.py
|
|
|
|
[testenv:lint]
|
|
basepython = python2.7
|
|
# separating into multiple invocations disables cross package
|
|
# duplicate code checking; if one of the commands fails, others will
|
|
# continue, but tox return code will reflect previous error
|
|
commands =
|
|
{[base]install_packages}
|
|
python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
|
|
|
|
[testenv:mypy]
|
|
basepython = python3
|
|
commands =
|
|
{[base]install_packages}
|
|
{[base]pip_install} .[dev3]
|
|
mypy {[base]source_paths}
|
|
|
|
[testenv:apacheconftest]
|
|
#basepython = python2.7
|
|
commands =
|
|
{[base]pip_install} acme . certbot-apache certbot-compatibility-test
|
|
{toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules
|
|
passenv =
|
|
SERVER
|
|
|
|
[testenv:apacheconftest-with-pebble]
|
|
commands =
|
|
{toxinidir}/tests/pebble-fetch.sh
|
|
{[testenv:apacheconftest]commands}
|
|
passenv =
|
|
HOME
|
|
GOPATH
|
|
PEBBLEPATH
|
|
setenv =
|
|
SERVER=https://localhost:14000/dir
|
|
|
|
[testenv:nginxroundtrip]
|
|
commands =
|
|
{[base]pip_install} acme . certbot-apache certbot-nginx
|
|
python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
|
|
|
|
# This is a duplication of the command line in testenv:le_auto to
|
|
# allow users to run the modification check by running `tox`
|
|
[testenv:modification]
|
|
commands =
|
|
python {toxinidir}/tests/modification-check.py
|
|
|
|
[testenv:apache_compat]
|
|
commands =
|
|
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
|
|
docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
|
|
docker run --rm -it apache-compat -c apache.tar.gz -vvvv
|
|
whitelist_externals =
|
|
docker
|
|
passenv =
|
|
DOCKER_*
|
|
|
|
[testenv:nginx_compat]
|
|
commands =
|
|
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
|
|
docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
|
|
docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie
|
|
whitelist_externals =
|
|
docker
|
|
passenv =
|
|
DOCKER_*
|
|
|
|
[testenv:le_auto_trusty]
|
|
# At the moment, this tests under Python 2.7 only, as only that version is
|
|
# readily available on the Trusty Docker image.
|
|
commands =
|
|
python {toxinidir}/tests/modification-check.py
|
|
docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source
|
|
docker run --rm -t -i lea
|
|
whitelist_externals =
|
|
docker
|
|
passenv =
|
|
DOCKER_*
|
|
TRAVIS_BRANCH
|
|
|
|
[testenv:le_auto_xenial]
|
|
# At the moment, this tests under Python 2.7 only.
|
|
commands =
|
|
docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source
|
|
docker run --rm -t -i lea
|
|
whitelist_externals =
|
|
docker
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:le_auto_jessie]
|
|
# At the moment, this tests under Python 2.7 only, as only that version is
|
|
# readily available on the Wheezy Docker image.
|
|
commands =
|
|
docker build -f letsencrypt-auto-source/Dockerfile.jessie -t lea letsencrypt-auto-source
|
|
docker run --rm -t -i lea
|
|
whitelist_externals =
|
|
docker
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:le_auto_centos6]
|
|
# At the moment, this tests under Python 2.6 only, as only that version is
|
|
# readily available on the CentOS 6 Docker image.
|
|
commands =
|
|
docker build -f letsencrypt-auto-source/Dockerfile.centos6 -t lea letsencrypt-auto-source
|
|
docker run --rm -t -i lea
|
|
whitelist_externals =
|
|
docker
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:docker_dev]
|
|
# tests the Dockerfile-dev file to ensure development with it works
|
|
# as expected
|
|
commands =
|
|
docker-compose run --rm --service-ports development bash -c 'tox -e lint'
|
|
whitelist_externals =
|
|
docker-compose
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:integration]
|
|
commands =
|
|
{[base]pip_install} acme . certbot-nginx certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests \
|
|
--acme-server={env:ACME_SERVER:pebble} \
|
|
--cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
|
|
--cov-config=certbot-ci/certbot_integration_tests/.coveragerc
|
|
coverage report --include 'certbot/*' --show-missing --fail-under=67
|
|
coverage report --include 'certbot-nginx/*' --show-missing --fail-under=74
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:integration-certbot-oldest]
|
|
commands =
|
|
{[base]pip_install} .
|
|
{[base]pip_install} certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/certbot_tests \
|
|
--acme-server={env:ACME_SERVER:pebble}
|
|
passenv = DOCKER_*
|
|
setenv = {[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:integration-nginx-oldest]
|
|
commands =
|
|
{[base]pip_install} certbot-nginx
|
|
{[base]pip_install} certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/nginx_tests \
|
|
--acme-server={env:ACME_SERVER:pebble}
|
|
passenv = DOCKER_*
|
|
setenv = {[testenv:py27-oldest]setenv}
|
|
|
|
[testenv:travis-test-farm-tests-base]
|
|
changedir = tests/letstest
|
|
commands =
|
|
./travis-setup.sh
|
|
deps = -rtests/letstest/requirements.txt
|
|
passenv =
|
|
AWS_*
|
|
TRAVIS_*
|
|
encrypted_*
|
|
setenv = AWS_DEFAULT_REGION=us-east-1
|
|
|
|
[testenv:travis-test-farm-apache2]
|
|
changedir = {[testenv:travis-test-farm-tests-base]changedir}
|
|
commands =
|
|
{[testenv:travis-test-farm-tests-base]commands}
|
|
python multitester.py apache2_targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_apache2.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH} --fast
|
|
deps = {[testenv:travis-test-farm-tests-base]deps}
|
|
passenv = {[testenv:travis-test-farm-tests-base]passenv}
|
|
setenv = {[testenv:travis-test-farm-tests-base]setenv}
|
|
|
|
[testenv:travis-test-farm-leauto-upgrades]
|
|
changedir = {[testenv:travis-test-farm-tests-base]changedir}
|
|
commands =
|
|
{[testenv:travis-test-farm-tests-base]commands}
|
|
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH} --fast
|
|
deps = {[testenv:travis-test-farm-tests-base]deps}
|
|
passenv = {[testenv:travis-test-farm-tests-base]passenv}
|
|
setenv = {[testenv:travis-test-farm-tests-base]setenv}
|
|
|
|
[testenv:travis-test-farm-certonly-standalone]
|
|
changedir = {[testenv:travis-test-farm-tests-base]changedir}
|
|
commands =
|
|
{[testenv:travis-test-farm-tests-base]commands}
|
|
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH} --fast
|
|
deps = {[testenv:travis-test-farm-tests-base]deps}
|
|
passenv = {[testenv:travis-test-farm-tests-base]passenv}
|
|
setenv = {[testenv:travis-test-farm-tests-base]setenv}
|
|
|
|
[testenv:travis-test-farm-sdists]
|
|
changedir = {[testenv:travis-test-farm-tests-base]changedir}
|
|
commands =
|
|
{[testenv:travis-test-farm-tests-base]commands}
|
|
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_sdists.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH} --fast
|
|
deps = {[testenv:travis-test-farm-tests-base]deps}
|
|
passenv = {[testenv:travis-test-farm-tests-base]passenv}
|
|
setenv = {[testenv:travis-test-farm-tests-base]setenv}
|