mirror of
https://github.com/certbot/certbot.git
synced 2026-05-28 04:34:11 -04:00
e2b7b62b98
The reason I want to do this is many of the targets of `test_sdists.sh` use Python 3.6 which [has reached its EOL](https://www.python.org/dev/peps/pep-0494/#lifespan). We could instead just stop running the test on these systems or install a newer version of Python 3 outside of OS packaging, but instead I decided to look into why we have these tests to begin with.
I introduced these tests many years ago in https://github.com/certbot/certbot/pull/4089 as a fix for https://github.com/certbot/certbot/issues/4044. Essentially the problem was the way packagers ran tests and the way we ran tests were slightly different. This difference could cause test failures when distros tried to run tests on our packages.
Since I did this, [we've switched to telling packagers to run tests using `pytest` like we do](5e76669c50/certbot/docs/packaging.rst (notes-for-package-maintainers)) and we've greatly reduced our reliance on OS packaging through things like `snap`.
Because of this, I think we should stop running this test, reducing our reliance on the heavy "test farm tests", and simplifying our CI pipeline. I think future problems here is quite unlikely and even if we have them, it should only affect tests on our non-primary distribution mechanisms which I think is a very minor concern.
When reviewing this PR, it's probably worth noting that I just replaced `targets.yaml` with the contents of `apache2_targets.yaml` since the Apache 2 tests are the only runs we're running with this change.
268 lines
9.9 KiB
INI
268 lines
9.9 KiB
INI
# Tox (https://tox.readthedocs.io/) is a tool for running tests in
|
|
# multiple virtualenvs. To use it, "pip install tox" and then run
|
|
# "tox" from this directory.
|
|
|
|
[tox]
|
|
skipsdist = true
|
|
envlist = {py3-cover,lint,mypy}-{win,posix}
|
|
|
|
[base]
|
|
# pip installs the requested packages in editable mode
|
|
pip_install = python {toxinidir}/tools/pip_install_editable.py
|
|
# pip installs the requested packages in editable mode and runs unit tests on
|
|
# them. Each package is installed and tested in the order they are provided
|
|
# before the script moves on to the next package. All dependencies are pinned
|
|
# to a specific version for increased stability for developers.
|
|
install_and_test = python {toxinidir}/tools/install_and_test.py
|
|
# Packages are listed on one line because tox seems to have inconsistent
|
|
# behavior with substitutions that contain line continuations, see
|
|
# https://github.com/tox-dev/tox/issues/2069 for more info.
|
|
dns_packages = certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud
|
|
win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx
|
|
all_packages = {[base]win_all_packages} certbot-apache
|
|
source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-cloudxns/certbot_dns_cloudxns certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx tests/lock_test.py
|
|
|
|
[testenv]
|
|
passenv =
|
|
CERTBOT_NO_PIN
|
|
platform =
|
|
win: win32
|
|
posix: ^(?!.*win32).*$
|
|
commands_pre = python {toxinidir}/tools/pipstrap.py
|
|
commands =
|
|
!cover-win: {[base]install_and_test} {[base]win_all_packages}
|
|
!cover-!win: {[base]install_and_test} {[base]all_packages}
|
|
!cover: python tests/lock_test.py
|
|
|
|
cover-win: {[base]pip_install} {[base]win_all_packages}
|
|
cover-!win: {[base]pip_install} {[base]all_packages} certbot-apache[dev]
|
|
cover: python tox.cover.py
|
|
# We always recreate the virtual environment to avoid problems like
|
|
# https://github.com/certbot/certbot/issues/7745.
|
|
recreate = true
|
|
setenv =
|
|
PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto}
|
|
PYTHONHASHSEED = 0
|
|
|
|
[testenv:oldest]
|
|
# Setting basepython allows the tests to fail fast if that version of Python
|
|
# isn't available instead of potentially trying to use a newer version of
|
|
# Python which is unlikely to work.
|
|
#
|
|
# This version should be kept in sync with the one declared in
|
|
# tools/pinning/oldest/pyproject.toml.
|
|
basepython = python3.6
|
|
commands =
|
|
{[testenv]commands}
|
|
setenv =
|
|
{[testenv]setenv}
|
|
CERTBOT_OLDEST=1
|
|
|
|
[testenv:acme-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]install_and_test} acme[test]
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:apache-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme[test] certbot[test] certbot-apache
|
|
pytest certbot-apache
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:apache-v2-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme[test] certbot[test] certbot-apache[dev]
|
|
pytest certbot-apache
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:certbot-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme[test] certbot[test]
|
|
pytest certbot
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:dns-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme[test] certbot[test] {[base]dns_packages}
|
|
pytest {[base]dns_packages}
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:nginx-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme[test] certbot[test] certbot-nginx
|
|
pytest certbot-nginx
|
|
python tests/lock_test.py
|
|
setenv =
|
|
{[testenv:oldest]setenv}
|
|
|
|
[testenv:external-mock]
|
|
commands =
|
|
python {toxinidir}/tools/pip_install.py mock
|
|
{[base]install_and_test} {[base]all_packages}
|
|
|
|
[testenv:lint{,-win,-posix}]
|
|
basepython = python3
|
|
# separating into multiple invocations disables cross package
|
|
# duplicate code checking; if one of the commands fails, others will
|
|
# continue, but tox return code will reflect previous error
|
|
commands =
|
|
win: {[base]pip_install} {[base]win_all_packages}
|
|
!win: {[base]pip_install} {[base]all_packages}
|
|
python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
|
|
|
|
[testenv:mypy{,-win,-posix}]
|
|
basepython = python3
|
|
commands =
|
|
win: {[base]pip_install} {[base]win_all_packages}
|
|
!win: {[base]pip_install} {[base]all_packages} certbot-ci
|
|
mypy {[base]source_paths}
|
|
|
|
[testenv:apacheconftest]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-apache
|
|
{toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test --debian-modules
|
|
passenv =
|
|
SERVER
|
|
|
|
[testenv:apacheconftest-external-with-pebble]
|
|
# Run apacheconftest with pebble and Certbot outside of tox's virtual
|
|
# environment.
|
|
commands =
|
|
{[base]pip_install} certbot-ci
|
|
{toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules
|
|
|
|
[testenv:apacheconftest-with-pebble]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-apache
|
|
{[testenv:apacheconftest-external-with-pebble]commands}
|
|
|
|
[testenv:nginxroundtrip]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-apache certbot-nginx
|
|
python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
|
|
|
|
[testenv:modification]
|
|
commands =
|
|
python {toxinidir}/tests/modification-check.py
|
|
|
|
[testenv:apache_compat]
|
|
commands =
|
|
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
|
|
docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
|
|
docker run --rm -t apache-compat -c apache.tar.gz -vvvv
|
|
whitelist_externals =
|
|
docker
|
|
passenv =
|
|
DOCKER_*
|
|
|
|
[testenv:nginx_compat]
|
|
commands =
|
|
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
|
|
docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
|
|
docker run --rm -t nginx-compat -c nginx.tar.gz -vv -aie
|
|
whitelist_externals =
|
|
docker
|
|
passenv =
|
|
DOCKER_*
|
|
|
|
[testenv:docker_dev]
|
|
# tests the Dockerfile-dev file to ensure development with it works
|
|
# as expected
|
|
commands =
|
|
docker-compose run --rm --service-ports development bash -c 'tox -e lint'
|
|
whitelist_externals =
|
|
docker-compose
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:integration]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-nginx certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests \
|
|
--acme-server={env:ACME_SERVER:pebble} \
|
|
--cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
|
|
--cov-config=certbot-ci/certbot_integration_tests/.coveragerc
|
|
coverage report --include 'certbot/*' --show-missing --fail-under=65
|
|
coverage report --include 'certbot-nginx/*' --show-missing --fail-under=74
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:integration-certbot]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/certbot_tests \
|
|
--acme-server={env:ACME_SERVER:pebble} \
|
|
--cov=acme --cov=certbot --cov-report= \
|
|
--cov-config=certbot-ci/certbot_integration_tests/.coveragerc
|
|
coverage report --include 'certbot/*' --show-missing --fail-under=62
|
|
|
|
[testenv:integration-dns-rfc2136]
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/rfc2136_tests \
|
|
--acme-server=pebble --dns-server=bind \
|
|
--numprocesses=1 \
|
|
--cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \
|
|
--cov-config=certbot-ci/certbot_integration_tests/.coveragerc
|
|
coverage report --include 'certbot/*' --show-missing --fail-under=45
|
|
coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=86
|
|
|
|
[testenv:integration-external]
|
|
# Run integration tests with Certbot outside of tox's virtual environment.
|
|
commands =
|
|
{[base]pip_install} certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests \
|
|
--acme-server={env:ACME_SERVER:pebble}
|
|
passenv = DOCKER_*
|
|
|
|
[testenv:integration-certbot-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/certbot_tests \
|
|
--acme-server={env:ACME_SERVER:pebble}
|
|
passenv = DOCKER_*
|
|
setenv = {[testenv:oldest]setenv}
|
|
|
|
[testenv:integration-nginx-oldest]
|
|
basepython =
|
|
{[testenv:oldest]basepython}
|
|
commands =
|
|
{[base]pip_install} acme certbot certbot-nginx certbot-ci
|
|
pytest certbot-ci/certbot_integration_tests/nginx_tests \
|
|
--acme-server={env:ACME_SERVER:pebble}
|
|
passenv = DOCKER_*
|
|
setenv = {[testenv:oldest]setenv}
|
|
|
|
[testenv:test-farm-tests-base]
|
|
changedir = letstest
|
|
# The package to install is in the current working directory because of the
|
|
# value of changedir.
|
|
commands = {[base]pip_install} .
|
|
passenv =
|
|
AWS_*
|
|
setenv = AWS_DEFAULT_REGION=us-east-1
|
|
|
|
[testenv:test-farm-apache2]
|
|
changedir = {[testenv:test-farm-tests-base]changedir}
|
|
commands =
|
|
{[testenv:test-farm-tests-base]commands}
|
|
{toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
|
|
passenv = {[testenv:test-farm-tests-base]passenv}
|
|
setenv = {[testenv:test-farm-tests-base]setenv}
|