diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 1e6d7a352..53bc0b5dd 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -23,14 +23,10 @@ jobs: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.9 TOXENV: integration-certbot - linux-oldest-tests-1: + linux-oldest: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.7 - TOXENV: '{acme,apache,apache-v2,certbot}-oldest' - linux-oldest-tests-2: - IMAGE_NAME: ubuntu-22.04 - PYTHON_VERSION: 3.7 - TOXENV: '{dns,nginx}-oldest' + TOXENV: oldest linux-py37: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.7 @@ -43,7 +39,7 @@ jobs: TOXENV: lint-posix linux-mypy: IMAGE_NAME: ubuntu-22.04 - TOXENV: mypy-posix + TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.8 diff --git a/tox.ini b/tox.ini index de98c3e25..ce279088e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,42 +1,60 @@ -# Tox (https://tox.readthedocs.io/) is a tool for running tests in -# multiple virtualenvs. To use it, "pip install tox" and then run -# "tox" from this directory. - [tox] -skipsdist = true # mypy doesn't current pass for us on Windows. Fixing that is being tracked by # https://github.com/certbot/certbot/issues/7803. -envlist = {cover,lint}-{win,posix},mypy-posix +envlist = {cover,lint}-{win,posix},mypy +skipsdist = true [base] -# pip installs the requested packages in editable mode -pip_install = python {toxinidir}/tools/pip_install_editable.py -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. All dependencies are pinned -# to a specific version for increased stability for developers. -install_and_test = python {toxinidir}/tools/install_and_test.py -# Packages are listed on one line because tox seems to have inconsistent +# Paths are listed on one line because tox seems to have inconsistent # behavior with substitutions that contain line continuations, see # https://github.com/tox-dev/tox/issues/2069 for more info. -dns_packages = certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud -win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx -all_packages = {[base]win_all_packages} certbot-apache source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx [testenv] platform = win: win32 posix: ^(?!.*win32).*$ -commands = - win: {[base]install_and_test} {[base]win_all_packages} - !win: {[base]install_and_test} {[base]all_packages} -# We always recreate the virtual environment to avoid problems like -# https://github.com/certbot/certbot/issues/7745. -recreate = true setenv = PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto} PYTHONHASHSEED = 0 +# The default install command is python -I -m pip install {opts} {packages} +install_command = python -I {toxinidir}/tools/pip_install.py {opts} {packages} +deps = + -e acme[test] + -e certbot[test] + !win: -e certbot-apache[dev] + -e certbot-dns-cloudflare + -e certbot-dns-digitalocean + -e certbot-dns-dnsimple + -e certbot-dns-dnsmadeeasy + -e certbot-dns-gehirn + -e certbot-dns-google + -e certbot-dns-linode + -e certbot-dns-luadns + -e certbot-dns-nsone + -e certbot-dns-ovh + -e certbot-dns-rfc2136 + -e certbot-dns-route53 + -e certbot-dns-sakuracloud + -e certbot-nginx +whitelist_externals = + echo + false +# This and the next few testenvs are a workaround for +# https://github.com/tox-dev/tox/issues/2858. +commands = + echo "Unrecognized environment name {envname}" + false + +[testenv:py] +commands = + python -m pytest {posargs} acme certbot certbot-apache certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx + +[testenv:py3{,7,8,9,10,11}] +commands = {[testenv:py]commands} + +[testenv:py3.{7,8,9,10,11}] +commands = {[testenv:py]commands} [testenv:oldest] # Setting basepython allows the tests to fail fast if that version of Python @@ -46,117 +64,58 @@ setenv = # This version should be kept in sync with the one declared in # tools/pinning/oldest/pyproject.toml. basepython = python3.7 -commands = - {[testenv]commands} -setenv = - {[testenv]setenv} - CERTBOT_OLDEST=1 - -[testenv:acme-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]install_and_test} acme[test] -setenv = - {[testenv:oldest]setenv} - -[testenv:apache-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-apache - pytest certbot-apache -setenv = - {[testenv:oldest]setenv} - -[testenv:apache-v2-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-apache[dev] - pytest certbot-apache -setenv = - {[testenv:oldest]setenv} - -[testenv:certbot-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] - pytest certbot -setenv = - {[testenv:oldest]setenv} - -[testenv:dns-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] {[base]dns_packages} - pytest {[base]dns_packages} -setenv = - {[testenv:oldest]setenv} - -[testenv:nginx-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-nginx - pytest certbot-nginx -setenv = - {[testenv:oldest]setenv} +setenv = CERTBOT_OLDEST=1 +commands = {[testenv:py]commands} [testenv:cover{,-win,-posix}] -commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} certbot-apache[dev] - python tox.cover.py +commands = python tox.cover.py [testenv:lint{,-win,-posix}] -basepython = python3 -# separating into multiple invocations disables cross package -# duplicate code checking; if one of the commands fails, others will -# continue, but tox return code will reflect previous error -commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} - python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} +commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} -[testenv:mypy{,-win,-posix}] -basepython = python3 -commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} certbot-ci - mypy {[base]source_paths} +[testenv:mypy] +deps = + {[testenv]deps} + -e certbot-ci +commands = mypy {[base]source_paths} [testenv:apacheconftest] +deps = + -e acme + -e certbot + -e certbot-apache commands = - {[base]pip_install} acme certbot certbot-apache {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test --debian-modules passenv = SERVER [testenv:apacheconftest-external-with-pebble] -# Run apacheconftest with pebble and Certbot outside of tox's virtual -# environment. +deps = + -e certbot-ci commands = - {[base]pip_install} certbot-ci {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules +description = Run apacheconftest with pebble and Certbot outside of tox's virtual environment. [testenv:apacheconftest-with-pebble] -commands = - {[base]pip_install} acme certbot certbot-apache - {[testenv:apacheconftest-external-with-pebble]commands} +deps = {[testenv:apacheconftest]deps} +commands = {[testenv:apacheconftest-external-with-pebble]commands} [testenv:nginxroundtrip] +deps = + -e acme + -e certbot + -e certbot-apache + -e certbot-nginx commands = - {[base]pip_install} acme certbot certbot-apache certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata [testenv:modification] +deps = commands = python {toxinidir}/tests/modification-check.py [testenv:apache_compat] +deps = commands = docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile . docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache . @@ -167,6 +126,7 @@ passenv = DOCKER_* [testenv:nginx_compat] +deps = commands = docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile . docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx . @@ -177,8 +137,8 @@ passenv = DOCKER_* [testenv:docker_dev] -# tests the Dockerfile-dev file to ensure development with it works -# as expected +description = tests the Dockerfile-dev file to ensure development with it works as expected +deps = commands = docker-compose run --rm --service-ports development bash -c 'tox -e lint' whitelist_externals = @@ -186,8 +146,12 @@ whitelist_externals = passenv = DOCKER_* [testenv:integration] +deps = + -e acme + -e certbot + -e certbot-nginx + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-nginx certbot-ci pytest certbot-ci/certbot_integration_tests \ --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ @@ -197,8 +161,11 @@ commands = passenv = DOCKER_* [testenv:integration-certbot] +deps = + -e acme + -e certbot + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-ci pytest certbot-ci/certbot_integration_tests/certbot_tests \ --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov-report= \ @@ -206,8 +173,12 @@ commands = coverage report --include 'certbot/*' --show-missing --fail-under=62 [testenv:integration-dns-rfc2136] +deps = + -e acme + -e certbot + -e certbot-dns-rfc2136 + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci pytest certbot-ci/certbot_integration_tests/rfc2136_tests \ --acme-server=pebble --dns-server=bind \ --numprocesses=1 \ @@ -217,46 +188,47 @@ commands = coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=86 [testenv:integration-external] -# Run integration tests with Certbot outside of tox's virtual environment. +description = Run integration tests with Certbot outside of tox's virtual environment. +deps = + -e certbot-ci commands = - {[base]pip_install} certbot-ci pytest certbot-ci/certbot_integration_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* [testenv:integration-certbot-oldest] +deps = + -e acme + -e certbot + -e certbot-ci basepython = {[testenv:oldest]basepython} commands = - {[base]pip_install} acme certbot certbot-ci pytest certbot-ci/certbot_integration_tests/certbot_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* setenv = {[testenv:oldest]setenv} [testenv:integration-nginx-oldest] +deps = + -e acme + -e certbot + -e certbot-nginx + -e certbot-ci basepython = {[testenv:oldest]basepython} commands = - {[base]pip_install} acme certbot certbot-nginx certbot-ci pytest certbot-ci/certbot_integration_tests/nginx_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* setenv = {[testenv:oldest]setenv} -[testenv:test-farm-tests-base] -changedir = letstest -# The package to install is in the current working directory because of the -# value of changedir. -commands = {[base]pip_install} . +[testenv:test-farm-apache2] passenv = AWS_* setenv = AWS_DEFAULT_REGION=us-east-1 - -[testenv:test-farm-apache2] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = - {[testenv:test-farm-tests-base]commands} - {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} +changedir = letstest +# The package to install is in the current working directory because of the +# value of changedir. +deps = -e . +commands = {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}