more fixes

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -17,6 +17,8 @@ jobs:
         linux-py310:
           PYTHON_VERSION: 3.10
           TOXENV: py310
+        linux-isolated:
+          TOXENV: 'isolated-{acme,certbot,apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}'
         linux-boulder-v2-integration-certbot-oldest:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot-oldest

tox.ini

@@ -5,6 +5,7 @@ envlist = {cover,lint}-{win,posix},mypy
 skipsdist = true
 
 [base]
+pytest = python -m pytest {posargs}
 # Paths are listed on one line because tox seems to have inconsistent
 # behavior with substitutions that contain line continuations, see
 # https://github.com/tox-dev/tox/issues/2069 for more info.
@@ -46,9 +47,14 @@ commands =
     echo "Unrecognized environment name {envname}"
     false
 
-[testenv:py{,-win,-posix}]
+[testenv:py-win]
 commands =
-    python -m pytest {posargs} acme certbot certbot-apache certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx
+    {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx
+
+[testenv:py{,-posix}]
+# We want to test everything we do on Windows plus the Apache plugin.
+commands =
+    {[testenv:py-win]commands} certbot-apache
 
 [testenv:py3{,7,8,9,10,11}]
 commands = {[testenv:py]commands}
@@ -82,21 +88,50 @@ commands = mypy {[base]source_paths}
 [testenv:isolated-acme]
 description = Tests acme without any Certbot components installed
 deps = -e acme[test]
-commands = python -m pytest acme
+commands = {[base]pytest} acme
 
 [testenv:isolated-certbot]
 description = Tests Certbot without any additional plugins installed
 deps =
     {[testenv:isolated-acme]deps}
     -e certbot[test]
-commands = python -m pytest certbot
+commands = {[base]pytest} certbot
 
-[testenv:isolated-plugin]
-description = Tests the given plugin without installing other plugins
+[testenv:isolated-{apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}]
+description = Tests the plugin without installing any other plugins
 deps =
     {[testenv:isolated-certbot]deps}
-    -e {posargs}
-commands = python -m pytest {posargs}
+    apache: -e certbot-apache[dev]
+    cloudflare: -e certbot-dns-cloudflare
+    digitalocean: -e certbot-dns-digitalocean
+    dnsimple: -e certbot-dns-dnsimple
+    dnsmadeeasy: -e certbot-dns-dnsmadeeasy
+    gehirn: -e certbot-dns-gehirn
+    google: -e certbot-dns-google
+    linode: -e certbot-dns-linode
+    luadns: -e certbot-dns-luadns
+    nsone: -e certbot-dns-nsone
+    ovh: -e certbot-dns-ovh
+    rfc2136: -e certbot-dns-rfc2136
+    route53: -e certbot-dns-route53
+    sakuracloud: -e certbot-dns-sakuracloud
+    nginx: -e certbot-nginx
+commands =
+    apache: {[base]pytest} certbot-apache
+    cloudflare: {[base]pytest} certbot-dns-cloudflare
+    digitalocean: {[base]pytest} certbot-dns-digitalocean
+    dnsimple: {[base]pytest} certbot-dns-dnsimple
+    dnsmadeeasy: {[base]pytest} certbot-dns-dnsmadeeasy
+    gehirn: {[base]pytest} certbot-dns-gehirn
+    google: {[base]pytest} certbot-dns-google
+    linode: {[base]pytest} certbot-dns-linode
+    luadns: {[base]pytest} certbot-dns-luadns
+    nsone: {[base]pytest} certbot-dns-nsone
+    ovh: {[base]pytest} certbot-dns-ovh
+    rfc2136: {[base]pytest} certbot-dns-rfc2136
+    route53: {[base]pytest} certbot-dns-route53
+    sakuracloud: {[base]pytest} certbot-dns-sakuracloud
+    nginx: {[base]pytest} certbot-nginx
 
 [testenv:apacheconftest]
 deps =
@@ -173,7 +208,7 @@ deps =
     -e certbot-nginx
     -e certbot-ci
 commands =
-    pytest certbot-ci/certbot_integration_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
@@ -187,7 +222,7 @@ deps =
     -e certbot
     -e certbot-ci
 commands =
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
@@ -200,7 +235,7 @@ deps =
     -e certbot-dns-rfc2136
     -e certbot-ci
 commands =
-    pytest certbot-ci/certbot_integration_tests/rfc2136_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/rfc2136_tests \
         --acme-server=pebble --dns-server=bind \
         --numprocesses=1 \
         --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \
@@ -213,7 +248,7 @@ description = Run integration tests with Certbot outside of tox's virtual enviro
 deps =
     -e certbot-ci
 commands =
-    pytest certbot-ci/certbot_integration_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 
@@ -225,7 +260,7 @@ deps =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
@@ -239,7 +274,7 @@ deps =
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    pytest certbot-ci/certbot_integration_tests/nginx_tests \
+    {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}