mirror of
https://github.com/certbot/certbot.git
synced 2026-06-07 15:52:08 -04:00
fa67b7ba0f
After getting a +1 from everyone on the team, this PR removes the use of `codecov` from the Certbot repo because we keep having problems with it.
Two noteworthy things about this PR are:
1. I left the text at 4ea98d830b/.azure-pipelines/INSTALL.md (add-a-secret-variable-to-a-pipeline-like-codecov_token) because I think it's useful to document how to set up a secret variable in general.
2. I'm not sure what the text "Option -e makes sure we fail fast and don't submit to codecov." in `tox.cover.py` refers to but it seems incorrect since `-e` isn't accepted or used by the script so I just deleted the line.
As part of this, I said I'd open an issue to track setting up coveralls (which seems to be the only real alternative to codecov) which is at https://github.com/certbot/certbot/issues/7810.
With my change, failure output looks something like:
```
$ tox -e py27-cover
...
Name Stmts Miss Cover Missing
------------------------------------------------------------------------------------------
certbot/certbot/__init__.py 1 0 100%
certbot/certbot/_internal/__init__.py 0 0 100%
certbot/certbot/_internal/account.py 191 4 98% 62-63, 206, 337
...
certbot/tests/storage_test.py 530 0 100%
certbot/tests/util_test.py 374 29 92% 211-213, 480-484, 489-499, 504-511, 545-547, 552-554
------------------------------------------------------------------------------------------
TOTAL 14451 647 96%
Command '['/path/to/certbot/dir/.tox/py27-cover/bin/python', '-m', 'coverage', 'report', '--fail-under', '100', '--include', 'certbot/*', '--show-missing']' returned non-zero exit status 2
Test coverage on certbot did not meet threshold of 100%.
ERROR: InvocationError for command /Users/bmw/Development/certbot/certbot/.tox/py27-cover/bin/python tox.cover.py (exited with code 1)
_________________________________________________________________________________________________________________________________________________________ summary _________________________________________________________________________________________________________________________________________________________
ERROR: py27-cover: commands failed
```
I printed the exception just so we're not throwing away information.
I think it's also possible we fail for a reason other than the threshold not meeting the percentage, but I've personally never seen this, `coverage report` output is not being captured so hopefully that would inform devs if something else is going on, and saying something like "Test coverage probably did not..." seems like overkill to me personally.
* remove codecov
* remove unused variable group
* remove codecov.yml
* Improve tox.cover.py failure output.
88 lines
3.1 KiB
Python
Executable file
88 lines
3.1 KiB
Python
Executable file
#!/usr/bin/env python
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
DEFAULT_PACKAGES = [
|
|
'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns',
|
|
'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy',
|
|
'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns',
|
|
'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53',
|
|
'certbot_dns_sakuracloud', 'certbot_nginx']
|
|
|
|
COVER_THRESHOLDS = {
|
|
'certbot': {'linux': 96, 'windows': 96},
|
|
'acme': {'linux': 100, 'windows': 99},
|
|
'certbot_apache': {'linux': 100, 'windows': 100},
|
|
'certbot_dns_cloudflare': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_cloudxns': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_digitalocean': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_dnsimple': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99},
|
|
'certbot_dns_gehirn': {'linux': 97, 'windows': 97},
|
|
'certbot_dns_google': {'linux': 99, 'windows': 99},
|
|
'certbot_dns_linode': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_luadns': {'linux': 98, 'windows': 98},
|
|
'certbot_dns_nsone': {'linux': 99, 'windows': 99},
|
|
'certbot_dns_ovh': {'linux': 97, 'windows': 97},
|
|
'certbot_dns_rfc2136': {'linux': 99, 'windows': 99},
|
|
'certbot_dns_route53': {'linux': 92, 'windows': 92},
|
|
'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97},
|
|
'certbot_nginx': {'linux': 97, 'windows': 97},
|
|
}
|
|
|
|
SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache']
|
|
|
|
|
|
def cover(package):
|
|
threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux']
|
|
|
|
pkg_dir = package.replace('_', '-')
|
|
|
|
if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS:
|
|
print((
|
|
'Info: currently {0} is not supported on Windows and will not be tested/covered.'
|
|
.format(pkg_dir)))
|
|
return
|
|
|
|
subprocess.check_call([sys.executable, '-m', 'pytest',
|
|
'--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir])
|
|
try:
|
|
subprocess.check_call([
|
|
sys.executable, '-m', 'coverage', 'report', '--fail-under',
|
|
str(threshold), '--include', '{0}/*'.format(pkg_dir),
|
|
'--show-missing'])
|
|
except subprocess.CalledProcessError as err:
|
|
print(err)
|
|
print('Test coverage on', pkg_dir,
|
|
'did not meet threshold of {0}%.'.format(threshold))
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
description = """
|
|
This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in
|
|
order to generate separate stats for each package. It should be removed once
|
|
those packages are moved to a separate repo."""
|
|
parser = argparse.ArgumentParser(description=description)
|
|
parser.add_argument('--packages', nargs='+')
|
|
|
|
args = parser.parse_args()
|
|
|
|
packages = args.packages or DEFAULT_PACKAGES
|
|
|
|
# --cov-append is on, make sure stats are correct
|
|
try:
|
|
os.remove('.coverage')
|
|
except OSError:
|
|
pass
|
|
|
|
for package in packages:
|
|
cover(package)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|