From 1b6005cc61f8b977af1bc5513994b4815280dd74 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 14 Dec 2017 18:15:42 -0800 Subject: [PATCH 1/8] Pin josepy in letsencrypt-auto (#5321) * pin josepy in le-auto * Put pinned versions in sorted order --- letsencrypt-auto-source/letsencrypt-auto | 11 +++++++---- .../pieces/dependency-requirements.txt | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 8d2e8a6b6..93e3e7b83 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -983,9 +983,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -1062,10 +1069,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 # Contains the requirements for the letsencrypt package. # diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index dec7ae7d0..0e2cec984 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -105,9 +105,16 @@ idna==2.5 \ ipaddress==1.0.16 \ --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +josepy==1.0.1 \ + --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \ + --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc linecache2==1.0.0 \ --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +# Using an older version of mock here prevents regressions of #5276. +mock==1.3.0 \ + --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ + --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 ordereddict==1.1 \ --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f packaging==16.8 \ @@ -184,7 +191,3 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -# Using an older version of mock here prevents regressions of #5276. -mock==1.3.0 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 From a1aea021e7a587ea9396b2ebbfcfaec10411ab86 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 18 Dec 2017 12:31:36 -0800 Subject: [PATCH 2/8] Pin dependencies in oldest tests (#5316) * Add tools/merge_requirements.py * Revert "Fix oldest tests by pinning Google DNS deps (#5000)" This reverts commit f68fba2be2fc342dd72deaaf048ab79e5a8fc2be. * Add tools/oldest_constraints.txt * Remove oldest constraints from tox.ini * Rename dev constraints file * Update tools/pip_install.sh * Update install_and_test.sh * Fix pip_install.sh * Don't cat when you can cp * Add ng-httpsclient to dev constraints for oldest tests * Bump tested setuptools version * Update dev_constraints comment * Better document oldest dependencies * test against oldest versions we say we require * Update dev constraints * Properly handle empty lines * Update constraints gen in pip_install * Remove duplicated zope.component * Reduce pyasn1-modules dependency * Remove blank line * pin back google-api-python-client * pin back uritemplate * pin josepy for oldest tests * Undo changes to install_and_test.sh * Update install_and_test.sh description * use split instead of partition --- ...ip_constraints.txt => dev_constraints.txt} | 25 ++++---- tools/install_and_test.sh | 5 +- tools/merge_requirements.py | 61 +++++++++++++++++++ tools/oldest_constraints.txt | 51 ++++++++++++++++ tools/pip_install.sh | 31 ++++++---- tox.ini | 36 +---------- 6 files changed, 151 insertions(+), 58 deletions(-) rename tools/{pip_constraints.txt => dev_constraints.txt} (71%) create mode 100755 tools/merge_requirements.py create mode 100644 tools/oldest_constraints.txt diff --git a/tools/pip_constraints.txt b/tools/dev_constraints.txt similarity index 71% rename from tools/pip_constraints.txt rename to tools/dev_constraints.txt index cacec37d6..afc362ff8 100644 --- a/tools/pip_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,16 +1,15 @@ # Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. We should avoid listing packages in -# both places because if both files are used as constraints for the same pip -# invocation, some constraints may be ignored due to pip's lack of dependency -# resolution. +# letsencrypt-auto's requirements file. alabaster==0.7.10 apipkg==1.4 +asn1crypto==0.22.0 astroid==1.3.5 +attrs==17.3.0 Babel==2.5.1 backports.shutil-get-terminal-size==1.0.0 boto3==1.4.7 botocore==1.7.41 -cloudflare==1.8.1 +cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 dns-lexicon==2.1.14 @@ -19,7 +18,7 @@ docutils==0.14 execnet==1.5.0 future==0.16.0 futures==3.1.1 -google-api-python-client==1.6.4 +google-api-python-client==1.5 httplib2==0.10.3 imagesize==0.7.1 ipdb==0.10.3 @@ -27,20 +26,22 @@ ipython==5.5.0 ipython-genutils==0.2.0 Jinja2==2.9.6 jmespath==0.9.3 +josepy==1.0.1 +logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -oauth2client==4.1.2 +ndg-httpsclient==0.3.2 +oauth2client==2.0.0 pathlib2==2.3.0 pexpect==4.2.1 pickleshare==0.7.4 -pkg-resources==0.0.0 pkginfo==1.4.1 pluggy==0.5.2 prompt-toolkit==1.0.15 ptyprocess==0.5.2 py==1.4.34 -pyasn1==0.3.7 -pyasn1-modules==0.1.5 +pyasn1==0.1.9 +pyasn1-modules==0.0.10 Pygments==2.2.0 pylint==1.4.2 pytest==3.2.5 @@ -48,7 +49,7 @@ pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.20.1 python-dateutil==2.6.1 -python-digitalocean==1.12 +python-digitalocean==1.11 PyYAML==3.12 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 @@ -65,6 +66,6 @@ tox==2.9.1 tqdm==4.19.4 traitlets==4.3.2 twine==1.9.1 -uritemplate==3.0.0 +uritemplate==0.6 virtualenv==15.1.0 wcwidth==0.1.7 diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index d57f0974e..25b6d548a 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -2,8 +2,9 @@ # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.sh for more information on the versions pinned +# to. if [ "$CERTBOT_NO_PIN" = 1 ]; then pip_install="pip install -q -e" diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py new file mode 100755 index 000000000..c8fb95351 --- /dev/null +++ b/tools/merge_requirements.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""Merges multiple Python requirements files into one file. + +Requirements files specified later take precedence over earlier ones. Only +simple SomeProject==1.2.3 format is currently supported. + +""" + +from __future__ import print_function + +import sys + + +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: mapping from a project to its pinned version + :rtype: dict + + """ + d = {} + with open(file_path) as f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + project, version = line.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(line)) + d[project] = version + return d + + +def print_requirements(requirements): + """Prints requirements to stdout. + + :param dict requirements: mapping from a project to its pinned version + + """ + print('\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items()))) + + +def merge_requirements_files(*files): + """Merges multiple requirements files together and prints the result. + + Requirement files specified later in the list take precedence over earlier + files. + + :param tuple files: paths to requirements files + + """ + d = {} + for f in files: + d.update(read_file(f)) + print_requirements(d) + + +if __name__ == '__main__': + merge_requirements_files(*sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt new file mode 100644 index 000000000..de2b83ad8 --- /dev/null +++ b/tools/oldest_constraints.txt @@ -0,0 +1,51 @@ +# This file contains the oldest versions of our dependencies we say we require +# in our packages or versions we need to support to maintain compatibility with +# the versions included in the various Linux distros where we are packaged. + +# CentOS/RHEL 7 EPEL constraints +cffi==1.6.0 +chardet==2.2.1 +configobj==4.7.2 +ipaddress==1.0.16 +mock==1.0.1 +ndg-httpsclient==0.3.2 +ply==3.4 +pyasn1==0.1.9 +pycparser==2.14 +pyOpenSSL==0.13.1 +pyparsing==1.5.6 +pyRFC3339==1.0 +python-augeas==0.5.0 +six==1.9.0 +# setuptools 0.9.8 is the actual version packaged, but some other dependencies +# in this file require setuptools>=1.0 and there are no relevant changes for us +# between these versions. +setuptools==1.0.0 +urllib3==1.10.2 +zope.component==4.1.0 +zope.event==4.0.3 +zope.interface==4.0.5 + +# Debian Jessie Backports constraints +PyICU==1.8 +colorama==0.3.2 +enum34==1.0.3 +html5lib==0.999 +idna==2.0 +pbr==1.8.0 +pytz==2012rc0 + +# Our setup.py constraints +cloudflare==1.5.1 +cryptography==1.2.0 +google-api-python-client==1.5 +oauth2client==2.0 +parsedatetime==1.3 +pyparsing==1.5.5 +python-digitalocean==1.11 +requests[security]==2.4.1 + +# Ubuntu Xenial constraints +ConfigArgParse==0.10.0 +funcsigs==0.4 +zope.hookable==4.0.4 diff --git a/tools/pip_install.sh b/tools/pip_install.sh index fafd58e54..d2aae4a43 100755 --- a/tools/pip_install.sh +++ b/tools/pip_install.sh @@ -1,17 +1,26 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions +#!/bin/bash -e +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt and +# tools/dev_constraints.txt is used, otherwise, a combination of certbot-auto's +# requirements file and tools/dev_constraints.txt is used. The other file +# always takes precedence over tools/dev_constraints.txt. # get the root of the Certbot repo -my_path=$("$(dirname $0)/readlink.py" $0) -repo_root=$(dirname $(dirname $my_path)) -requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" -certbot_auto_constraints=$(mktemp) -trap "rm -f $certbot_auto_constraints" EXIT -# extract pinned requirements without hashes -sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $certbot_auto_constraints -dev_constraints="$(dirname $my_path)/pip_constraints.txt" +tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) +dev_constraints="$tools_dir/dev_constraints.txt" +merge_reqs="$tools_dir/merge_requirements.py" +test_constraints=$(mktemp) +trap "rm -f $test_constraints" EXIT + +if [ "$CERTBOT_OLDEST" = 1 ]; then + cp "$tools_dir/oldest_constraints.txt" "$test_constraints" +else + repo_root=$(dirname "$tools_dir") + certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" + sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" +fi set -x # install the requested packages using the pinned requirements as constraints -pip install -q --constraint $certbot_auto_constraints --constraint $dev_constraints "$@" +pip install -q --constraint <("$merge_reqs" "$dev_constraints" "$test_constraints") "$@" diff --git a/tox.ini b/tox.ini index bb421daa5..6ebf681ed 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,8 @@ envlist = modification,py{26,33,34,35,36},cover,lint pip_install = {toxinidir}/tools/pip_install_editable.sh # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using certbot-auto's requirements file as -# constraints. +# before the script moves on to the next package. All dependencies are pinned +# to a specific version for increased stability for developers. install_and_test = {toxinidir}/tools/install_and_test.sh py26_packages = acme[dev] \ @@ -82,36 +81,7 @@ commands = {[testenv]commands} setenv = {[testenv]setenv} - CERTBOT_NO_PIN=1 -deps = - PyOpenSSL==0.13 - cffi==1.5.2 - configargparse==0.10.0 - configargparse==0.10.0 - configobj==4.7.2 - cryptography==1.2.3 - enum34==0.9.23 - google-api-python-client==1.5 - idna==2.0 - ipaddress==1.0.16 - mock==1.0.1 - ndg-httpsclient==0.3.2 - oauth2client==2.0 - parsedatetime==1.4 - pyasn1-modules==0.0.5 - pyasn1==0.1.9 - pyparsing==1.5.6 - pyrfc3339==1.0 - pytest==3.2.5 - python-augeas==0.4.1 - pytz==2012c - requests[security]==2.6.0 - setuptools==0.9.8 - six==1.9.0 - urllib3==1.10 - zope.component==4.0.2 - zope.event==4.0.1 - zope.interface==4.0.5 + CERTBOT_OLDEST=1 [testenv:py27_install] basepython = python2.7 From d6b11fea722ab71584a2bd50cb731a5f67b0e375 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 Dec 2017 16:16:45 -0800 Subject: [PATCH 3/8] More pip dependency resolution workarounds (#5339) * remove pyopenssl and six deps * remove outdated tox.ini dep requirement --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ee108c514..ce505a62e 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,9 @@ readme = read_file(os.path.join(here, 'README.rst')) changes = read_file(os.path.join(here, 'CHANGES.rst')) version = meta['version'] -# Please update tox.ini when modifying dependency version requirements -# This package relies on requests, however, it isn't specified here to avoid -# masking the more specific request requirements in acme. See -# https://github.com/pypa/pip/issues/988 for more info. +# This package relies on PyOpenSSL, requests, and six, however, it isn't +# specified here to avoid masking the more specific request requirements in +# acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but @@ -44,13 +43,11 @@ install_requires = [ 'cryptography>=1.2', # load_pem_x509_certificate 'mock', 'parsedatetime>=1.3', # Calendar.parseDT - 'PyOpenSSL', 'pyrfc3339', 'pytz', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', - 'six', 'zope.component', 'zope.interface', ] From ed2168aaa8c8a7e1bef449e60167b53d501d173a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 21 Dec 2017 16:55:21 -0800 Subject: [PATCH 4/8] Fix auto_tests on systems with new bootstrappers (#5348) --- letsencrypt-auto-source/tests/auto_test.py | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index 2fa03105d..156466c82 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -30,6 +30,10 @@ sys.path.insert(0, dirname(tests_dir())) from build import build as build_le_auto +BOOTSTRAP_FILENAME = 'certbot-auto-bootstrap-version.txt' +"""Name of the file where certbot-auto saves its bootstrap version.""" + + class RequestHandler(BaseHTTPRequestHandler): """An HTTPS request handler which is quiet and serves a specific folder.""" @@ -296,17 +300,31 @@ class AutoTests(TestCase): def test_phase2_upgrade(self): """Test a phase-2 upgrade without a phase-1 upgrade.""" - with temp_paths() as (le_auto_path, venv_dir): - resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), - 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, - 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} - with serving(resources) as base_url: + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), + 'v99.9.9/letsencrypt-auto': self.NEW_LE_AUTO, + 'v99.9.9/letsencrypt-auto.sig': self.NEW_LE_AUTO_SIG} + with serving(resources) as base_url: + pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') + with temp_paths() as (le_auto_path, venv_dir): + install_le_auto(self.NEW_LE_AUTO, le_auto_path) + + # Create venv saving the correct bootstrap script version + out, err = run_le_auto(le_auto_path, venv_dir, base_url, + PIP_FIND_LINKS=pip_find_links) + self.assertFalse('Upgrading certbot-auto ' in out) + self.assertTrue('Creating virtual environment...' in out) + with open(join(venv_dir, BOOTSTRAP_FILENAME)) as f: + bootstrap_version = f.read() + + # Create a new venv with an old letsencrypt version + with temp_paths() as (le_auto_path, venv_dir): venv_bin = join(venv_dir, 'bin') makedirs(venv_bin) set_le_script_version(venv_dir, '0.0.1') + with open(join(venv_dir, BOOTSTRAP_FILENAME), 'w') as f: + f.write(bootstrap_version) install_le_auto(self.NEW_LE_AUTO, le_auto_path) - pip_find_links=join(tests_dir(), 'fake-letsencrypt', 'dist') out, err = run_le_auto(le_auto_path, venv_dir, base_url, PIP_FIND_LINKS=pip_find_links) From 5388842e5b3868e29caf545fb771a23e7fce4143 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Jan 2018 17:49:22 -0800 Subject: [PATCH 5/8] Fix pytest on macOS in Travis (#5360) * Add tools/pytest.sh * pass TRAVIS through in tox.ini * Use tools/pytest.sh to run pytest * Add quiet to pytest.ini * ignore pytest cache --- .gitignore | 3 +++ pytest.ini | 2 ++ tools/install_and_test.sh | 2 +- tools/pytest.sh | 15 +++++++++++++++ tox.cover.sh | 3 ++- tox.ini | 9 +++++++++ 6 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100755 tools/pytest.sh diff --git a/.gitignore b/.gitignore index b63e40d1c..e018cf938 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ tests/letstest/*.pem tests/letstest/venv/ .venv + +# pytest cache +/.cache diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..b64550cb7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --quiet diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh index 25b6d548a..0d39e0594 100755 --- a/tools/install_and_test.sh +++ b/tools/install_and_test.sh @@ -23,5 +23,5 @@ for requirement in "$@" ; do # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333. pkg=$(echo "$pkg" | tr - _) fi - pytest --numprocesses auto --quiet --pyargs $pkg + "$(dirname $0)/pytest.sh" --pyargs $pkg done diff --git a/tools/pytest.sh b/tools/pytest.sh new file mode 100755 index 000000000..8e3619d5d --- /dev/null +++ b/tools/pytest.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Runs pytest with the provided arguments, adding --numprocesses to the command +# line. This argument is set to "auto" if the environmnent variable TRAVIS is +# not set, otherwise, it is set to 2. This works around +# https://github.com/pytest-dev/pytest-xdist/issues/9. Currently every Travis +# environnment provides two cores. See +# https://docs.travis-ci.com/user/reference/overview/#Virtualization-environments. + +if ${TRAVIS:-false}; then + NUMPROCESSES="2" +else + NUMPROCESSES="auto" +fi + +pytest --numprocesses "$NUMPROCESSES" "$@" diff --git a/tox.cover.sh b/tox.cover.sh index 2b5a3cf19..bc0e5a8bf 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -51,7 +51,8 @@ cover () { fi pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses auto --pyargs "$1" + pytest="$(dirname $0)/tools/pytest.sh" + "$pytest" --cov "$pkg_dir" --cov-append --cov-report= --pyargs "$1" coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing } diff --git a/tox.ini b/tox.ini index 6ebf681ed..20f5cda32 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,7 @@ commands = deps = setuptools==36.8.0 wheel==0.29.0 +passenv = TRAVIS [testenv] commands = @@ -69,12 +70,16 @@ commands = setenv = PYTHONPATH = {toxinidir} PYTHONHASHSEED = 0 +passenv = + {[testenv:py26]passenv} [testenv:py33] commands = {[testenv]commands} deps = wheel==0.29.0 +passenv = + {[testenv]passenv} [testenv:py27-oldest] commands = @@ -82,6 +87,8 @@ commands = setenv = {[testenv]setenv} CERTBOT_OLDEST=1 +passenv = + {[testenv]passenv} [testenv:py27_install] basepython = python2.7 @@ -93,6 +100,8 @@ basepython = python2.7 commands = {[base]install_packages} ./tox.cover.sh +passenv = + {[testenv]passenv} [testenv:lint] # recent versions of pylint do not support Python 2.6 (#97, #187) From a7d00ee21b454115fc0ce831b13f7902d4b62c37 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Jan 2018 13:59:29 -0800 Subject: [PATCH 6/8] print as a string (#5359) --- certbot-nginx/certbot_nginx/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e9d4e36d4..8af474c5e 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -182,7 +182,7 @@ class NginxConfigurator(common.Installer): self.parser.add_server_directives(vhost, cert_directives, replace=True) logger.info("Deployed Certificate to VirtualHost %s for %s", - vhost.filep, vhost.names) + vhost.filep, ", ".join(vhost.names)) self.save_notes += ("Changed vhost at %s with addresses of %s\n" % (vhost.filep, From a3a66cd25d8340e982481e7adf4a521c09f0f35e Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 00:36:16 +0200 Subject: [PATCH 7/8] Use apache2ctl modules for Gentoo systems. (#5349) * Do not call Apache binary for module reset in cleanup() * Use apache2ctl modules for Gentoo --- .../certbot_apache/override_gentoo.py | 8 +++ .../certbot_apache/tests/gentoo_test.py | 49 +++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/certbot-apache/certbot_apache/override_gentoo.py b/certbot-apache/certbot_apache/override_gentoo.py index d4d4e96b9..92f1d4a20 100644 --- a/certbot-apache/certbot_apache/override_gentoo.py +++ b/certbot-apache/certbot_apache/override_gentoo.py @@ -49,6 +49,7 @@ class GentooParser(parser.ApacheParser): def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ self.parse_sysconfig_var() + self.update_modules() def parse_sysconfig_var(self): """ Parses Apache CLI options from Gentoo configuration file """ @@ -56,3 +57,10 @@ class GentooParser(parser.ApacheParser): "APACHE2_OPTS") for k in defines.keys(): self.variables[k] = defines[k] + + def update_modules(self): + """Get loaded modules from httpd process, and add them to DOM""" + mod_cmd = [self.configurator.constant("apache_cmd"), "modules"] + matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module") + for mod in matches: + self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/tests/gentoo_test.py b/certbot-apache/certbot_apache/tests/gentoo_test.py index 0f2b96818..cfbaffac7 100644 --- a/certbot-apache/certbot_apache/tests/gentoo_test.py +++ b/certbot-apache/certbot_apache/tests/gentoo_test.py @@ -2,6 +2,8 @@ import os import unittest +import mock + from certbot_apache import override_gentoo from certbot_apache import obj from certbot_apache.tests import util @@ -46,9 +48,10 @@ class MultipleVhostsTestGentoo(util.ApacheTest): config_root=config_root, vhost_root=vhost_root) - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, - os_info="gentoo") + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"): + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, + os_info="gentoo") self.vh_truth = get_vh_truth( self.temp_dir, "gentoo_apache/apache") @@ -78,9 +81,47 @@ class MultipleVhostsTestGentoo(util.ApacheTest): self.config.parser.apacheconfig_filep = os.path.realpath( os.path.join(self.config.parser.root, "../conf.d/apache2")) self.config.parser.variables = {} - self.config.parser.update_runtime_variables() + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() for define in defines: self.assertTrue(define in self.config.parser.variables.keys()) + @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess") + def test_no_binary_configdump(self, mock_subprocess): + """Make sure we don't call binary dumps other than modules from Apache + as this is not supported in Gentoo currently""" + + with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"): + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertFalse(mock_subprocess.called) + + self.config.parser.update_runtime_variables() + self.config.parser.reset_modules() + self.assertTrue(mock_subprocess.called) + + @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg") + def test_opportunistic_httpd_runtime_parsing(self, mock_get): + mod_val = ( + 'Loaded Modules:\n' + ' mock_module (static)\n' + ' another_module (static)\n' + ) + def mock_get_cfg(command): + """Mock httpd process stdout""" + if command == ['apache2ctl', 'modules']: + return mod_val + mock_get.side_effect = mock_get_cfg + self.config.parser.modules = set() + + with mock.patch("certbot.util.get_os_info") as mock_osi: + # Make sure we have the have the CentOS httpd constants + mock_osi.return_value = ("gentoo", "123") + self.config.parser.update_runtime_variables() + + self.assertEquals(mock_get.call_count, 1) + self.assertEquals(len(self.config.parser.modules), 4) + self.assertTrue("mod_another.c" in self.config.parser.modules) + if __name__ == "__main__": unittest.main() # pragma: no cover From a1713c0b79b99108ae3a1233cb3e3dc3bef2908a Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Fri, 5 Jan 2018 21:08:38 +0200 Subject: [PATCH 8/8] Broader git ignore for pytest cache files (#5361) Make gitignore take pytest cache directories in to account, even if they reside in subdirectories. If pytest is run for a certain module, ie. `pytest certbot-apache` the cache directory is created under `certbot-apache` directory. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e018cf938..a01d2e1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ tests/letstest/venv/ .venv # pytest cache -/.cache +.cache