[Unix] Create a framework for certbot integration tests: PART 1 (#6578)

* First part

* Several optimizations about the docker env setup

* Documentation

* Various corrections and documentation. Add acme and certbot explicitly as dependencies of certbot-ci.

* Correct a variable misinterpreted as a pytest hook

* Correct strict parsing option on pebble

* Refactor acme setup to be executed from pytest hooks.

* Pass TRAVIS env variable to trigger specific xdist logic

* Retrigger build.

* Work in progress

* Config operational

* Propagate to xdist

* Corrections on acme and misc

* Correct subnet for pebble

* Remove gobetween, as tls-sni challenges are not tested anymore.

* Improve pebble setup. Reduce LOC.

* Update acme.py

* Optimize acme ca setup, with less temporary assets

* Silent setup

* Clean code

* Remove unused workspace

* Use default network driver

* Remove bridge

* Update package documentation

* Remove rerun capability for integration tests, not needed.

* Add documentation

* Variable for all ports and subnets used by the stack

* Update certbot-ci/certbot_integration_tests/conftest.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/acme.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/misc.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update tox.ini

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/misc.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/acme.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/utils/acme.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Update certbot-ci/certbot_integration_tests/conftest.py

Co-Authored-By: adferrand <adferrand@users.noreply.github.com>

* Rename to acme_server

* Add comment

* Refactor in a unique context fixture

* Remove the need of CERTBOT_ACME_XDIST environment variable

* Remove nonstrict/strict options in pebble

* Clean dependencies

* Clean tox

* Change function name

* Add comment about coveragerc specificities

* Change a comment.

* Update setup.py

* Update conftest.py

* Use the production-ready docker-compose.yml file for Pebble

* New style class

* Tune pebble to have a stable test environment

* Pin a dependency

certbot-ci/certbot_integration_tests/.coveragerc

@@ -0,0 +1,8 @@
+[run]
+# Avoid false warnings because certbot packages are not installed in the thread that executes
+# the coverage: indeed, certbot is launched as a CLI from a subprocess.
+disable_warnings = module-not-imported,no-data-collected
+
+[report]
+# Exclude unit tests in coverage during integration tests.
+omit = **/*_test.py,**/tests/*,**/certbot_nginx/parser_obj.py

certbot-ci/certbot_integration_tests/__init__.py

@@ -0,0 +1 @@
+"""Package certbot_integration_test is for tests that require a live acme ca server instance"""

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -0,0 +1,16 @@
+class IntegrationTestsContext(object):
+    """General fixture describing a certbot integration tests context"""
+    def __init__(self, request):
+        self.request = request
+        if hasattr(request.config, 'slaveinput'):  # Worker node
+            self.worker_id = request.config.slaveinput['slaveid']
+            self.acme_xdist = request.config.slaveinput['acme_xdist']
+        else:  # Primary node
+            self.worker_id = 'primary'
+            self.acme_xdist = request.config.acme_xdist
+        self.directory_url = self.acme_xdist['directory_url']
+        self.tls_alpn_01_port = self.acme_xdist['https_port'][self.worker_id]
+        self.http_01_port = self.acme_xdist['http_port'][self.worker_id]
+
+    def cleanup(self):
+        pass

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -0,0 +1,40 @@
+import requests
+import urllib3
+
+import pytest
+
+from certbot_integration_tests.certbot_tests import context as certbot_context
+
+
+@pytest.fixture()
+def context(request):
+    # Fixture request is a built-in pytest fixture describing current test request.
+    integration_test_context = certbot_context.IntegrationTestsContext(request)
+    try:
+        yield integration_test_context
+    finally:
+        integration_test_context.cleanup()
+
+
+def test_hello_1(context):
+    assert context.http_01_port
+    assert context.tls_alpn_01_port
+    try:
+        response = requests.get(context.directory_url, verify=False)
+        response.raise_for_status()
+        assert response.json()
+        response.close()
+    except urllib3.exceptions.InsecureRequestWarning:
+        pass
+
+
+def test_hello_2(context):
+    assert context.http_01_port
+    assert context.tls_alpn_01_port
+    try:
+        response = requests.get(context.directory_url, verify=False)
+        response.raise_for_status()
+        assert response.json()
+        response.close()
+    except urllib3.exceptions.InsecureRequestWarning:
+        pass

certbot-ci/certbot_integration_tests/conftest.py

@@ -0,0 +1,92 @@
+"""
+General conftest for pytest execution of all integration tests lying
+in the certbot_integration tests package.
+As stated by pytest documentation, conftest module is used to set on
+for a directory a specific configuration using built-in pytest hooks.
+
+See https://docs.pytest.org/en/latest/reference.html#hook-reference
+"""
+import contextlib
+import sys
+import subprocess
+
+from certbot_integration_tests.utils import acme_server as acme_lib
+
+
+def pytest_addoption(parser):
+    """
+    Standard pytest hook to add options to the pytest parser.
+    :param parser: current pytest parser that will be used on the CLI
+    """
+    parser.addoption('--acme-server', default='pebble',
+                     choices=['boulder-v1', 'boulder-v2', 'pebble'],
+                     help='select the ACME server to use (boulder-v1, boulder-v2, '
+                          'pebble), defaulting to pebble')
+
+
+def pytest_configure(config):
+    """
+    Standard pytest hook used to add a configuration logic for each node of a pytest run.
+    :param config: the current pytest configuration
+    """
+    if not hasattr(config, 'slaveinput'):  # If true, this is the primary node
+        with _print_on_err():
+            config.acme_xdist = _setup_primary_node(config)
+
+
+def pytest_configure_node(node):
+    """
+    Standard pytest-xdist hook used to configure a worker node.
+    :param node: current worker node
+    """
+    node.slaveinput['acme_xdist'] = node.config.acme_xdist
+
+
+@contextlib.contextmanager
+def _print_on_err():
+    """
+    During pytest-xdist setup, stdout is used for nodes communication, so print is useless.
+    However, stderr is still available. This context manager transfers stdout to stderr
+    for the duration of the context, allowing to display prints to the user.
+    """
+    old_stdout = sys.stdout
+    sys.stdout = sys.stderr
+    try:
+        yield
+    finally:
+        sys.stdout = old_stdout
+
+
+def _setup_primary_node(config):
+    """
+    Setup the environment for integration tests.
+    Will:
+        - check runtime compatiblity (Docker, docker-compose, Nginx)
+        - create a temporary workspace and the persistent GIT repositories space
+        - configure and start paralleled ACME CA servers using Docker
+        - transfer ACME CA servers configurations to pytest nodes using env variables
+    :param config: Configuration of the pytest primary node
+    """
+    # Check for runtime compatibility: some tools are required to be available in PATH
+    try:
+        subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
+    except (subprocess.CalledProcessError, OSError):
+        raise ValueError('Error: docker is required in PATH to launch the integration tests, '
+                         'but is not installed or not available for current user.')
+
+    try:
+        subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT)
+    except (subprocess.CalledProcessError, OSError):
+        raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, '
+                         'but is not installed or not available for current user.')
+
+    # Parameter numprocesses is added to option by pytest-xdist
+    workers = ['primary'] if not config.option.numprocesses\
+        else ['gw{0}'.format(i) for i in range(config.option.numprocesses)]
+
+    # By calling setup_acme_server we ensure that all necessary acme server instances will be
+    # fully started. This runtime is reflected by the acme_xdist returned.
+    acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers)
+    print('ACME xdist config:\n{0}'.format(acme_xdist))
+
+    return acme_xdist

certbot-ci/certbot_integration_tests/nginx_tests/context.py

@@ -0,0 +1,5 @@
+from certbot_integration_tests.certbot_tests import context as certbot_context
+
+
+class IntegrationTestsContext(certbot_context.IntegrationTestsContext):
+    """General fixture describing a certbot-nginx integration tests context"""

certbot-ci/certbot_integration_tests/nginx_tests/test_main.py

@@ -0,0 +1,17 @@
+import pytest
+
+from certbot_integration_tests.nginx_tests import context as nginx_context
+
+
+@pytest.fixture()
+def context(request):
+    # Fixture request is a built-in pytest fixture describing current test request.
+    integration_test_context = nginx_context.IntegrationTestsContext(request)
+    try:
+        yield integration_test_context
+    finally:
+        integration_test_context.cleanup()
+
+
+def test_hello(context):
+    print(context.directory_url)

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -0,0 +1,194 @@
+"""Module to setup an ACME CA server environment able to run multiple tests in parallel"""
+from __future__ import print_function
+import tempfile
+import atexit
+import os
+import subprocess
+import shutil
+import sys
+from os.path import join
+
+import requests
+import json
+import yaml
+
+from certbot_integration_tests.utils import misc
+
+# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble.
+CHALLTESTSRV_PORT = 8055
+HTTP_01_PORT = 5002
+
+
+def setup_acme_server(acme_server, nodes):
+    """
+    This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel
+    execution of integration tests against the unique http-01 port expected by the ACME CA server.
+    Instances are properly closed and cleaned when the Python process exits using atexit.
+    Typically all pytest integration tests will be executed in this context.
+    This method returns an object describing ports and directory url to use for each pytest node
+    with the relevant pytest xdist node.
+    :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
+    :param str[] nodes: list of node names that will be setup by pytest xdist
+    :return: a dict describing the challenge ports that have been setup for the nodes
+    :rtype: dict
+    """
+    acme_type = 'pebble' if acme_server == 'pebble' else 'boulder'
+    acme_xdist = _construct_acme_xdist(acme_server, nodes)
+    workspace = _construct_workspace(acme_type)
+
+    _prepare_traefik_proxy(workspace, acme_xdist)
+    _prepare_acme_server(workspace, acme_type, acme_xdist)
+
+    return acme_xdist
+
+
+def _construct_acme_xdist(acme_server, nodes):
+    """Generate and return the acme_xdist dict"""
+    acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT}
+
+    # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble.
+    if acme_server == 'pebble':
+        acme_xdist['directory_url'] = 'https://localhost:14000/dir'
+    else:  # boulder
+        port = 4001 if acme_server == 'boulder-v2' else 4000
+        acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port)
+
+    acme_xdist['http_port'] = {node: port for (node, port)
+                               in zip(nodes, range(5200, 5200 + len(nodes)))}
+    acme_xdist['https_port'] = {node: port for (node, port)
+                                in zip(nodes, range(5100, 5100 + len(nodes)))}
+
+    return acme_xdist
+
+
+def _construct_workspace(acme_type):
+    """Create a temporary workspace for integration tests stack"""
+    workspace = tempfile.mkdtemp()
+
+    def cleanup():
+        """Cleanup function to call that will teardown relevant dockers and their configuration."""
+        for instance in [acme_type, 'traefik']:
+            print('=> Tear down the {0} instance...'.format(instance))
+            instance_path = join(workspace, instance)
+            try:
+                if os.path.isfile(join(instance_path, 'docker-compose.yml')):
+                    _launch_command(['docker-compose', 'down'], cwd=instance_path)
+            except subprocess.CalledProcessError:
+                pass
+            print('=> Finished tear down of {0} instance.'.format(acme_type))
+
+        shutil.rmtree(workspace)
+
+    # Here with atexit we ensure that clean function is called no matter what.
+    atexit.register(cleanup)
+
+    return workspace
+
+
+def _prepare_acme_server(workspace, acme_type, acme_xdist):
+    """Configure and launch the ACME server, Boulder or Pebble"""
+    print('=> Starting {0} instance deployment...'.format(acme_type))
+    instance_path = join(workspace, acme_type)
+    try:
+        # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production.
+        _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type),
+                         '--single-branch', '--depth=1', instance_path])
+        if acme_type == 'boulder':
+            # Allow Boulder to ignore usual limit rate policies, useful for tests.
+            os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
+                      join(instance_path, 'test/rate-limit-policies.yml'))
+        if acme_type == 'pebble':
+            # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
+            # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
+            with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler:
+                config = yaml.load(file_handler.read())
+
+            config['services']['pebble'].setdefault('environment', [])\
+                .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0'])
+            with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler:
+                file_handler.write(yaml.dump(config))
+
+        # Launch the ACME CA server.
+        _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
+
+        # Wait for the ACME CA server to be up.
+        print('=> Waiting for {0} instance to respond...'.format(acme_type))
+        misc.check_until_timeout(acme_xdist['directory_url'])
+
+        # Configure challtestsrv to answer any A record request with ip of the docker host.
+        acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50'
+        response = requests.post('http://localhost:{0}/set-default-ipv4'
+                                 .format(acme_xdist['challtestsrv_port']),
+                                 json={'ip': '{0}.1'.format(acme_subnet)})
+        response.raise_for_status()
+
+        print('=> Finished {0} instance deployment.'.format(acme_type))
+    except BaseException:
+        print('Error while setting up {0} instance.'.format(acme_type))
+        raise
+
+
+def _prepare_traefik_proxy(workspace, acme_xdist):
+    """Configure and launch Traefik, the HTTP reverse proxy"""
+    print('=> Starting traefik instance deployment...')
+    instance_path = join(workspace, 'traefik')
+    traefik_subnet = '10.33.33'
+    traefik_api_port = 8056
+    try:
+        os.mkdir(instance_path)
+
+        with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h:
+            file_h.write('''\
+version: '3'
+services:
+  traefik:
+    image: traefik
+    command: --api --rest
+    ports:
+      - {http_01_port}:80
+      - {traefik_api_port}:8080
+    networks:
+      traefiknet:
+        ipv4_address: {traefik_subnet}.2
+networks:
+  traefiknet:
+    ipam:
+      config:
+        - subnet: {traefik_subnet}.0/24
+'''.format(traefik_subnet=traefik_subnet,
+           traefik_api_port=traefik_api_port,
+           http_01_port=HTTP_01_PORT))
+
+        _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
+
+        misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port))
+        config = {
+            'backends': {
+                node: {
+                    'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}}
+                } for node, port in acme_xdist['http_port'].items()
+            },
+            'frontends': {
+                node: {
+                    'backend': node, 'passHostHeader': True,
+                    'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}}
+                } for node in acme_xdist['http_port'].keys()
+            }
+        }
+        response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port),
+                                data=json.dumps(config))
+        response.raise_for_status()
+
+        print('=> Finished traefik instance deployment.')
+    except BaseException:
+        print('Error while setting up traefik instance.')
+        raise
+
+
+def _launch_command(command, cwd=os.getcwd()):
+    """Launch silently an OS command, output will be displayed in case of failure"""
+    try:
+        subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True)
+    except subprocess.CalledProcessError as e:
+        sys.stderr.write(e.output)
+        raise

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -0,0 +1,45 @@
+"""
+Misc module contains stateless functions that could be used during pytest execution,
+or outside during setup/teardown of the integration tests environment.
+"""
+import os
+import time
+import contextlib
+
+import requests
+
+
+def check_until_timeout(url):
+    """
+    Wait and block until given url responds with status 200, or raise an exception
+    after 150 attempts.
+    :param str url: the URL to test
+    :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL
+    """
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    for _ in range(0, 150):
+        time.sleep(1)
+        try:
+            if requests.get(url, verify=False).status_code == 200:
+                return
+        except requests.exceptions.ConnectionError:
+            pass
+
+    raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url))
+
+
+@contextlib.contextmanager
+def execute_in_given_cwd(cwd):
+    """
+    Context manager that will execute any command in the given cwd after entering context,
+    and restore current cwd when context is destroyed.
+    :param str cwd: the path to use as the temporary current workspace for python execution
+    """
+    current_cwd = os.getcwd()
+    try:
+        os.chdir(cwd)
+        yield
+    finally:
+        os.chdir(current_cwd)

certbot-ci/setup.py

@@ -0,0 +1,45 @@
+from setuptools import setup
+from setuptools import find_packages
+
+
+version = '0.32.0.dev0'
+
+install_requires = [
+    'pytest',
+    'pytest-cov',
+    'pytest-xdist',
+    'pytest-sugar',
+    'coverage',
+    'requests',
+    'pyyaml',
+]
+
+setup(
+    name='certbot-ci',
+    version=version,
+    description="Certbot continuous integration framework",
+    url='https://github.com/certbot/certbot',
+    author="Certbot Project",
+    author_email='client-dev@letsencrypt.org',
+    license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+    ],
+
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=install_requires,
+)

tools/dev_constraints.txt

@@ -51,6 +51,7 @@ pytest==3.2.5
 pytest-cov==2.5.1
 pytest-forked==0.2
 pytest-xdist==1.22.5
+pytest-sugar==0.9.2
 python-dateutil==2.6.1
 python-digitalocean==1.11
 PyYAML==3.13

tox.ini

@@ -250,3 +250,15 @@ commands =
 whitelist_externals =
     docker-compose
 passenv = DOCKER_*
+
+[testenv:integration]
+commands =
+    {[base]pip_install} acme . certbot-nginx certbot-ci
+    pytest {toxinidir}/certbot-ci/certbot_integration_tests \
+        --acme-server={env:ACME_SERVER:pebble} \
+        --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
+        --cov-config={toxinidir}/certbot-ci/certbot_integration_tests/.coveragerc \
+        -W 'ignore:Unverified HTTPS request'
+    coverage report --fail-under=65 --show-missing
+passenv =
+    DOCKER_*