From d434b92945f04f84b10fee1048cb45a15d3a36ba Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 9 Jul 2020 20:33:25 +0200 Subject: [PATCH] Build the DNS plugins snaps (#8129) Fixes #8041 This PR makes Azure Pipeline build the DNS plugins snaps for the 3 architectures during the CI. It leverages the existing logic for building the Certbot snap in order to deploy a QEMU environment with Docker, and leverages the local PyPI index to speed up the build when installing `cffi` and `cryptography`. All DNS plugins snaps are constructed in one unique docker container, in order to save the time required to install the system dependencies upon first start of `snapcraft`, and so speed up significantly the build. Finally, all `amd64` DNS plugins snaps are built within 6 minutes. For `arm64` and `armhf`, it is around 40 mins: this is quite fast in fact, considering that 14 DNS plugins snaps are built. However, this is still an extremely heavy task to make the full 3 architectures builds, even for Azure Pipelines and its 10 parallel jobs capability. That is why I make the `arm64` and `armhf` builds be skipped for the `full-test-suite`, and let them run only for `nightly` and `release`. This means however that these builds will not be done for the release branches. If this is a problem, I can put a more elaborate suspend condition to triggers the builds in this case. All snaps are stored in the pipeline artifacts storage, making them available for publication during a `release` pipeline. The PR is set as Draft for now, because I use temporarily `pr_test-suite` to validate the packaging jobs when commits are pushed. Once the PR is ready, I will revert it back to the normal configuration (run the standard tests). * Configure a script to build DNS snaps * Focus on packaging * Trigger all architectures * Add extra index * Prepare conditional suspend * Set final suspend logic * Set final suspend value * Loop for publication * Use python3 * Clean before build * Add a test * Add test job in Azure * Preserve env * Apply normal config for pipelines * Skip QEMU jobs only for test branches * Makes snap run tests depends also on the Certbot snap build * Update .azure-pipelines/templates/jobs/packaging-jobs.yml Co-authored-by: Brad Warren * Update .azure-pipelines/templates/stages/deploy-stage.yml Co-authored-by: Brad Warren * More accurate way to get the plugin snap name * Integrate DNS snap tests into certbot-ci * Fixes * Update certbot-ci/snap_integration_tests/conftest.py Co-authored-by: Brad Warren * Update certbot-ci/snap_integration_tests/conftest.py Co-authored-by: Brad Warren * Clean an _init_.py file Co-authored-by: Brad Warren --- .../templates/jobs/packaging-jobs.yml | 67 ++++++++++++++-- .../templates/stages/deploy-stage.yml | 9 ++- certbot-ci/snap_integration_tests/__init__.py | 0 certbot-ci/snap_integration_tests/conftest.py | 40 ++++++++++ .../dns_tests/__init__.py | 0 .../dns_tests/test_main.py | 42 ++++++++++ {snap/local => tools/snap}/README.md | 0 {snap/local => tools/snap}/build.sh | 3 +- tools/snap/build_dns.sh | 73 ++++++++++++++++++ {snap/local => tools/snap}/common.sh | 0 .../snap}/compile_native_wheels.sh | 0 .../cffi-1.14.0-cp38-cp38-linux_aarch64.whl | Bin .../cffi-1.14.0-cp38-cp38-linux_armv7l.whl | Bin ...yptography-2.8-cp38-cp38-linux_aarch64.whl | Bin ...ryptography-2.8-cp38-cp38-linux_armv7l.whl | Bin 15 files changed, 226 insertions(+), 8 deletions(-) create mode 100644 certbot-ci/snap_integration_tests/__init__.py create mode 100644 certbot-ci/snap_integration_tests/conftest.py create mode 100644 certbot-ci/snap_integration_tests/dns_tests/__init__.py create mode 100644 certbot-ci/snap_integration_tests/dns_tests/test_main.py rename {snap/local => tools/snap}/README.md (100%) rename {snap/local => tools/snap}/build.sh (90%) create mode 100755 tools/snap/build_dns.sh rename {snap/local => tools/snap}/common.sh (100%) rename {snap/local => tools/snap}/compile_native_wheels.sh (100%) rename {snap/local => tools/snap}/packages/cffi/cffi-1.14.0-cp38-cp38-linux_aarch64.whl (100%) rename {snap/local => tools/snap}/packages/cffi/cffi-1.14.0-cp38-cp38-linux_armv7l.whl (100%) rename {snap/local => tools/snap}/packages/cryptography/cryptography-2.8-cp38-cp38-linux_aarch64.whl (100%) rename {snap/local => tools/snap}/packages/cryptography/cryptography-2.8-cp38-cp38-linux_armv7l.whl (100%) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 97904b0ba..9742269e4 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -63,15 +63,17 @@ jobs: matrix: amd64: ARCH: amd64 - arm64: - ARCH: arm64 - armhf: - ARCH: armhf + # Do not run the QEMU jobs for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + arm64: + ARCH: arm64 + armhf: + ARCH: armhf pool: vmImage: ubuntu-18.04 steps: - script: | - snap/local/build.sh ${ARCH} + tools/snap/build.sh ${ARCH} mv *.snap $(Build.ArtifactStagingDirectory) displayName: Build Certbot snap - task: PublishPipelineArtifact@1 @@ -79,6 +81,29 @@ jobs: path: $(Build.ArtifactStagingDirectory) artifact: snap-$(arch) displayName: Store snap artifact + - job: snap_dns_build + strategy: + matrix: + amd64: + ARCH: amd64 + # Do not run the QEMU jobs for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + arm64: + ARCH: arm64 + armhf: + ARCH: armhf + pool: + vmImage: ubuntu-18.04 + steps: + - script: | + tools/snap/build_dns.sh ${ARCH} ALL + mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory) + displayName: Build Certbot DNS snaps + - task: PublishPipelineArtifact@1 + inputs: + path: $(Build.ArtifactStagingDirectory) + artifact: dns-snap-$(arch) + displayName: Store snaps artifacts - job: snap_run dependsOn: snap_build pool: @@ -100,3 +125,35 @@ jobs: - script: | python -m tox -e integration-external,apacheconftest-external-with-pebble displayName: Run tox + - job: snap_dns_run + dependsOn: + - snap_build + - snap_dns_build + pool: + vmImage: ubuntu-18.04 + steps: + - script: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends snapd + displayName: Install dependencies + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true + - task: DownloadPipelineArtifact@2 + inputs: + artifact: snap-amd64 + path: $(Build.SourcesDirectory)/snap + displayName: Retrieve Certbot snap + - task: DownloadPipelineArtifact@2 + inputs: + artifact: dns-snap-amd64 + path: $(Build.SourcesDirectory)/snap + displayName: Retrieve Certbot DNS plugins snaps + - script: | + python3 -m venv venv + venv/bin/python tools/pip_install.py -e certbot-ci + displayName: Prepare Certbot-CI + - script: | + sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap + displayName: Test DNS plugins snaps diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index 53d2be5a1..daf64aabe 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -32,6 +32,11 @@ stages: artifact: snap-$(arch) path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snap + - task: DownloadPipelineArtifact@2 + inputs: + artifact: dns-snap-$(arch) + path: $(Build.SourcesDirectory)/snap + displayName: Retrieve DNS plugins snaps - task: DownloadSecureFile@1 name: snapcraftCfg inputs: @@ -39,5 +44,7 @@ stages: - bash: | mkdir -p .snapcraft ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg - snapcraft upload --release=edge snap/*.snap + for SNAP_FILE in snap/*.snap; do + snapcraft upload --release=edge "${SNAP_FILE}" + done displayName: Publish to Snap store diff --git a/certbot-ci/snap_integration_tests/__init__.py b/certbot-ci/snap_integration_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/snap_integration_tests/conftest.py b/certbot-ci/snap_integration_tests/conftest.py new file mode 100644 index 000000000..e9c6176c3 --- /dev/null +++ b/certbot-ci/snap_integration_tests/conftest.py @@ -0,0 +1,40 @@ +""" +General conftest for pytest execution of all integration tests lying +in the snap_installer_integration tests package. +As stated by pytest documentation, conftest module is used to set on +for a directory a specific configuration using built-in pytest hooks. + +See https://docs.pytest.org/en/latest/reference.html#hook-reference +""" +import glob +import os + + +def pytest_addoption(parser): + """ + Standard pytest hook to add options to the pytest parser. + :param parser: current pytest parser that will be used on the CLI + """ + parser.addoption('--snap-folder', required=True, + help='set the folder path where snaps to test are located') + parser.addoption('--allow-persistent-changes', action='store_true', + help='needs to be set, and confirm that the test will make persistent changes on this machine') + + +def pytest_configure(config): + """ + Standard pytest hook used to add a configuration logic for each node of a pytest run. + :param config: the current pytest configuration + """ + if not config.option.allow_persistent_changes: + raise RuntimeError('This integration test would install the Certbot snap on your machine. ' + 'Please run it again with the `--allow-persistent-changes` flag set to acknowledge.') + + +def pytest_generate_tests(metafunc): + """ + Generate (multiple) parametrized calls to a test function. + """ + if "dns_snap_path" in metafunc.fixturenames: + snap_dns_path_list = glob.glob(os.path.join(metafunc.config.getoption('snap_folder'), 'certbot-dns-*_*.snap')) + metafunc.parametrize("dns_snap_path", snap_dns_path_list) diff --git a/certbot-ci/snap_integration_tests/dns_tests/__init__.py b/certbot-ci/snap_integration_tests/dns_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py new file mode 100644 index 000000000..ef133dc59 --- /dev/null +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import pytest +import subprocess +import glob +import os +import re + + +@pytest.fixture(autouse=True, scope="module") +def install_certbot_snap(request): + with pytest.raises(Exception): + subprocess.check_call(['certbot', '--version']) + try: + snap_path = glob.glob(os.path.join(request.config.getoption("snap_folder"), + 'certbot_*.snap'))[0] + subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path]) + subprocess.check_call(['certbot', '--version']) + yield + finally: + subprocess.call(['snap', 'remove', 'certbot']) + + +def test_dns_plugin_install(dns_snap_path): + """ + Test that each DNS plugin Certbot snap can be installed + and is usable with the Certbot snap. + """ + plugin_name = re.match(r'^certbot-(dns-\w+)_.*\.snap$', + os.path.basename(dns_snap_path)).group(1) + snap_name = 'certbot-{0}'.format(plugin_name) + assert plugin_name not in subprocess.check_output(['certbot', 'plugins', '--prepare'], + universal_newlines=True) + + try: + subprocess.check_call(['snap', 'install', '--dangerous', dns_snap_path]) + subprocess.check_call(['snap', 'set', 'certbot', 'trust-plugin-with-root=ok']) + subprocess.check_call(['snap', 'connect', 'certbot:plugin', snap_name]) + + assert plugin_name in subprocess.check_output(['certbot', 'plugins', '--prepare'], + universal_newlines=True) + finally: + subprocess.call(['snap', 'remove', 'plugin_name']) diff --git a/snap/local/README.md b/tools/snap/README.md similarity index 100% rename from snap/local/README.md rename to tools/snap/README.md diff --git a/snap/local/build.sh b/tools/snap/build.sh similarity index 90% rename from snap/local/build.sh rename to tools/snap/build.sh index eccb99760..ef34c479a 100755 --- a/snap/local/build.sh +++ b/tools/snap/build.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Cross-compile the Certbot snap from local sources for the specified architecture, -# and install it if this architecture is also the the current machine one. +# Cross-compile the Certbot snap from local sources for the specified architecture. # This script is designed for CI tests purpose. # Usage: build.sh [amd64,arm64,armhf] set -ex diff --git a/tools/snap/build_dns.sh b/tools/snap/build_dns.sh new file mode 100755 index 000000000..aba008d7e --- /dev/null +++ b/tools/snap/build_dns.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Cross-compile the specified Certbot DNS plugins snaps from local sources for the specified architecture. +# This script is designed for CI tests purpose. +# Usage: build.sh [amd64,arm64,armhf] [DNS_PLUGIN1,DNS_PLUGIN2 or ALL] +set -ex + +SNAP_ARCH=$1 +DNS_PLUGINS=$2 + +if [[ -z "${SNAP_ARCH}" ]]; then + echo "You need to specify the target architecture" + exit 1 +fi + +if [[ -z "${DNS_PLUGINS}" ]]; then + echo "You need to specify the DNS plugins" + exit 1 +fi + +if [[ "${DNS_PLUGINS}" = "ALL" ]]; then + DNS_PLUGINS=$(find . -maxdepth 1 -type d -name "certbot-dns-*" -exec basename {} \; | paste -sd "," -) +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")" + +# shellcheck source=common.sh +source "${DIR}/common.sh" + +RegisterQemuHandlers +ResolveArch "${SNAP_ARCH}" + +pushd "${DIR}/packages" +"${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 & +HTTP_SERVER_PID="$!" +popd + +function cleanup() { + kill "${HTTP_SERVER_PID}" +} + +trap cleanup EXIT + +SCRIPT=$(mktemp /tmp/script.XXXXXX.sh) +chmod +x "${SCRIPT}" + +SNAP_CONSTRAINTS=$(mktemp /tmp/snap-constraints.XXXXXX.txt) +python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > "${SNAP_CONSTRAINTS}" + +cat << "EOF" >> "${SCRIPT}" +#!/bin/bash +set -ex +IFS="," +for DNS_PLUGIN in ${DNS_PLUGINS}; do + pushd "${DNS_PLUGIN}" + cp /snap-constraints.txt . + snapcraft clean + snapcraft + popd +done +EOF + +docker run \ + --rm \ + --net=host \ + -v "${CERTBOT_DIR}:/certbot" \ + -v "${SCRIPT}:/script.sh" \ + -v "${SNAP_CONSTRAINTS}:/snap-constraints.txt" \ + -w "/certbot" \ + -e "DNS_PLUGINS=${DNS_PLUGINS}" \ + -e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \ + "adferrand/snapcraft:${DOCKER_ARCH}-stable" \ + /script.sh diff --git a/snap/local/common.sh b/tools/snap/common.sh similarity index 100% rename from snap/local/common.sh rename to tools/snap/common.sh diff --git a/snap/local/compile_native_wheels.sh b/tools/snap/compile_native_wheels.sh similarity index 100% rename from snap/local/compile_native_wheels.sh rename to tools/snap/compile_native_wheels.sh diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp38-cp38-linux_aarch64.whl b/tools/snap/packages/cffi/cffi-1.14.0-cp38-cp38-linux_aarch64.whl similarity index 100% rename from snap/local/packages/cffi/cffi-1.14.0-cp38-cp38-linux_aarch64.whl rename to tools/snap/packages/cffi/cffi-1.14.0-cp38-cp38-linux_aarch64.whl diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp38-cp38-linux_armv7l.whl b/tools/snap/packages/cffi/cffi-1.14.0-cp38-cp38-linux_armv7l.whl similarity index 100% rename from snap/local/packages/cffi/cffi-1.14.0-cp38-cp38-linux_armv7l.whl rename to tools/snap/packages/cffi/cffi-1.14.0-cp38-cp38-linux_armv7l.whl diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp38-cp38-linux_aarch64.whl b/tools/snap/packages/cryptography/cryptography-2.8-cp38-cp38-linux_aarch64.whl similarity index 100% rename from snap/local/packages/cryptography/cryptography-2.8-cp38-cp38-linux_aarch64.whl rename to tools/snap/packages/cryptography/cryptography-2.8-cp38-cp38-linux_aarch64.whl diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp38-cp38-linux_armv7l.whl b/tools/snap/packages/cryptography/cryptography-2.8-cp38-cp38-linux_armv7l.whl similarity index 100% rename from snap/local/packages/cryptography/cryptography-2.8-cp38-cp38-linux_armv7l.whl rename to tools/snap/packages/cryptography/cryptography-2.8-cp38-cp38-linux_armv7l.whl