Merge branch 'master' into plugin-stability

This commit is contained in:
Erica Portnoy 2020-07-22 16:13:39 -07:00
commit 8e783d026e
22 changed files with 250 additions and 71 deletions

View file

@ -58,54 +58,48 @@ jobs:
set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
displayName: Run certbot integration tests
- job: snap_build
strategy:
matrix:
amd64:
ARCH: amd64
# Do not run the QEMU jobs for test branches
${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
arm64:
ARCH: arm64
armhf:
ARCH: armhf
- job: snaps_build
pool:
vmImage: ubuntu-18.04
timeoutInMinutes: 0
variables:
# Do not run the heavy non-amd64 builds for test branches
${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
ARCHS: amd64 arm64 armhf
${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}:
ARCHS: amd64
steps:
- script: |
tools/snap/build.sh ${ARCH}
sudo apt-get update
sudo apt-get install -y --no-install-recommends snapd
sudo snap install --classic snapcraft
displayName: Install dependencies
- task: UsePythonVersion@0
inputs:
versionSpec: 3.8
addToPath: true
- task: DownloadSecureFile@1
name: credentials
inputs:
secureFile: launchpad-credentials
- script: |
git config --global user.email "$(Build.RequestedForEmail)"
git config --global user.name "$(Build.RequestedFor)"
mkdir -p ~/.local/share/snapcraft/provider/launchpad
cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
displayName: Build snaps
- script: |
mv *.snap $(Build.ArtifactStagingDirectory)
displayName: Build Certbot snap
- task: PublishPipelineArtifact@1
inputs:
path: $(Build.ArtifactStagingDirectory)
artifact: snap-$(arch)
displayName: Store snap artifact
- job: snap_dns_build
strategy:
matrix:
amd64:
ARCH: amd64
# Do not run the QEMU jobs for test branches
${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
arm64:
ARCH: arm64
armhf:
ARCH: armhf
pool:
vmImage: ubuntu-18.04
steps:
- script: |
tools/snap/build_dns.sh ${ARCH} ALL
mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
displayName: Build Certbot DNS snaps
displayName: Prepare artifacts
- task: PublishPipelineArtifact@1
inputs:
path: $(Build.ArtifactStagingDirectory)
artifact: dns-snap-$(arch)
artifact: snaps
displayName: Store snaps artifacts
- job: snap_run
dependsOn: snap_build
dependsOn: snaps_build
pool:
vmImage: ubuntu-18.04
steps:
@ -116,19 +110,17 @@ jobs:
displayName: Install dependencies
- task: DownloadPipelineArtifact@2
inputs:
artifact: snap-amd64
artifact: snaps
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot snap
displayName: Retrieve Certbot snaps
- script: |
sudo snap install --dangerous --classic snap/*.snap
sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
displayName: Install Certbot snap
- script: |
python -m tox -e integration-external,apacheconftest-external-with-pebble
displayName: Run tox
- job: snap_dns_run
dependsOn:
- snap_build
- snap_dns_build
dependsOn: snaps_build
pool:
vmImage: ubuntu-18.04
steps:
@ -142,18 +134,13 @@ jobs:
addToPath: true
- task: DownloadPipelineArtifact@2
inputs:
artifact: snap-amd64
artifact: snaps
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot snap
- task: DownloadPipelineArtifact@2
inputs:
artifact: dns-snap-amd64
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot DNS plugins snaps
displayName: Retrieve Certbot snaps
- script: |
python3 -m venv venv
venv/bin/python tools/pip_install.py -e certbot-ci
displayName: Prepare Certbot-CI
- script: |
sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap
sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
displayName: Test DNS plugins snaps

View file

@ -9,14 +9,6 @@ stages:
# prevent automated deploys from breaking. Remembering to do this is
# also tracked by https://github.com/certbot/certbot/issues/7931.
- job: publish_snap
strategy:
matrix:
amd64:
ARCH: amd64
arm64:
ARCH: arm64
armhf:
ARCH: armhf
pool:
vmImage: ubuntu-18.04
variables:
@ -29,14 +21,9 @@ stages:
displayName: Install dependencies
- task: DownloadPipelineArtifact@2
inputs:
artifact: snap-$(arch)
artifact: snaps
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot snap
- task: DownloadPipelineArtifact@2
inputs:
artifact: dns-snap-$(arch)
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve DNS plugins snaps
displayName: Retrieve Certbot snaps
- task: DownloadSecureFile@1
name: snapcraftCfg
inputs:

View file

@ -17,6 +17,8 @@ def pytest_addoption(parser):
"""
parser.addoption('--snap-folder', required=True,
help='set the folder path where snaps to test are located')
parser.addoption('--snap-arch', default='amd64',
help='set the architecture do test (default: amd64)')
parser.addoption('--allow-persistent-changes', action='store_true',
help='needs to be set, and confirm that the test will make persistent changes on this machine')
@ -36,5 +38,8 @@ def pytest_generate_tests(metafunc):
Generate (multiple) parametrized calls to a test function.
"""
if "dns_snap_path" in metafunc.fixturenames:
snap_dns_path_list = glob.glob(os.path.join(metafunc.config.getoption('snap_folder'), 'certbot-dns-*_*.snap'))
snap_arch = metafunc.config.getoption('snap_arch')
snap_folder = metafunc.config.getoption('snap_folder')
snap_dns_path_list = glob.glob(os.path.join(snap_folder,
'certbot-dns-*_{0}.snap'.format(snap_arch)))
metafunc.parametrize("dns_snap_path", snap_dns_path_list)

View file

@ -11,8 +11,9 @@ def install_certbot_snap(request):
with pytest.raises(Exception):
subprocess.check_call(['certbot', '--version'])
try:
snap_path = glob.glob(os.path.join(request.config.getoption("snap_folder"),
'certbot_*.snap'))[0]
snap_folder = request.config.getoption("snap_folder")
snap_arch = request.config.getoption("snap_arch")
snap_path = glob.glob(os.path.join(snap_folder, 'certbot_*_{0}.snap'.format(snap_arch)))[0]
subprocess.check_call(['snap', 'install', '--classic', '--dangerous', snap_path])
subprocess.check_call(['certbot', '--version'])
yield

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -17,6 +17,8 @@ parts:
snapcraftctl set-version `grep ^version $SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .

View file

@ -60,6 +60,7 @@ parts:
# effect so we now stage the file to keep the auto-generated cffi file.
stage-packages:
- libaugeas0
- libpython3.8-dev
# added to stage python:
- libpython3-stdlib
- libpython3.8-stdlib
@ -73,7 +74,9 @@ parts:
- python3-pkg-resources
- python3.8-minimal
# To build cryptography and cffi if needed
build-packages: [libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev]
build-environment:
- SNAPCRAFT_PYTHON_VENV_ARGS: --system-site-packages
override-pull: |
snapcraftctl pull
cd $SNAPCRAFT_PART_SRC

166
tools/snap/build_remote.py Executable file
View file

@ -0,0 +1,166 @@
#!/usr/bin/env python3
import argparse
import glob
import datetime
from multiprocessing import Pool, Process, Manager, Event
import re
import subprocess
import sys
from os.path import join, realpath, dirname, basename
CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__))))
PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))]
def _execute_build(target, archs, status, workspace):
process = subprocess.Popen([
'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', '--build-on', ','.join(archs)
], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, cwd=workspace)
for line in process.stdout:
_extract_state(target, line, status)
return process.wait()
def _build_snap(target, archs, status):
status[target] = {arch: '...' for arch in archs}
if target == 'certbot':
workspace = CERTBOT_DIR
else:
workspace = join(CERTBOT_DIR, target)
subprocess.check_output(
('"{0}" tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt '
'| grep -v python-augeas > "{1}/snap-constraints.txt"').format(sys.executable, workspace),
shell=True, cwd=CERTBOT_DIR)
retry = 3
while retry:
exit_code = _execute_build(target, archs, status, workspace)
print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with exit code {exit_code}.')
sys.stdout.flush()
# Retry if the snapcraft remote-build command has been interrupted.
if exit_code == 0 and 'Failed to build' not in status[target].values():
break
retry = retry - 1
return {target: workspace}
def _extract_state(project, output, status):
match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output)
if match:
arch = match.group(1)
state = status[project]
state[arch] = match.group(2)
# You need to reassign the value of status[project] here (rather than doing
# something like status[project][arch] = match.group(2)) for the state change
# to propagate to other processes. See
# https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for
# more info.
status[project] = state
def _dump_status_helper(archs, status):
headers = ['project', *archs]
print(''.join(f'| {item:<25}' for item in headers))
print(f'|{"-" * 26}' * len(headers))
for project, states in sorted(status.items()):
print(''.join(f'| {item:<25}' for item in [project, *[states[arch] for arch in archs]]))
print(f'|{"-" * 26}' * len(headers))
print()
sys.stdout.flush()
def _dump_status(archs, status, stop_event):
while not stop_event.wait(10):
print('Remote build status at {0}'.format(datetime.datetime.now()))
_dump_status_helper(archs, status)
def _dump_status_final(archs, status):
print('Results for remote build finished at {0}'.format(datetime.datetime.now()))
_dump_status_helper(archs, status)
def _dump_results(targets, archs, status, workspaces):
failures = False
for target in targets:
for arch in archs:
result = status[target][arch]
if result != 'Successfully built':
failures = True
with open(join(workspaces[target], '{0}_{1}.txt'.format(target, arch))) as file_h:
build_output = file_h.read()
print('Output for failed build target={0} arch={1}'.format(target, arch))
print('-------------------------------------------')
print(build_output)
print('-------------------------------------------')
print()
if not failures:
print('All builds succeeded.')
else:
print('Some builds failed.')
return failures
def main():
parser = argparse.ArgumentParser()
parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS],
help='the list of snaps to build')
parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default='amd64',
help='the architectures for which snaps are built')
args = parser.parse_args()
archs = set(args.archs)
targets = set(args.targets)
if 'ALL' in targets:
targets.remove('ALL')
targets.update(['certbot', 'DNS_PLUGINS'])
if 'DNS_PLUGINS' in targets:
targets.remove('DNS_PLUGINS')
targets.update(PLUGINS)
print('Start remote snap builds...')
print(f' - archs: {", ".join(archs)}')
print(f' - projects: {", ".join(sorted(targets))}')
print()
with Manager() as manager, Pool(processes=len(targets)) as pool:
status = manager.dict()
stop_event = Event()
state_process = Process(target=_dump_status, args=(archs, status, stop_event))
state_process.start()
async_results = [pool.apply_async(_build_snap, (target, archs, status)) for target in targets]
workspaces = {}
for async_result in async_results:
workspaces.update(async_result.get())
stop_event.set()
state_process.join()
failures = _dump_results(targets, archs, status, workspaces)
_dump_status_final(archs, status)
return 1 if failures else 0
if __name__ == '__main__':
sys.exit(main())

View file

@ -3,7 +3,7 @@
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CERTBOT_DIR="$(dirname "${DIR}")"
CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")"
for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do
PLUGIN=$(basename "${PLUGIN_PATH}")
@ -29,6 +29,8 @@ parts:
snapcraftctl set-version \`grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"\`
build-environment:
- SNAP_BUILD: "True"
# To build cryptography and cffi if needed
build-packages: [gcc, libffi-dev, libssl-dev, python3-dev]
certbot-metadata:
plugin: dump
source: .