From 86c986194b57ff1fdab51ec64c2d3b522b49708e Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 17 Jul 2020 14:58:31 +0200 Subject: [PATCH] Updates --- .../templates/jobs/packaging-jobs.yml | 5 +-- tools/snap/build_dns_remote.sh | 39 ------------------- tools/snap/build_remote.py | 29 ++++++++------ tools/snap/build_remote.sh | 18 --------- 4 files changed, 19 insertions(+), 72 deletions(-) delete mode 100755 tools/snap/build_dns_remote.sh delete mode 100755 tools/snap/build_remote.sh diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 0f2dccfb7..0be7f04c1 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -81,12 +81,11 @@ jobs: mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials python3 tools/snap/build_remote.py ALL --archs amd64 arm64 armhf - find . -iname *.snap - displayName: Build Certbot + DNS plugins snaps + displayName: Build snaps - script: | mv *.snap $(Build.ArtifactStagingDirectory) mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory) - displayName: Build prepare artifacts + displayName: Prepare artifacts - task: PublishPipelineArtifact@1 inputs: path: $(Build.ArtifactStagingDirectory) diff --git a/tools/snap/build_dns_remote.sh b/tools/snap/build_dns_remote.sh deleted file mode 100755 index 32a09d62f..000000000 --- a/tools/snap/build_dns_remote.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -ex - -SNAP_ARCHS=$1 -DNS_PLUGINS=$2 - -if [[ -z "${SNAP_ARCHS}" ]]; then - echo "You need to specify at least one target architecture" - exit 1 -fi - -if [[ -z "${DNS_PLUGINS}" ]]; then - echo "You need to specify the DNS plugins" - exit 1 -fi - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")" - -if [[ "${DNS_PLUGINS}" = "ALL" ]]; then - DNS_PLUGINS=$(find ${CERTBOT_DIR} -maxdepth 1 -type d -name "certbot-dns-*" -exec basename {} \; | paste -sd "," -) -fi - -trap popd EXIT - -function run() { - local DNS_PLUGIN=$1 - pushd "${CERTBOT_DIR}/${DNS_PLUGIN}" - python3 ../tools/strip_hashes.py ../letsencrypt-auto-source/pieces/dependency-requirements.txt | grep -v python-augeas > snap-constraints.txt - snapcraft remote-build --launchpad-accept-public-upload --build-on="${SNAP_ARCHS}" - popd -} - -IFS="," -for DNS_PLUGIN in ${DNS_PLUGINS}; do - run "${DNS_PLUGIN}" & -done - -wait diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index d3996de3a..490ab3f8e 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -14,7 +14,7 @@ CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] -def _build_snap(target, archs, status): +def _build_remote_snap(target, archs, status): status[target] = {} if target == 'certbot': workspace = CERTBOT_DIR @@ -31,13 +31,13 @@ def _build_snap(target, archs, status): line = process.stdout.readline() while line: - _extract_state(target, line, status) + _extract_remote_state(target, line, status) line = process.stdout.readline() return {target: workspace} -def _extract_state(project, output, status): +def _extract_remote_state(project, output, status): match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) @@ -57,13 +57,17 @@ def _extract_state(project, output, status): status[project] = state -def _dump_status(status): +def _dump_remote_status(status, final=False): while True: - print('Build status at {0}'.format(datetime.datetime.now())) - print('W = waiting, B = building, U = uploading, F = fail, S = success') + if final: + print('Results for remote build finished at {0}'.format(datetime.datetime.now())) + print('F = fail, S = success') + else: + print('Remote build status at {0}'.format(datetime.datetime.now())) + print('W = waiting, B = building, U = uploading, F = fail, S = success') print(' project amd64 arm64 armhf ') print('---------------------------+-------+-------+-------') - for project, states in status.items(): + for project, states in sorted(status.items()): print(' {0} | {1} | {2} | {3} '.format( project + ' ' * (25 - len(project)), states.get('arm64', 'W'), states.get('arm64', 'W'), states.get('armhf', 'W'))) @@ -71,10 +75,10 @@ def _dump_status(status): sys.stdout.flush() - time.sleep(5) + time.sleep(10) -def _dump_results(targets, archs, status, workspaces): +def _dump_remote_results(targets, archs, status, workspaces): failures = False for target in targets: for arch in archs: @@ -120,11 +124,11 @@ def main(): status = Manager().dict() - state_process = Process(target=_dump_status, args=(status,)) + state_process = Process(target=_dump_remote_status, args=(status,)) state_process.start() pool = Pool(processes=len(targets)) - async_results = [pool.apply_async(_build_snap, (target, archs, status)) for target in targets] + async_results = [pool.apply_async(_build_remote_snap, (target, archs, status)) for target in targets] workspaces = {} for async_result in async_results: @@ -132,7 +136,8 @@ def main(): state_process.terminate() - failures = _dump_results(targets, archs, status, workspaces) + failures = _dump_remote_results(targets, archs, status, workspaces) + _dump_remote_status(status, final=True) return 1 if failures else 0 diff --git a/tools/snap/build_remote.sh b/tools/snap/build_remote.sh deleted file mode 100755 index e56238684..000000000 --- a/tools/snap/build_remote.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -ex - -SNAP_ARCHS=$1 - -if [[ -z "${SNAP_ARCHS}" ]]; then - echo "You need to specify at least one target architecture" - exit 1 -fi - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")" - -trap popd EXIT - -pushd "${CERTBOT_DIR}" -snapcraft remote-build --launchpad-accept-public-upload --build-on="${SNAP_ARCHS}" -popd