diff --git a/.gitignore b/.gitignore index 34987d319..064e7fffe 100644 --- a/.gitignore +++ b/.gitignore @@ -58,5 +58,3 @@ parts prime stage *.snap -snap-constraints.txt -qemu-* diff --git a/.travis.yml b/.travis.yml index 19e2eb0d1..8e893bd24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ before_script: # Use Travis retry feature for farm tests since they are flaky - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi' - export TOX_TESTENV_PASSENV=TRAVIS - - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh $TARGET_ARCH; fi' + - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi' # Only build pushes to the master branch, PRs, and branches beginning with # `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or @@ -39,45 +39,6 @@ not-on-master: ¬-on-master extended-test-suite: &extended-test-suite if: type = cron OR (type = push AND branch != master) -# Common configuration for all snap tasks -snap-config: &snap-config - dist: bionic - addons: - apt: - packages: - - nginx-light - git: - # By default, Travis clones the repo to a depth of 50 commits which can - # break the ability to use `git describe` to set the version of the - # snap. This setting removes the --depth flag from git commands solving - # this problem. See - # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth - # for more info. - depth: false - deploy: - # This section relies on credentials stored in a SNAP_TOKEN environment - # variable in Travis. See - # https://docs.travis-ci.com/user/deployment/snaps/ for more info. - # This credential has a maximum lifetime of 1 year and the current - # credential will expire on 4/22/2021. The value of SNAP_TOKEN will - # need to be updated to use a new credential before then to prevent - # automated deploys from breaking. Remembering to do this is also - # tracked by https://github.com/certbot/certbot/issues/7931. - 'on': - # Deploy on release tags or nightly runs from any branch. We only try - # to deploy from the certbot/certbot repo to prevent errors if forks - # of this repo try to run tests. - all_branches: true - condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron - repo: certbot/certbot - provider: snap - snap: certbot_*.snap - channel: edge - # skip_cleanup is needed to prevent Travis from deleting the snaps we - # just built and tested. See - # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. - skip_cleanup: true - matrix: include: # Main test suite @@ -270,17 +231,49 @@ matrix: - libaugeas0 <<: *extended-test-suite - stage: "Snap" - env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble TARGET_ARCH=amd64 - <<: *snap-config - <<: *extended-test-suite - - env: SNAP=true TARGET_ARCH=i386 - <<: *snap-config - <<: *extended-test-suite - - env: SNAP=true TARGET_ARCH=arm64 - <<: *snap-config - <<: *extended-test-suite - - env: SNAP=true TARGET_ARCH=armhf - <<: *snap-config + sudo: required + env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble + addons: + apt: + packages: + - nginx-light + snaps: + - name: snapcraft + channel: stable + confinement: classic + - name: lxd + channel: stable + git: + # By default, Travis clones the repo to a depth of 50 commits which can + # break the ability to use `git describe` to set the version of the + # snap. This setting removes the --depth flag from git commands solving + # this problem. See + # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth + # for more info. + depth: false + deploy: + # This section relies on credentials stored in a SNAP_TOKEN environment + # variable in Travis. See + # https://docs.travis-ci.com/user/deployment/snaps/ for more info. + # This credential has a maximum lifetime of 1 year and the current + # credential will expire on 4/22/2021. The value of SNAP_TOKEN will + # need to be updated to use a new credential before then to prevent + # automated deploys from breaking. Remembering to do this is also + # tracked by https://github.com/certbot/certbot/issues/7931. + 'on': + # Deploy on release tags or nightly runs from any branch. We only try + # to deploy from the certbot/certbot repo to prevent errors if forks + # of this repo try to run tests. + all_branches: true + condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron + repo: certbot/certbot + provider: snap + snap: certbot_*.snap + channel: edge + # skip_cleanup is needed to prevent Travis from deleting the snaps we + # just built and tested. See + # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. + skip_cleanup: true <<: *extended-test-suite # container-based infrastructure @@ -311,7 +304,7 @@ install: 'tools/pip_install.py -I tox virtualenv' # script command. It is set only to `travis_retry` during farm tests, in # order to trigger the Travis retry feature, and compensate the inherent # flakiness of these specific tests. -script: 'if [[ ! -z "$TOXENV" ]]; then $TRAVIS_RETRY tox; fi' +script: '$TRAVIS_RETRY tox' notifications: email: false diff --git a/snap/local/build_and_install.sh b/snap/local/build_and_install.sh index b5883bc8a..4330c762b 100755 --- a/snap/local/build_and_install.sh +++ b/snap/local/build_and_install.sh @@ -1,53 +1,20 @@ #!/bin/bash -# Cross-compile the Certbot snap from local sources for the specified architecture, -# and install it if this architecture is also the the current machine one. -# This script is designed for CI tests purpose. -# Usage: build_and_install.sh [amd64,arm64,armhf] set -ex -if [[ -z "${TRAVIS}" ]]; then +if [[ -z "$TRAVIS" ]]; then echo "This script makes global changes to the system it is run on so should only be run in CI." exit 1 fi -SNAP_ARCH=$1 - -if [[ -z "${SNAP_ARCH}" ]]; then - echo "You need to specify the target architecture" - exit 1 -fi - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -CERTBOT_DIR="$(dirname "$(dirname "${DIR}")")" - -# shellcheck source=common.sh -source "${DIR}/common.sh" - -RegisterQemuHandlers -ResolveArch "${SNAP_ARCH}" - +# Add the current user to the lxd group so they can run `snapcraft --use-lxd` +# without sudo since running the command without sudo is required by newer +# versions of snapcraft. +sudo usermod -aG lxd "$USER" +sudo /snap/bin/lxd.migrate -yes +sudo /snap/bin/lxd waitready +sudo /snap/bin/lxd init --auto tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > snap-constraints.txt - -pushd "${DIR}/packages" -"${CERTBOT_DIR}/tools/simple_http_server.py" 8080 >/dev/null 2>&1 & -HTTP_SERVER_PID="$!" -popd - -function cleanup() { - kill "${HTTP_SERVER_PID}" -} - -trap cleanup EXIT - -docker run \ - --rm \ - --net=host \ - -v "${CERTBOT_DIR}:/certbot" \ - -w "/certbot" \ - -e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \ - "adferrand/snapcraft:${DOCKER_ARCH}-stable" \ - snapcraft - -if [[ "$(arch)" == "${QEMU_ARCH}" ]]; then - sudo snap install --dangerous --classic *.snap -fi +# Run snapcraft with the lxd group since it has not been added to the current +# shell. +sg lxd -c 'snapcraft --use-lxd' +sudo snap install --dangerous --classic *.snap diff --git a/snap/local/common.sh b/snap/local/common.sh deleted file mode 100644 index 00a7614e8..000000000 --- a/snap/local/common.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Common bash functions useful for cross-compiling Certbot snaps. - -# Resolve the Snap architecture to Docker architecture (DOCKER_ARCH variable) -# and QEMU architecture (QEMU_ARCH variable). -# Usage: ResolveArch [amd64|i386|arm64|armhf] -ResolveArch() { - local SNAP_ARCH=$1 - - case "${SNAP_ARCH}" in - "amd64") - DOCKER_ARCH="amd64" - QEMU_ARCH="x86_64" - ;; - "i386") - DOCKER_ARCH="i386" - QEMU_ARCH="i386" - ;; - "arm64") - DOCKER_ARCH="arm64v8" - QEMU_ARCH="aarch64" - ;; - "armhf") - DOCKER_ARCH="arm32v7" - QEMU_ARCH="arm" - ;; - "*") - echo "Not supported build architecture '$1'." >&2 - exit 1 - esac -} - -# Downloads QEMU static binary file for architecture -# Usage: DownloadQemuStatic [x86_64|aarch64|arm] DEST_DIR -DownloadQemuStatic() { - local QEMU_ARCH=$1 - local DEST_DIR=$2 - local QEMU_DOWNLOAD_URL - local QEMU_LATEST_TAG - - if [ ! -f "${DIR}/qemu-${QEMU_ARCH}-static" ]; then - QEMU_DOWNLOAD_URL="https://github.com/multiarch/qemu-user-static/releases/download" - QEMU_LATEST_TAG=$(curl -s https://api.github.com/repos/multiarch/qemu-user-static/tags \ - | grep 'name.*v[0-9]' \ - | head -n 1 \ - | cut -d '"' -f 4) - echo "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz" - curl -SL "${QEMU_DOWNLOAD_URL}/${QEMU_LATEST_TAG}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz" \ - | tar xzv -C "${DEST_DIR}" - fi -} - -# Executes the QEMU register script -# Usage: RegisterQemuHandlers -RegisterQemuHandlers() { - docker run --rm --privileged multiarch/qemu-user-static:register --reset -} diff --git a/snap/local/compile_native_wheels.sh b/snap/local/compile_native_wheels.sh deleted file mode 100755 index 5dbb0f5d6..000000000 --- a/snap/local/compile_native_wheels.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Cross-compile cryptography and cffi native wheels for arm64 and armhf architectures, -# on the versions required by the current pinning of Certbot dependencies. -# Wheels are stored in snap/local/packages folder to speed up cross-compilation of Certbot snap. -set -ex - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -TARGET_ARCHS="i386 arm64 armhf" - -rm -rf "${DIR}/packages/"* - -# shellcheck source=common.sh -source "${DIR}/common.sh" - -RegisterQemuHandlers - -tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > "${DIR}/snap-constraints.txt" -for SNAP_ARCH in ${TARGET_ARCHS}; do - ResolveArch "${SNAP_ARCH}" - DownloadQemuStatic "${QEMU_ARCH}" "${DIR}" - - docker run \ - --rm \ - -v "${DIR}/qemu-${QEMU_ARCH}-static:/usr/bin/qemu-${QEMU_ARCH}-static" \ - -v "${DIR}:/workspace" \ - -w "/workspace" \ - "${DOCKER_ARCH}/ubuntu:18.04" \ - sh -c "\ - apt-get update \ -&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-dev libffi-dev libssl-dev gcc \ -&& mkdir -p /build \ -&& python3 -m venv /build/venv \ -&& /build/venv/bin/pip install wheel \ -&& /build/venv/bin/pip wheel cryptography cffi -c snap-constraints.txt -w /build \ -&& mkdir -p /workspace/packages/cffi /workspace/packages/cryptography \ -&& mv /build/cryptography-* /workspace/packages/cryptography \ -&& mv /build/cffi-* /workspace/packages/cffi \ -&& chmod 777 /workspace/packages /workspace/packages/cffi /workspace/packages/cryptography \ -&& chmod 666 /workspace/packages/cffi/* /workspace/packages/cryptography/* -" -done diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl deleted file mode 100644 index 36dfa9ade..000000000 Binary files a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_aarch64.whl and /dev/null differ diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl deleted file mode 100644 index 26f107cc9..000000000 Binary files a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-linux_armv7l.whl and /dev/null differ diff --git a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl b/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl deleted file mode 100644 index 1e5491b6e..000000000 Binary files a/snap/local/packages/cffi/cffi-1.14.0-cp36-cp36m-manylinux1_i686.whl and /dev/null differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl deleted file mode 100644 index 3a969945a..000000000 Binary files a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_aarch64.whl and /dev/null differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl deleted file mode 100644 index ea49f5dab..000000000 Binary files a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_armv7l.whl and /dev/null differ diff --git a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl b/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl deleted file mode 100644 index 18d7e6fd7..000000000 Binary files a/snap/local/packages/cryptography/cryptography-2.8-cp36-cp36m-linux_i686.whl and /dev/null differ