Merge remote-tracking branch 'adrien/full-azure-pipelines' into test-azure-deploy

This commit is contained in:
Brad Warren 2020-07-02 13:54:47 -07:00
commit a39b8c3dd6
28 changed files with 475 additions and 504 deletions

View file

@ -1,13 +1,13 @@
# Advanced pipeline for running our full test suite on demand.
# Advanced pipeline for running our full test suite on demand and for release branches.
trigger:
- '*.x'
# When changing these triggers, please ensure the documentation under
# "Running tests in CI" is still correct.
- azure-test-*
- test-*
pr: none
jobs:
# Any addition here should be reflected in the advanced and release pipelines.
# It is advised to declare all jobs here as templates to improve maintainability.
- template: templates/tests-suite.yml
- template: templates/installer-tests.yml
stages:
- template: templates/stages/test-and-package-stage.yml
# Notify failures only for release branches.
- ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}:
- template: templates/stages/notify-failure-stage.yml

View file

@ -1,18 +0,0 @@
# Advanced pipeline for running our full test suite on protected branches.
trigger:
- '*.x'
pr: none
# This pipeline is also nightly run on master
schedules:
- cron: "0 4 * * *"
displayName: Nightly build
branches:
include:
- master
always: true
jobs:
# Any addition here should be reflected in the advanced-test and release pipelines.
# It is advised to declare all jobs here as templates to improve maintainability.
- template: templates/tests-suite.yml
- template: templates/installer-tests.yml

View file

@ -5,4 +5,4 @@ pr:
- '*.x'
jobs:
- template: templates/tests-suite.yml
- template: templates/jobs/standard-tests-jobs.yml

View file

@ -0,0 +1,15 @@
# Nightly pipeline running each day for master.
trigger: none
pr: none
schedules:
- cron: "0 4 * * *"
displayName: Nightly build
branches:
include:
- master
always: true
stages:
- template: templates/stages/test-and-package-stage.yml
- template: templates/stages/deploy-stage.yml
- template: templates/stages/notify-failure-stage.yml

View file

@ -5,9 +5,8 @@ trigger:
- v*
pr: none
jobs:
# Any addition here should be reflected in the advanced and advanced-test pipelines.
# It is advised to declare all jobs here as templates to improve maintainability.
- template: templates/tests-suite.yml
- template: templates/installer-tests.yml
- template: templates/changelog.yml
stages:
- template: templates/stages/test-and-package-stage.yml
- template: templates/stages/changelog-stage.yml
- template: templates/stages/deploy-stage.yml
- template: templates/stages/notify-failure-stage.yml

View file

@ -1,14 +0,0 @@
jobs:
- job: changelog
pool:
vmImage: vs2017-win2016
steps:
- bash: |
CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
"${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
displayName: Prepare changelog
- task: PublishPipelineArtifact@1
inputs:
path: $(Build.ArtifactStagingDirectory)
artifact: changelog
displayName: Publish changelog

View file

@ -0,0 +1,96 @@
jobs:
- job: extended_test
variables:
- name: IMAGE_NAME
value: ubuntu-18.04
- group: certbot-common
strategy:
matrix:
linux-py36:
PYTHON_VERSION: 3.6
TOXENV: py36
linux-py37:
PYTHON_VERSION: 3.7
TOXENV: py37
linux-py37-nopin:
PYTHON_VERSION: 3.7
TOXENV: py37
CERTBOT_NO_PIN: 1
linux-boulder-v1-integration-certbot-oldest:
TOXENV: integration-certbot-oldest
ACME_SERVER: boulder-v1
linux-boulder-v2-integration-certbot-oldest:
TOXENV: integration-certbot-oldest
ACME_SERVER: boulder-v2
linux-boulder-v1-integration-nginx-oldest:
TOXENV: integration-nginx-oldest
ACME_SERVER: boulder-v1
linux-boulder-v2-integration-nginx-oldest:
TOXENV: integration-nginx-oldest
ACME_SERVER: boulder-v2
linux-boulder-v1-py27-integration:
PYTHON_VERSION: 2.7
TOXENV: integration
ACME_SERVER: boulder-v1
linux-boulder-v2-py27-integration:
PYTHON_VERSION: 2.7
TOXENV: integration
ACME_SERVER: boulder-v2
linux-boulder-v1-py35-integration:
PYTHON_VERSION: 3.5
TOXENV: integration
ACME_SERVER: boulder-v1
linux-boulder-v2-py35-integration:
PYTHON_VERSION: 3.5
TOXENV: integration
ACME_SERVER: boulder-v2
linux-boulder-v1-py36-integration:
PYTHON_VERSION: 3.6
TOXENV: integration
ACME_SERVER: boulder-v1
linux-boulder-v2-py36-integration:
PYTHON_VERSION: 3.6
TOXENV: integration
ACME_SERVER: boulder-v2
linux-boulder-v1-py37-integration:
PYTHON_VERSION: 3.7
TOXENV: integration
ACME_SERVER: boulder-v1
linux-boulder-v2-py37-integration:
PYTHON_VERSION: 3.7
TOXENV: integration
ACME_SERVER: boulder-v2
linux-boulder-v1-py38-integration:
PYTHON_VERSION: 3.8
TOXENV: integration
ACME_SERVER: boulder-v1
linux-boulder-v2-py38-integration:
PYTHON_VERSION: 3.8
TOXENV: integration
ACME_SERVER: boulder-v2
nginx-compat:
TOXENV: nginx_compat
le-auto-jessie:
TOXENV: le_auto_jessie
le-auto-centos6:
TOXENV: le_auto_centos6
le-auto-oraclelinux6:
TOXENV: le_auto_oraclelinux6
docker-dev:
TOXENV: docker_dev
farmtest-apache2:
PYTHON_VERSION: 3.7
TOXENV: test-farm-apache2
farmtest-leauto-upgrades:
PYTHON_VERSION: 3.7
TOXENV: test-farm-leauto-upgrades
farmtest-certonly-standalone:
PYTHON_VERSION: 3.7
TOXENV: test-farm-certonly-standalone
farmtest-sdists:
PYTHON_VERSION: 3.7
TOXENV: test-farm-sdists
pool:
vmImage: $(IMAGE_NAME)
steps:
- template: ../steps/tox-steps.yml

View file

@ -32,8 +32,7 @@ jobs:
vmImage: $(imageName)
steps:
- powershell: |
$currentVersion = $PSVersionTable.PSVersion
if ($currentVersion.Major -ne 5) {
if ($PSVersionTable.PSVersion.Major -ne 5) {
throw "Powershell version is not 5.x"
}
condition: eq(variables['imageName'], 'vs2017-win2016')
@ -59,3 +58,45 @@ jobs:
set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
displayName: Run certbot integration tests
- job: snap_build
strategy:
matrix:
amd64:
ARCH: amd64
arm64:
ARCH: arm64
armhf:
ARCH: armhf
pool:
vmImage: ubuntu-18.04
steps:
- script: |
snap/local/build.sh ${ARCH}
mv *.snap $(Build.ArtifactStagingDirectory)
displayName: Build Certbot snap
- task: PublishPipelineArtifact@1
inputs:
path: $(Build.ArtifactStagingDirectory)
artifact: snap-$(arch)
displayName: Store snap artifact
- job: snap_run
dependsOn: snap_build
pool:
vmImage: ubuntu-18.04
steps:
- script: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends nginx-light snapd
python tools/pip_install.py -U tox
displayName: Install dependencies
- task: DownloadPipelineArtifact@2
inputs:
artifact: snap-amd64
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot snap
- script: |
sudo snap install --dangerous --classic snap/*.snap
displayName: Install Certbot snap
- script: |
python -m tox -e integration-external,apacheconftest-external-with-pebble
displayName: Run tox

View file

@ -0,0 +1,73 @@
jobs:
- job: test
strategy:
matrix:
macos-py27:
IMAGE_NAME: macOS-10.14
PYTHON_VERSION: 2.7
TOXENV: py27
macos-py38:
IMAGE_NAME: macOS-10.14
PYTHON_VERSION: 3.8
TOXENV: py38
windows-py35:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.5
TOXENV: py35
windows-py37-cover:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.7
TOXENV: py37-cover
windows-integration-certbot:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.7
TOXENV: integration-certbot
linux-oldest-tests-1:
IMAGE_NAME: ubuntu-18.04
TOXENV: py27-{acme,apache,apache-v2,certbot}-oldest
linux-oldest-tests-2:
IMAGE_NAME: ubuntu-18.04
TOXENV: py27-{dns,nginx}-oldest
linux-py27:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 2.7
TOXENV: py27
linux-py35:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 3.5
TOXENV: py35
linux-py38-cover:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 3.8
TOXENV: py38-cover
linux-py37-lint:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 3.7
TOXENV: lint
linux-py35-mypy:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 3.5
TOXENV: mypy
linux-integration:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 2.7
TOXENV: integration
ACME_SERVER: pebble
apache-compat:
IMAGE_NAME: ubuntu-18.04
TOXENV: apache_compat
le-auto-xenial:
IMAGE_NAME: ubuntu-18.04
TOXENV: le_auto_xenial
apacheconftest:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 2.7
TOXENV: apacheconftest-with-pebble
nginxroundtrip:
IMAGE_NAME: ubuntu-18.04
PYTHON_VERSION: 2.7
TOXENV: nginxroundtrip
pool:
vmImage: $(IMAGE_NAME)
steps:
- template: ../steps/tox-steps.yml

View file

@ -0,0 +1,16 @@
stages:
- stage: Changelog
jobs:
- job: prepare
pool:
vmImage: vs2017-win2016
steps:
- bash: |
CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
"${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
displayName: Prepare changelog
- task: PublishPipelineArtifact@1
inputs:
path: $(Build.ArtifactStagingDirectory)
artifact: changelog
displayName: Publish changelog

View file

@ -0,0 +1,43 @@
stages:
- stage: Deploy
jobs:
# This job relies on a snapcraft.cfg preconfigured with credential,
# stored as a secure file in Azure Pipeline.
# This credential has a maximum lifetime of 1 year and the current
# credential will expire on 6/25/2021. The content of snapcraft.cfg
# will need to be updated to use a new credential before then to
# prevent automated deploys from breaking. Remembering to do this is
# also tracked by https://github.com/certbot/certbot/issues/7931.
- job: publish_snap
strategy:
matrix:
amd64:
ARCH: amd64
arm64:
ARCH: arm64
armhf:
ARCH: armhf
pool:
vmImage: ubuntu-18.04
variables:
- group: certbot-common
steps:
- bash: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends snapd
sudo snap install --classic snapcraft
displayName: Install dependencies
- task: DownloadPipelineArtifact@2
inputs:
artifact: snap-$(arch)
path: $(Build.SourcesDirectory)/snap
displayName: Retrieve Certbot snap
- task: DownloadSecureFile@1
name: snapcraftCfg
inputs:
secureFile: snapcraft.cfg
- bash: |
mkdir -p .snapcraft
ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
snapcraft push --release=edge snap/*.snap
displayName: Publish to Snap store

View file

@ -0,0 +1,18 @@
stages:
- stage: On_Failure
jobs:
- job: notify_mattermost
variables:
- group: certbot-common
pool:
vmImage: ubuntu-latest
steps:
- bash: |
MESSAGE="\
---\n\
##### Azure Pipeline
*Repo* $(Build.Repository.ID) - *Pipeline* $(Build.DefinitionName) #$(Build.BuildNumber) - *Branch/PR* $(Build.SourceBranchName)\n\
:warning: __Pipeline has failed__: [Link to the build](https://dev.azure.com/$(Build.Repository.ID)/_build/results?buildId=$(Build.BuildId)&view=results)\n\n\
---"
curl -i -X POST --data-urlencode "payload={\"text\":\"${MESSAGE}\"}" "$(MATTERMOST_URL)"
condition: failed()

View file

@ -0,0 +1,6 @@
stages:
- stage: TestAndPackage
jobs:
- template: ../jobs/standard-tests-jobs.yml
- template: ../jobs/extended-tests-jobs.yml
- template: ../jobs/packaging-jobs.yml

View file

@ -0,0 +1,56 @@
steps:
- bash: |
brew install augeas
condition: startswith(variables['IMAGE_NAME'], 'macOS')
displayName: Install MacOS dependencies
- bash: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
python-dev \
gcc \
libaugeas0 \
libssl-dev \
libffi-dev \
ca-certificates \
nginx-light \
openssl
sudo systemctl stop nginx
condition: startswith(variables['IMAGE_NAME'], 'ubuntu')
displayName: Install Linux dependencies
- task: UsePythonVersion@0
inputs:
versionSpec: $(PYTHON_VERSION)
addToPath: true
condition: ne(variables['PYTHON_VERSION'], '')
# tools/pip_install.py is used to pin packages to a known working version
# except in tests where the environment variable CERTBOT_NO_PIN is set.
# virtualenv is listed here explicitly to make sure it is upgraded when
# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
# version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also
# set, pip updates dependencies it thinks are already satisfied to avoid some
# problems with its lack of real dependency resolution.
- bash: |
python tools/pip_install.py -I tox virtualenv
displayName: Install runtime dependencies
- task: DownloadSecureFile@1
name: testFarmPem
inputs:
secureFile: azure-test-farm.pem
condition: contains(variables['TOXENV'], 'test-farm')
- bash: |
ln -s $(testFarmPem.secureFilePath) tests/letstest/test-farm.pem
condition: contains(variables['TOXENV'], 'test-farm')
- bash: |
export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
[ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
env
if [[ "${TOXENV}" == *"oldest"* ]]; then
tools/run_oldest_tests.sh
else
python -m tox
fi
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_EC2_PEM_FILE: $(testFarmPem.secureFilePath)
displayName: Run tox

View file

@ -1,39 +0,0 @@
jobs:
- job: test
strategy:
matrix:
macos-py27:
IMAGE_NAME: macOS-10.14
PYTHON_VERSION: 2.7
TOXENV: py27
macos-py38:
IMAGE_NAME: macOS-10.14
PYTHON_VERSION: 3.8
TOXENV: py38
windows-py35:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.5
TOXENV: py35
windows-py37-cover:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.7
TOXENV: py37-cover
windows-integration-certbot:
IMAGE_NAME: vs2017-win2016
PYTHON_VERSION: 3.7
TOXENV: integration-certbot
PYTEST_ADDOPTS: --numprocesses 4
pool:
vmImage: $(IMAGE_NAME)
steps:
- bash: brew install augeas
condition: startswith(variables['IMAGE_NAME'], 'macOS')
displayName: Install Augeas
- task: UsePythonVersion@0
inputs:
versionSpec: $(PYTHON_VERSION)
addToPath: true
- script: python tools/pip_install.py -U tox coverage
displayName: Install dependencies
- script: python -m tox
displayName: Run tox

View file

@ -1,324 +0,0 @@
language: python
dist: xenial
cache:
directories:
- $HOME/.cache/pip
before_script:
# On Travis, the fastest parallelization for integration tests has proved to be 4.
- 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
# Use Travis retry feature for farm tests since they are flaky
- 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi'
- export TOX_TESTENV_PASSENV=TRAVIS
- 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh $TARGET_ARCH; fi'
# Only build pushes to the master branch, PRs, and branches beginning with
# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or
# `vdigit(s).digit(s).digit(s)`. As documented at
# https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches,
# this includes tags so pushing tags of the form `vdigit(s).digit(s).digit(s)`
# will also trigger tests. This reduces the number of simultaneous Travis runs,
# which speeds turnaround time on review since there is a cap of on the number
# of simultaneous runs.
branches:
# When changing these branches, please ensure the documentation under
# "Running tests in CI" is still correct.
only:
- master
- /^\d+\.\d+\.x$/ # this matches our point release branches
- /^v\d+\.\d+\.\d+$/ # this matches our release tags
- /^(travis-)?test-.*$/
# Jobs for the main test suite are always executed (including on PRs) except for pushes on master.
not-on-master: &not-on-master
if: NOT (type = push AND branch = master)
# Jobs for the extended test suite are executed for cron jobs and pushes to
# non-development branches.
extended-test-suite: &extended-test-suite
if: type = cron OR (type = push AND branch != master)
# Common configuration for all snap tasks
snap-config: &snap-config
dist: bionic
addons:
apt:
packages:
- nginx-light
git:
# By default, Travis clones the repo to a depth of 50 commits which can
# break the ability to use `git describe` to set the version of the
# snap. This setting removes the --depth flag from git commands solving
# this problem. See
# https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth
# for more info.
depth: false
deploy:
# This section relies on credentials stored in a SNAP_TOKEN environment
# variable in Travis. See
# https://docs.travis-ci.com/user/deployment/snaps/ for more info.
# This credential has a maximum lifetime of 1 year and the current
# credential will expire on 4/22/2021. The value of SNAP_TOKEN will
# need to be updated to use a new credential before then to prevent
# automated deploys from breaking. Remembering to do this is also
# tracked by https://github.com/certbot/certbot/issues/7931.
'on':
# Deploy on release tags or nightly runs from any branch. We only try
# to deploy from the certbot/certbot repo to prevent errors if forks
# of this repo try to run tests.
all_branches: true
condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron
repo: certbot/certbot
provider: snap
snap: certbot_*.snap
channel: edge
# skip_cleanup is needed to prevent Travis from deleting the snaps we
# just built and tested. See
# https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup.
skip_cleanup: true
matrix:
include:
# Main test suite
- stage: "Test"
python: "2.7"
env: ACME_SERVER=pebble TOXENV=integration
<<: *not-on-master
# As documented at
# https://docs.travis-ci.com/user/build-stages/#how-to-define-build-stages,
# the previous stage will be automatically applied to all subsequent jobs
# until a new stage is defined.
# This job is always executed, including on master
- python: "3.8"
env: TOXENV=py38-cover FYI="py38 tests + code coverage"
- python: "3.7"
env: TOXENV=lint
<<: *not-on-master
- python: "3.5"
env: TOXENV=mypy
<<: *not-on-master
- python: "2.7"
# Ubuntu Trusty or older must be used because the oldest version of
# cryptography we support cannot be compiled against the version of
# OpenSSL in Xenial or newer.
dist: trusty
env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest'
<<: *not-on-master
- python: "2.7"
env: TOXENV=py27
<<: *not-on-master
- python: "3.5"
env: TOXENV=py35
<<: *not-on-master
- sudo: required
env: TOXENV=apache_compat
services: docker
addons:
<<: *not-on-master
- sudo: required
env: TOXENV=le_auto_xenial
services: docker
<<: *not-on-master
- python: "2.7"
env: TOXENV=apacheconftest-with-pebble
<<: *not-on-master
- python: "2.7"
env: TOXENV=nginxroundtrip
<<: *not-on-master
# Extended test suite on cron jobs and pushes to tested branches other than master
- sudo: required
env: TOXENV=nginx_compat
services: docker
addons:
<<: *extended-test-suite
- python: "3.7"
env:
- TOXENV=travis-test-farm-apache2
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
<<: *extended-test-suite
- python: "3.7"
env:
- TOXENV=travis-test-farm-leauto-upgrades
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
git:
depth: false # This is needed to have the history to checkout old versions of certbot-auto.
<<: *extended-test-suite
- python: "3.7"
env:
- TOXENV=travis-test-farm-certonly-standalone
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
<<: *extended-test-suite
- python: "3.7"
env:
- TOXENV=travis-test-farm-sdists
- secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw="
<<: *extended-test-suite
- python: "3.7"
env: TOXENV=py37 CERTBOT_NO_PIN=1
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v1 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v2 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
# Ubuntu Trusty or older must be used because the oldest version of
# cryptography we support cannot be compiled against the version of
# OpenSSL in Xenial or newer.
dist: trusty
sudo: required
services: docker
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
# Ubuntu Trusty or older must be used because the oldest version of
# cryptography we support cannot be compiled against the version of
# OpenSSL in Xenial or newer.
dist: trusty
sudo: required
services: docker
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
# Ubuntu Trusty or older must be used because the oldest version of
# cryptography we support cannot be compiled against the version of
# OpenSSL in Xenial or newer.
dist: trusty
sudo: required
services: docker
<<: *extended-test-suite
- python: "2.7"
env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
# Ubuntu Trusty or older must be used because the oldest version of
# cryptography we support cannot be compiled against the version of
# OpenSSL in Xenial or newer.
dist: trusty
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.6"
env: TOXENV=py36
<<: *extended-test-suite
- python: "3.7"
env: TOXENV=py37
<<: *extended-test-suite
- python: "3.5"
env: ACME_SERVER=boulder-v1 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.5"
env: ACME_SERVER=boulder-v2 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.6"
env: ACME_SERVER=boulder-v1 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.6"
env: ACME_SERVER=boulder-v2 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.7"
env: ACME_SERVER=boulder-v1 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.7"
env: ACME_SERVER=boulder-v2 TOXENV=integration
sudo: required
services: docker
<<: *extended-test-suite
- python: "3.8"
env: ACME_SERVER=boulder-v1 TOXENV=integration
<<: *extended-test-suite
- python: "3.8"
env: ACME_SERVER=boulder-v2 TOXENV=integration
<<: *extended-test-suite
- sudo: required
env: TOXENV=le_auto_jessie
services: docker
<<: *extended-test-suite
- sudo: required
env: TOXENV=le_auto_centos6
services: docker
<<: *extended-test-suite
- sudo: required
env: TOXENV=le_auto_oraclelinux6
services: docker
<<: *extended-test-suite
- sudo: required
env: TOXENV=docker_dev
services: docker
addons:
apt:
packages: # don't install nginx and apache
- libaugeas0
<<: *extended-test-suite
- stage: "Snap"
env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble TARGET_ARCH=amd64
<<: *snap-config
<<: *extended-test-suite
- env: SNAP=true TARGET_ARCH=arm64
<<: *snap-config
<<: *extended-test-suite
- env: SNAP=true TARGET_ARCH=armhf
<<: *snap-config
<<: *extended-test-suite
# container-based infrastructure
sudo: false
addons:
apt:
packages: # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
- python-dev
- gcc
- libaugeas0
- libssl-dev
- libffi-dev
- ca-certificates
# For certbot-nginx integration testing
- nginx-light
- openssl
# tools/pip_install.py is used to pin packages to a known working version
# except in tests where the environment variable CERTBOT_NO_PIN is set.
# virtualenv is listed here explicitly to make sure it is upgraded when
# CERTBOT_NO_PIN is set to work around failures we've seen when using an older
# version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also
# set, pip updates dependencies it thinks are already satisfied to avoid some
# problems with its lack of real dependency resolution.
install: 'tools/pip_install.py -I tox virtualenv'
# Most of the time TRAVIS_RETRY is an empty string, and has no effect on the
# script command. It is set only to `travis_retry` during farm tests, in
# order to trigger the Travis retry feature, and compensate the inherent
# flakiness of these specific tests.
script: 'if [[ ! -z "$TOXENV" ]]; then $TRAVIS_RETRY tox; fi'
notifications:
email: false
irc:
if: NOT branch =~ ^(travis-)?test-.*$
channels:
# This is set to a secure variable to prevent forks from sending
# notifications. This value was created by installing
# https://github.com/travis-ci/travis.rb and running
# `travis encrypt "chat.freenode.net#certbot-devel"`.
- secure: "EWW66E2+KVPZyIPR8ViENZwfcup4Gx3/dlimmAZE0WuLwxDCshBBOd3O8Rf6pBokEoZlXM5eDT6XdyJj8n0DLslgjO62pExdunXpbcMwdY7l1ELxX2/UbnDTE6UnPYa09qVBHNG7156Z6yE0x2lH4M9Ykvp0G0cubjPQHylAwo0="
on_success: never
on_failure: always

View file

@ -17,8 +17,8 @@ install_requires = [
# 1.1.0+ is required to avoid the warnings described at
# https://github.com/certbot/josepy/issues/13.
'josepy>=1.1.0',
# Connection.set_tlsext_host_name (>=0.13)
'PyOpenSSL>=0.13.1',
# Connection.set_tlsext_host_name (>=0.13) + matching Xenial requirements (>=0.15.1)
'PyOpenSSL>=0.15.1',
'pyrfc3339',
'pytz',
'requests[security]>=2.6.0', # security extras added in 2.4.1

View file

@ -73,9 +73,9 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
|build-status|
.. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master
:target: https://travis-ci.com/certbot/certbot
:alt: Travis CI status
.. |build-status| image:: https://img.shields.io/azure-devops/build/certbot/ba534f81-a483-4b9b-9b4e-a60bec8fee72/2/master
:target: https://dev.azure.com/certbot/certbot/_build?definitionId=2
:alt: Azure Pipelines CI status
.. Do not modify this comment unless you know what you're doing. tag:links-end

View file

@ -137,7 +137,7 @@ of output can make it hard to find specific failures when they happen.
Running automated integration tests
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Generally it is sufficient to open a pull request and let Github and Travis run
Generally it is sufficient to open a pull request and let Github and Azure Pipelines run
integration tests for you. However, you may want to run them locally before submitting
your pull request. You need Docker and docker-compose installed and working.
@ -202,12 +202,8 @@ using an HTTP-01 challenge on a machine with Python 3:
Running tests in CI
~~~~~~~~~~~~~~~~~~~
Certbot uses both Azure Pipelines and Travis to run continuous integration
tests. If you are using our Azure and Travis setup, a branch whose name starts
with `test-` will run all Azure and Travis tests on that branch. If the branch
name starts with `azure-test-`, it will run all of our Azure tests and none of
our Travis tests. If the branch starts with `travis-test-`, only our Travis
tests will be run.
Certbot uses Azure Pipelines to run continuous integration tests. If you are using our
Azure setup, a branch whose name starts with `test-` will run all tests on that branch.
Code components and layout
==========================
@ -455,7 +451,7 @@ Steps:
containing your pull request to squash or amend commits. We use `squash
merges <https://github.com/blog/2141-squash-your-commits>`_ on PRs and
rewriting commits makes changes harder to track between reviews.
6. Did your tests pass on Travis? If they didn't, fix any errors.
6. Did your tests pass on Azure Pipelines? If they didn't, fix any errors.
.. _ask for help:

View file

@ -2,7 +2,7 @@
# Cross-compile the Certbot snap from local sources for the specified architecture,
# and install it if this architecture is also the the current machine one.
# This script is designed for CI tests purpose.
# Usage: build_and_install.sh [amd64,arm64,armhf]
# Usage: build.sh [amd64,arm64,armhf]
set -ex
SNAP_ARCH=$1
@ -40,7 +40,3 @@ docker run \
-e "PIP_EXTRA_INDEX_URL=http://localhost:8080" \
"adferrand/snapcraft:${DOCKER_ARCH}-stable" \
snapcraft
if [[ "$(arch)" == "${QEMU_ARCH}" ]]; then
sudo snap install --dangerous --classic *.snap
fi

View file

@ -63,10 +63,6 @@ parser.add_argument('aws_profile',
parser.add_argument('test_script',
default='test_letsencrypt_auto_certonly_standalone.sh',
help='path of bash script in to deploy and run')
#parser.add_argument('--script_args',
# nargs='+',
# help='space-delimited list of arguments to pass to the bash test script',
# required=False)
parser.add_argument('--repo',
default='https://github.com/letsencrypt/letsencrypt.git',
help='certbot git repo to use')
@ -238,20 +234,20 @@ def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20
# Fabric Routines
#-------------------------------------------------------------------------------
def local_git_clone(local_cxn, repo_url):
"clones master of repo_url"
"""clones master of repo_url"""
local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
local_cxn.local('cd %s && git clone %s letsencrypt'% (LOGDIR, repo_url))
local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt'% LOGDIR)
def local_git_branch(local_cxn, repo_url, branch_name):
"clones branch <branch_name> of repo_url"
"""clones branch <branch_name> of repo_url"""
local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
local_cxn.local('cd %s && git clone %s letsencrypt --branch %s --single-branch'%
(LOGDIR, repo_url, branch_name))
local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR)
def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True):
"clones specified pull request from repo_url and optionally merges into master"
"""clones specified pull request from repo_url and optionally merges into master"""
local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
local_cxn.local('cd %s && git clone %s letsencrypt' % (LOGDIR, repo_url))
local_cxn.local('cd %s && cd letsencrypt && '
@ -264,20 +260,20 @@ def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True):
local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR)
def local_repo_to_remote(cxn):
"copies local tarball of repo to remote"
"""copies local tarball of repo to remote"""
filename = 'le.tar.gz'
local_path = os.path.join(LOGDIR, filename)
cxn.put(local=local_path, remote='')
cxn.run('tar xzf %s' % filename)
def local_repo_clean(local_cxn):
"delete tarball"
"""delete tarball"""
filename = 'le.tar.gz'
local_path = os.path.join(LOGDIR, filename)
local_cxn.local('rm %s' % local_path)
def deploy_script(cxn, scriptpath, *args):
"copies to remote and executes local script"
"""copies to remote and executes local script"""
cxn.put(local=scriptpath, remote='', preserve_mode=True)
scriptfile = os.path.split(scriptpath)[1]
args_str = ' '.join(args)
@ -425,23 +421,23 @@ def main():
print("Making local dir for test repo and logs: %s"%LOGDIR)
local_cxn.local('mkdir %s'%LOGDIR)
# figure out what git object to test and locally create it in LOGDIR
print("Making local git repo")
try:
# figure out what git object to test and locally create it in LOGDIR
print("Making local git repo")
if cl_args.pull_request != '~':
print('Testing PR %s '%cl_args.pull_request,
print('Testing PR %s ' % cl_args.pull_request,
"MERGING into master" if cl_args.merge_master else "")
local_git_PR(local_cxn, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
elif cl_args.branch != '~':
print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
print('Testing branch %s of %s' % (cl_args.branch, cl_args.repo))
local_git_branch(local_cxn, cl_args.repo, cl_args.branch)
else:
print('Testing master of %s'%cl_args.repo)
print('Testing current branch of %s' % cl_args.repo)
local_git_clone(local_cxn, cl_args.repo)
except BaseException:
print("FAIL: trouble with git repo")
traceback.print_exc()
exit()
exit(1)
# Set up EC2 instances

View file

@ -1,10 +0,0 @@
#!/bin/bash -ex
#
# Preps the test farm tests to be run in Travis.
if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
echo This script must be run in Travis on a non-pull request build
exit 1
fi
openssl aes-256-cbc -K "${encrypted_9a387195a62e_key}" -iv "${encrypted_9a387195a62e_iv}" -in travis-test-farm.pem.enc -out travis-test-farm.pem -d

View file

@ -59,9 +59,9 @@ def set_up():
command = set_up_command(config_dir, logs_dir, work_dir, nginx_dir)
dirs = [logs_dir, config_dir, work_dir]
# Travis and Circle CI set CI to true so we
# will always test Nginx's lock during CI
if os.environ.get('CI') == 'true' or util.exe_exists('nginx'):
# If Nginx is installed, do the test, otherwise skip it.
# Issue https://github.com/certbot/certbot/issues/8121 tracks the work to remove this control.
if util.exe_exists('nginx'):
dirs.append(nginx_dir)
else:
logger.warning('Skipping Nginx lock tests')

View file

@ -57,7 +57,7 @@ def validate_scripts_content(repo_path, temp_cwd):
'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd)
# Compare file against current version in the target branch
branch = os.environ.get('TRAVIS_BRANCH', 'master')
branch = os.environ.get('TARGET_BRANCH', 'master')
url = (
'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto'
.format(branch))

View file

@ -12,7 +12,6 @@ ndg-httpsclient==0.3.2
ply==3.4
pyasn1==0.1.9
pycparser==2.14
pyOpenSSL==0.13.1
pyRFC3339==1.0
python-augeas==0.5.0
oauth2client==4.0.0
@ -49,6 +48,7 @@ requests[security]==2.6.0
# Ubuntu Xenial constraints
ConfigArgParse==0.10.0
pyOpenSSL==0.15.1
funcsigs==0.4
zope.hookable==4.0.4

36
tools/run_oldest_tests.sh Executable file
View file

@ -0,0 +1,36 @@
#!/bin/bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
pushd "${DIR}/../"
function cleanup() {
rm -f "${DOCKERFILE}"
popd
}
trap cleanup EXIT
DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX)
cat << "EOF" >> "${DOCKERFILE}"
FROM ubuntu:16.04
COPY pipstrap.py /tmp/pipstrap.py
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
python-dev python-pip python-setuptools \
gcc libaugeas0 libssl-dev libffi-dev \
git ca-certificates nginx-light openssl curl \
&& curl -fsSL https://get.docker.com | bash /dev/stdin \
&& python /tmp/pipstrap.py \
&& python -m pip install tox \
&& rm -rf /var/lib/apt/lists/*
EOF
docker build -f "${DOCKERFILE}" -t oldest-worker ./letsencrypt-auto-source/pieces
docker run --rm --network=host -w "${PWD}" \
-v /var/run/docker.sock:/var/run/docker.sock \
-v "${PWD}:${PWD}" -v /tmp:/tmp \
-e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \
oldest-worker python -m tox

77
tox.ini
View file

@ -165,12 +165,13 @@ commands =
[testenv:modification]
commands =
python {toxinidir}/tests/modification-check.py
passenv = TARGET_BRANCH
[testenv:apache_compat]
commands =
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
docker run --rm -it apache-compat -c apache.tar.gz -vvvv
docker run --rm -t apache-compat -c apache.tar.gz -vvvv
whitelist_externals =
docker
passenv =
@ -180,7 +181,7 @@ passenv =
commands =
docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie
docker run --rm -t nginx-compat -c nginx.tar.gz -vv -aie
whitelist_externals =
docker
passenv =
@ -191,19 +192,19 @@ passenv =
commands =
python {toxinidir}/tests/modification-check.py
docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source
docker run --rm -t -i lea
docker run --rm -t lea
whitelist_externals =
docker
passenv =
DOCKER_*
TRAVIS_BRANCH
TARGET_BRANCH
[testenv:le_auto_jessie]
# At the moment, this tests under Python 2.7 only, as only that version is
# readily available on the Wheezy Docker image.
commands =
docker build -f letsencrypt-auto-source/Dockerfile.jessie -t lea letsencrypt-auto-source
docker run --rm -t -i lea
docker run --rm -t lea
whitelist_externals =
docker
passenv = DOCKER_*
@ -213,7 +214,7 @@ passenv = DOCKER_*
# readily available on the CentOS 6 Docker image.
commands =
docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=centos -t lea letsencrypt-auto-source
docker run --rm -t -i lea
docker run --rm -t lea
whitelist_externals =
docker
passenv = DOCKER_*
@ -223,7 +224,7 @@ passenv = DOCKER_*
# readily available on the Oracle Linux 6 Docker image.
commands =
docker build -f letsencrypt-auto-source/Dockerfile.redhat6 --build-arg REDHAT_DIST_FLAVOR=oraclelinux -t lea letsencrypt-auto-source
docker run --rm -t -i lea
docker run --rm -t lea
whitelist_externals =
docker
passenv = DOCKER_*
@ -283,49 +284,37 @@ commands =
passenv = DOCKER_*
setenv = {[testenv:py27-oldest]setenv}
[testenv:travis-test-farm-tests-base]
[testenv:test-farm-tests-base]
changedir = tests/letstest
commands =
./travis-setup.sh
deps = -rtests/letstest/requirements.txt
passenv =
AWS_*
TRAVIS_*
encrypted_*
setenv = AWS_DEFAULT_REGION=us-east-1
[testenv:travis-test-farm-apache2]
changedir = {[testenv:travis-test-farm-tests-base]changedir}
commands =
{[testenv:travis-test-farm-tests-base]commands}
python multitester.py apache2_targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_apache2.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH}
deps = {[testenv:travis-test-farm-tests-base]deps}
passenv = {[testenv:travis-test-farm-tests-base]passenv}
setenv = {[testenv:travis-test-farm-tests-base]setenv}
[testenv:test-farm-apache2]
changedir = {[testenv:test-farm-tests-base]changedir}
commands = python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
deps = {[testenv:test-farm-tests-base]deps}
passenv = {[testenv:test-farm-tests-base]passenv}
setenv = {[testenv:test-farm-tests-base]setenv}
[testenv:travis-test-farm-leauto-upgrades]
changedir = {[testenv:travis-test-farm-tests-base]changedir}
commands =
{[testenv:travis-test-farm-tests-base]commands}
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH}
deps = {[testenv:travis-test-farm-tests-base]deps}
passenv = {[testenv:travis-test-farm-tests-base]passenv}
setenv = {[testenv:travis-test-farm-tests-base]setenv}
[testenv:test-farm-leauto-upgrades]
changedir = {[testenv:test-farm-tests-base]changedir}
commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir}
deps = {[testenv:test-farm-tests-base]deps}
passenv = {[testenv:test-farm-tests-base]passenv}
setenv = {[testenv:test-farm-tests-base]setenv}
[testenv:travis-test-farm-certonly-standalone]
changedir = {[testenv:travis-test-farm-tests-base]changedir}
commands =
{[testenv:travis-test-farm-tests-base]commands}
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH}
deps = {[testenv:travis-test-farm-tests-base]deps}
passenv = {[testenv:travis-test-farm-tests-base]passenv}
setenv = {[testenv:travis-test-farm-tests-base]setenv}
[testenv:test-farm-certonly-standalone]
changedir = {[testenv:test-farm-tests-base]changedir}
commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir}
deps = {[testenv:test-farm-tests-base]deps}
passenv = {[testenv:test-farm-tests-base]passenv}
setenv = {[testenv:test-farm-tests-base]setenv}
[testenv:travis-test-farm-sdists]
changedir = {[testenv:travis-test-farm-tests-base]changedir}
commands =
{[testenv:travis-test-farm-tests-base]commands}
python multitester.py targets.yaml travis-test-farm.pem SET_BY_ENV scripts/test_sdists.sh --repo {env:TRAVIS_BUILD_DIR} --branch {env:TRAVIS_BRANCH}
deps = {[testenv:travis-test-farm-tests-base]deps}
passenv = {[testenv:travis-test-farm-tests-base]passenv}
setenv = {[testenv:travis-test-farm-tests-base]setenv}
[testenv:test-farm-sdists]
changedir = {[testenv:test-farm-tests-base]changedir}
commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
deps = {[testenv:test-farm-tests-base]deps}
passenv = {[testenv:test-farm-tests-base]passenv}
setenv = {[testenv:test-farm-tests-base]setenv}