From 0f6486ec7fdc92b1c77d4d4701641a52ce941792 Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Wed, 27 Jun 2018 15:38:45 +0100 Subject: [PATCH 01/33] Initial commit * Add renewal timer * Install libaugeas0 in python-augeas part build This part needs libaugeas0 to build. * Bump to 0.26.1 * Always act directly on upstream master I want to keep this always working, so move to master. We can reintroduce upstream stable releases when we are ready for general use. Closes: #5 That particular issue seems to no longer happen. Presumably something changed in upstream git or in PyPI. If it happens again, hopefully I'll have CI against upstream master up by then and I'll be able to pin it down. * Add empty Travis build * Add Travis automatic snap edge publication * Add integration test This uses upstream's test suite from their source tree to check the built snap to make sure it behaves as expected, before attempting upload to the store. * Point Augeas to its lens library Augeas defaults to looking in /usr/share/augeas/lenses, which in a snap isn't found at this path, but inside $SNAP. So set AUGEAS_LENS_LIB to where the lenses can be found within the snap. This fixes the Apache plugin that uses Augeas. --- snap/.snapcraft/travis_snapcraft.cfg | Bin 0 -> 2448 bytes snap/.travis.yml | 11 +++++ snap/snapcraft.yaml | 67 +++++++++++++++++++++++++++ snap/test.bash | 34 ++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 snap/.snapcraft/travis_snapcraft.cfg create mode 100644 snap/.travis.yml create mode 100644 snap/snapcraft.yaml create mode 100644 snap/test.bash diff --git a/snap/.snapcraft/travis_snapcraft.cfg b/snap/.snapcraft/travis_snapcraft.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d54f41155fe3ff5850721619d5e34d6b0ad85afc GIT binary patch literal 2448 zcmV;B32*ioqW&}!9P1blSSTry(okTVjYkP?5Wp~YL%*7glNCXMj}n<x)aFiR0p~b(~Lr-2a%WaDxNW za?T9WiJ>=#5)Tv?W^tOlDqEo4dg!o4Bsp-DcwCs0&#uf66A*9Je9ZncL}uaGpw%iI z2^X^wfNaDms2C;|Gsk-%Hy$nN9+sqSkSmk}rD&I;?a9O_C~o%wWi5e@C%g!8Ff0Bo z#hU>DS&>7I+i?1)LQHx?n5)hVWPMPL^=f_lwmisSuxt7`qG5#~$3Uk0B?_#3Gu|$y zA?h1^nZ=AQf>X3UNbKbly&DRX$~<*(t=e6sVrUKEwRLsOy0h~*>U?9KuOhThsG3Dk z(m}Qa?^ew_=*BVFBySco&{Qr>6mYKiP~W6aPn*$ibOLO*aW4Rb! zH1`)DqVjauEA#DjT0+#QNEN%!10i!VGh9)($2gq%!e+Ak$#MTRbA)JsJNS6 z!dQ7w;L61&A=I{~I3fsbl^4(<(EB6*R}rx{NPv66sFYV&2?zi-rY*Iti%UxnS%Wla zJWzamc&fCV0LSWNi6rm!32AazC_g@@tnoeO4;Uyi4**X&ni0 z!@|b)O0QMjb!?CV>7Br$X{<0f!fZw1WEQU%QKP9wy#PQmV=aWlb)XckcsZ!MKg9b_ zrgzTjPBNzx@IepXqUjwLL9f2INb(l@i+)obL=^nrbGLnA>xBv%{yO~dN9p)ukkLz= zt#h^pA;p(uCihtoFaj{S8U}H*{sV}>za$3iW4Zi>M`Q(C%v4r+GK-usI-d&3K+OSs z)!{?S*`>5QWVHV-KoT=LC!ycSAQ_cM)QtU3jwm=6fO`d;YD7q8Yo#EacGsgGwYlGh z|4FDD2m>FB3d;ptjq{g5Pxkw0=Wy+vR>`f17L@CBG5kdq~8Lo*>RcGMRt7#Q237y zD|mr7V-~-TBv-?Lty9i9Ro10M5=P{L4)D^%l?#m4m?<*+2W-ExQ3~Gq81sd66B^Dt z7o4{k%oP}2OasXZvGlIn8f`qzH% zs0gJIA@_`Tpsl+U$!twrGg*qk z`oNJOxdC$PCyDFz;hynHJkR#j>265mFUgsaXWAzgEn42N1AQ0o;Qm!b{gb5CInGAY zj8Tm5S-vo>ss@(q0%BZ9B_~6e-wh|(Bol-32pwUY)0hG=#(fh^>$L&btbk7t_wW0t zc6B^FgrN8AFj+}N!SYWDMTs`?m;W+Gc5*BYE&4JK2?2~Rakjy113&GLm2|t{jvC6G zqYB_{0A#v5@7_GbjBQ)KGhzTRFsUpa{NyhG^krSR%Z8edP}_AschHegr+w_A0=)GQ zQNgX?psx`(HhfK(J93l`&f49!HYE|;3Y+wq43WIJ(=8Ec_C8^b81a*9LeT< zTfIiq%Eh?|-ie{5ozKc>!@N%t@_cLih>6U2Hk-wCg}KlGwU8O^jP*nuOLdlal@;Z4 z6=o9ooLrc)uv4ilB-_x_6>W3-#uYK>cHXg|iNMl$f+5mKPOlpWGxC6SyAn6vl>M`8 zyb_WU4C}T%bZ|kh7Pu5lRqN{YICM|?O=)+WNe~&XMfRH4ORFNO0<^CoGnGh)A;2P% z=-@5~;f(g3ml!7N8#Kw2@GgJ7gbOI4#JIvLS0F@FIL|R|yvdl9XN=z3U%IrfbGRN& zSm$m$BnSb)pOEN?vSf%%Lxd7FDuWH_s8{^fvgglT+Nn&k^B)$Tf+5LzOG8*J(e{1> zcPX;+Uaai526@f8zA2bdclu)#_jDQ-Hv<;=+OTi9IpJiYEQ^|%zWprvV@sXeCgs*K zX$>pCL(*~A0CH{2Z$9kWyI{1ZB`}q<>kHnd0=5U=lHhe0HZ+Mvhb$zYAkqPcWYc4@ zO#sdX4!Xus6raVi#} z?7SbP&}-?}8_+?B;brcRR7$-svQ8Qe*pC!LJJ--7{M?z?id&iI+N95Ygq4rGaL}jI zzg@1O@CVI+QQD@oEnQE21@d=Aocw!K-9AL3qb_btf0BilGrk8ud_6$QmROJ|WCXx` zU+(RdGK|dkFgNsT(#n`8tvrkF?N>av0Dv`cl(ejja|i3_H1`oa_5J!En2hP6+YL!M zIBhzz;jj3PDtKgr+I^_Kc8W!2*+E_V)T*OBqww_Wid151=!vll3u)HxK(#&)f&YYYsC>SyoUOOF(0Z{V(qivF0#(W6oYjMvR6$S|`s9Jl zS%Ztzb~ouF4Q>Ow7Q3#JZiI&!FXfNmx(1kv!w OnZUwb{V>Dqu1Ux#JH%H2 literal 0 HcmV?d00001 diff --git a/snap/.travis.yml b/snap/.travis.yml new file mode 100644 index 000000000..603609b6d --- /dev/null +++ b/snap/.travis.yml @@ -0,0 +1,11 @@ +language: bash +script: docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && cd $(pwd) && snapcraft" && sudo apt-get -y install snapd && sudo snap install --dangerous --classic *.snap && sudo bash test.bash +sudo: required +services: +- docker +deploy: + 'on': + branch: master + provider: script + script: openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d && docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + skip_cleanup: true diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 000000000..46425f182 --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,67 @@ +name: certbot +version: script +version-script: cd parts/certbot/src && git describe|sed s/^v// +summary: Automatically configure HTTPS using Let's Encrypt +description: | + The objective of Certbot, Let's Encrypt, and the ACME (Automated + Certificate Management Environment) protocol is to make it possible + to set up an HTTPS server and have it automatically obtain a + browser-trusted certificate, without any human intervention. This is + accomplished by running a certificate management agent on the web + server. + + This agent is used to: + - Automatically prove to the Let's Encrypt CA that you control the website + - Obtain a browser-trusted certificate and set it up on your web server + - Keep track of when your certificate is going to expire, and renew it + - Help you revoke the certificate if that ever becomes necessary. +confinement: classic +grade: devel + +apps: + certbot: + command: certbot + environment: + PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" + AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + renew: + command: certbot -q renew + daemon: oneshot + environment: + PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" + AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + passthrough: + # Run approximately twice a day with randomization + timer: 00:00~24:00/2 + +parts: + python-augeas: + plugin: python + source: git://github.com/basak/python-augeas + source-branch: snap + python-version: python2 + build-packages: [libaugeas0] + certbot: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + python-version: python2 + after: [python-augeas] + certbot-nginx: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + source-subdir: certbot-nginx + build-packages: [libaugeas-dev] + stage-packages: [libaugeas0] + after: [certbot] + python-version: python2 + certbot-apache: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + source-subdir: certbot-apache + build-packages: [libaugeas-dev] + stage-packages: [libaugeas0] + after: [certbot] + python-version: python2 diff --git a/snap/test.bash b/snap/test.bash new file mode 100644 index 000000000..e723b67bf --- /dev/null +++ b/snap/test.bash @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +apt-get -y install lsb-release net-tools wget python nginx + +wget https://github.com/docker/compose/releases/download/1.15.0-rc1/docker-compose-Linux-x86_64 -O /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose + +cat << EOF > /usr/local/bin/coverage +#!/bin/bash -xe + +if [ "\$1" != "run" ]; then + exit 0; +fi + +"\${@:7}" +EOF +chmod +x /usr/local/bin/coverage + +certbot_version=$(certbot --version 2>&1 | grep "^certbot" | cut -d " " -f 2) + +cd parts/certbot/src + +tests/boulder-fetch.sh +until curl http://localhost:4000/directory 2>/dev/null; do + echo waiting for boulder + sleep 1 +done +# Not needed under Travis Trusty? +#sed -i "s/'1.3.6.1.5.5.7.1.24'/-e '1.3.6.1.5.5.7.1.24' -e 'status_request'/g" tests/certbot-boulder-integration.sh +tests/boulder-integration.sh + +echo "Success!" From d63be466a85ed4633a24844b43ce78857c524053 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 3 May 2019 10:15:48 +0200 Subject: [PATCH 02/33] Various optimizations part 1 * Configure for python3 * Update tests * Use appropriate virtualenv * Install nginx for the integration tests * Try use LD_LIBRARY_PATH to find augeas shared library in snap when python-augeas is invoked * Update travis to use build-in setup capabilities * Update .travis.yml * Add acme build * Update tests * Try more recent dist * Update command * Clean tests * Add back augeas * Add env * Revert to last working snapcraft config * Add a gitignore * Reintegrate acme. Declare augeas in certbot parts * Use release version of certbot * Try new approach * Fix config * Directly install version of python-augeas from pypi * Restart from basic * Clone only once certbot repository. Use pinned versions of dependencies from certbot-auto. * Try relatively to source * Use snapcraft env variables * Strip hashes * Fix path * Redefine path * Continue to prepare the runtime * Fix command line * Update .travis.yml * Add back certbot-apache * Update snapcraft.yaml * Build snap against the latest release of certbot --- snap/.gitignore | 8 ++++++++ snap/.travis.yml | 24 +++++++++++++++++++--- snap/get_latest_version.py | 21 +++++++++++++++++++ snap/snapcraft.yaml | 42 ++++++++++++++++++-------------------- snap/test.bash | 34 ------------------------------ snap/test.sh | 10 +++++++++ 6 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 snap/.gitignore create mode 100644 snap/get_latest_version.py delete mode 100644 snap/test.bash create mode 100644 snap/test.sh diff --git a/snap/.gitignore b/snap/.gitignore new file mode 100644 index 000000000..2a339aeaa --- /dev/null +++ b/snap/.gitignore @@ -0,0 +1,8 @@ +.snapcraft +parts +prime +stage +*.snap +certbot +venv +.idea diff --git a/snap/.travis.yml b/snap/.travis.yml index 603609b6d..1abe677cd 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,11 +1,29 @@ +dist: xenial language: bash -script: docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && cd $(pwd) && snapcraft" && sudo apt-get -y install snapd && sudo snap install --dangerous --classic *.snap && sudo bash test.bash +install: + - EXPORT CERTBOT_VERSION=$(python3 get_latest_version.py) + - git clone https://github.com/certbot/certbot.git certbot --branch ${CERTBOT_VERSION} + - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt +script: + - docker run -v $(pwd):$(pwd) -t -e CERTBOT_VERSION=${CERTBOT_VERSION} snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" + - sudo snap install --dangerous --classic *.snap + - sudo bash test.sh sudo: required +addons: + apt: + packages: + - snapd + - nginx-light + - python3-venv + - python3-requests + - python3-packaging services: -- docker + - docker deploy: 'on': branch: master provider: script - script: openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d && docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + script: | + openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d + docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" skip_cleanup: true diff --git a/snap/get_latest_version.py b/snap/get_latest_version.py new file mode 100644 index 000000000..b9a57e2aa --- /dev/null +++ b/snap/get_latest_version.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import sys + +import requests +from packaging import version + + +def main(): + result = requests.get('https://api.github.com/repos/certbot/certbot/tags') + result.raise_for_status() + + tags = [version.parse(entry['name'].replace('v', '')) for entry in result.json()] + tags.sort() + + latest_tag = 'v{0}'.format(tags[-1]) + + sys.stdout.write(latest_tag) + + +if __name__ == '__main__': + main() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 46425f182..0a3772ad6 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -2,6 +2,7 @@ name: certbot version: script version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt +base: core18 description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated Certificate Management Environment) protocol is to make it possible @@ -23,45 +24,42 @@ apps: command: certbot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist" + LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" renew: command: certbot -q renew daemon: oneshot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" passthrough: # Run approximately twice a day with randomization timer: 00:00~24:00/2 parts: - python-augeas: + acme: plugin: python - source: git://github.com/basak/python-augeas - source-branch: snap - python-version: python2 - build-packages: [libaugeas0] + source: certbot + source-subdir: acme + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 certbot: plugin: python - source: git://github.com/certbot/certbot - source-branch: master - python-version: python2 - after: [python-augeas] - certbot-nginx: - plugin: python - source: git://github.com/certbot/certbot - source-branch: master - source-subdir: certbot-nginx - build-packages: [libaugeas-dev] - stage-packages: [libaugeas0] - after: [certbot] - python-version: python2 + source: certbot + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 + python-packages: + - acme + after: [acme] certbot-apache: plugin: python - source: git://github.com/certbot/certbot - source-branch: master + source: certbot source-subdir: certbot-apache + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 + python-packages: + - certbot build-packages: [libaugeas-dev] stage-packages: [libaugeas0] after: [certbot] - python-version: python2 diff --git a/snap/test.bash b/snap/test.bash deleted file mode 100644 index e723b67bf..000000000 --- a/snap/test.bash +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -ex - -apt-get -y install lsb-release net-tools wget python nginx - -wget https://github.com/docker/compose/releases/download/1.15.0-rc1/docker-compose-Linux-x86_64 -O /usr/local/bin/docker-compose -chmod +x /usr/local/bin/docker-compose - -cat << EOF > /usr/local/bin/coverage -#!/bin/bash -xe - -if [ "\$1" != "run" ]; then - exit 0; -fi - -"\${@:7}" -EOF -chmod +x /usr/local/bin/coverage - -certbot_version=$(certbot --version 2>&1 | grep "^certbot" | cut -d " " -f 2) - -cd parts/certbot/src - -tests/boulder-fetch.sh -until curl http://localhost:4000/directory 2>/dev/null; do - echo waiting for boulder - sleep 1 -done -# Not needed under Travis Trusty? -#sed -i "s/'1.3.6.1.5.5.7.1.24'/-e '1.3.6.1.5.5.7.1.24' -e 'status_request'/g" tests/certbot-boulder-integration.sh -tests/boulder-integration.sh - -echo "Success!" diff --git a/snap/test.sh b/snap/test.sh new file mode 100644 index 000000000..cf2418448 --- /dev/null +++ b/snap/test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +cd certbot + +python3 -m venv venv +. venv/bin/activate +pip install -e certbot-ci + +pytest certbot-ci/certbot_integration_tests/certbot_tests --numprocesses 4 --acme-server=pebble From e0b72d9a62db7260ff5114422f839dd80e40c5ae Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Tue, 20 Aug 2019 14:47:23 +0100 Subject: [PATCH 03/33] Travis improvements * Add Travis notifications * Adjust automatic snap deployment configuration Travis now has a documented[1] "snap" provider and the previous experimental mechanism seems to have stopped working, presumably because it was deprecated in favour of this new mechanism. [1] https://docs.travis-ci.com/user/deployment/snaps/ --- snap/.travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/snap/.travis.yml b/snap/.travis.yml index 1abe677cd..5e5b17037 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -22,8 +22,11 @@ services: deploy: 'on': branch: master - provider: script - script: | - openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + provider: snap + snap: certbot_*.snap + channel: edge skip_cleanup: true +notifications: + email: + recipients: [robie.basak@canonical.com] + on_failure: change From 2c622944ddaa8c4641df33bb2386839a8d74e9ef Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 21 Aug 2019 23:45:04 +0200 Subject: [PATCH 04/33] Various optimizations part 2 * Revert logic of building against a tag * Fix schema, add nginx * Update snapcraft.yaml * Update snapcraft.yaml * Update snapcraft.yaml * Update test.sh * Update test.sh * Update test.sh * Update test.sh * Config tests * Add an apache test * Relaunch CI * Clean config * Install venv * Decompose steps * Update test.sh * Use virtual environment * Update python-augeas * Add fork python-augeas * Update .travis.yml * Exclusion rule * Try with after --- snap/.travis.yml | 23 +++++++---------------- snap/get_latest_version.py | 21 --------------------- snap/snapcraft.yaml | 19 +++++++++++++------ snap/test.sh | 11 ++++++----- 4 files changed, 26 insertions(+), 48 deletions(-) delete mode 100644 snap/get_latest_version.py diff --git a/snap/.travis.yml b/snap/.travis.yml index 5e5b17037..3c934b7bd 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,24 +1,15 @@ -dist: xenial -language: bash +language: python +before_install: + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light snapd python3-venv + - sudo -E /etc/init.d/nginx stop install: - - EXPORT CERTBOT_VERSION=$(python3 get_latest_version.py) - - git clone https://github.com/certbot/certbot.git certbot --branch ${CERTBOT_VERSION} + - git clone https://github.com/certbot/certbot.git certbot --branch master - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt +before_script: + - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" script: - - docker run -v $(pwd):$(pwd) -t -e CERTBOT_VERSION=${CERTBOT_VERSION} snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" - sudo snap install --dangerous --classic *.snap - sudo bash test.sh -sudo: required -addons: - apt: - packages: - - snapd - - nginx-light - - python3-venv - - python3-requests - - python3-packaging -services: - - docker deploy: 'on': branch: master diff --git a/snap/get_latest_version.py b/snap/get_latest_version.py deleted file mode 100644 index b9a57e2aa..000000000 --- a/snap/get_latest_version.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -import sys - -import requests -from packaging import version - - -def main(): - result = requests.get('https://api.github.com/repos/certbot/certbot/tags') - result.raise_for_status() - - tags = [version.parse(entry['name'].replace('v', '')) for entry in result.json()] - tags.sort() - - latest_tag = 'v{0}'.format(tags[-1]) - - sys.stdout.write(latest_tag) - - -if __name__ == '__main__': - main() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 0a3772ad6..b7bfab9ca 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -2,7 +2,6 @@ name: certbot version: script version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt -base: core18 description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated Certificate Management Environment) protocol is to make it possible @@ -38,6 +37,12 @@ apps: timer: 00:00~24:00/2 parts: + python-augeas: + plugin: python + source: git://github.com/basak/python-augeas + source-branch: snap + python-version: python3 + build-packages: [libaugeas-dev] acme: plugin: python source: certbot @@ -49,8 +54,6 @@ parts: source: certbot constraints: $SNAPCRAFT_PART_SRC/constraints.txt python-version: python3 - python-packages: - - acme after: [acme] certbot-apache: plugin: python @@ -58,8 +61,12 @@ parts: source-subdir: certbot-apache constraints: $SNAPCRAFT_PART_SRC/constraints.txt python-version: python3 - python-packages: - - certbot - build-packages: [libaugeas-dev] + after: [python-augeas, certbot] stage-packages: [libaugeas0] + certbot-nginx: + plugin: python + source: certbot + source-subdir: certbot-nginx + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 after: [certbot] diff --git a/snap/test.sh b/snap/test.sh index cf2418448..e89743d47 100644 --- a/snap/test.sh +++ b/snap/test.sh @@ -1,10 +1,11 @@ #!/bin/bash set -ex -cd certbot - python3 -m venv venv -. venv/bin/activate -pip install -e certbot-ci +venv/bin/python -m pip install -e certbot/certbot-ci +venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 -pytest certbot-ci/certbot_integration_tests/certbot_tests --numprocesses 4 --acme-server=pebble +# DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! +if [ -n "$TRAVIS" ]; then + venv/bin/python certbot/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules +fi From b69f5588f4c05d184a5d1e983ffa7898affa0e4d Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Tue, 3 Sep 2019 15:41:11 +0100 Subject: [PATCH 05/33] Continued improvements * Remove legacy Store upload credentials These have not been needed since 5d7969a. * Work around dev part dependency failure case Get pip to install certbot from its VCS repository directly during the build of the nginx and apache plugin parts. This works around issue #12 when it affects the interaction between the apache or nginx plugin and certbot itself. It does not work around the case where the same problem occurs in the interaction between certbot and acme. This looks harder to work around because pip's VCS URL handling doesn't appear to include a facility to install from a subdirectory of a git repository and this is where the acme source is located. * Switch to using lxd for the snapcraft run The Docker image is 16.04 only. Before we can switch to 18.04, we need to remove Docker, which means using the snapcraft snap using Travis' new snap support, and then using the lxd functionality in snapcraft so that it is enabled to build in the appropriate environment. * Switch build to use core 18 Fixes: #14 * Move constraints into a list This seems to be a requirement either either newer snapcraft, snapcraft in lxd or in the move to core18 (it isn't clear to me which). This fixes the error: Failed to load plugin: properties failed to load for certbot: The 'constraints' property does not match the required schema: '$SNAPCRAFT_PART_SRC/constraints.txt' is not of type 'array' * version-script -> snapcraftctl set-version The use of version-script seems to break with either newer snapcraft, snapcraft with lxd or core18 (it's not clear to me which). The breakage is related to "parts/certbot/src" not being found. This can be fixed with $SNAPCRAFT_PART_SRC, but this doesn't seem to be defined at "version-script time". However https://snapcraft.io/docs/deprecation-notices/dn10 deprecates the use of version-script, and if we convert to the recommended new way, then we use override-pull instead and $SNAPCRAFT_PART_SRC is defined there, so this conveniently fixes both problems at once. * Do not explicitly install snapd Since this is now handled by the Travis addon, we do not need to do it explicitly. --- snap/.snapcraft/travis_snapcraft.cfg | Bin 2448 -> 0 bytes snap/.travis.yml | 14 ++++++++++++-- snap/snapcraft.yaml | 17 +++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) delete mode 100644 snap/.snapcraft/travis_snapcraft.cfg diff --git a/snap/.snapcraft/travis_snapcraft.cfg b/snap/.snapcraft/travis_snapcraft.cfg deleted file mode 100644 index d54f41155fe3ff5850721619d5e34d6b0ad85afc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2448 zcmV;B32*ioqW&}!9P1blSSTry(okTVjYkP?5Wp~YL%*7glNCXMj}n<x)aFiR0p~b(~Lr-2a%WaDxNW za?T9WiJ>=#5)Tv?W^tOlDqEo4dg!o4Bsp-DcwCs0&#uf66A*9Je9ZncL}uaGpw%iI z2^X^wfNaDms2C;|Gsk-%Hy$nN9+sqSkSmk}rD&I;?a9O_C~o%wWi5e@C%g!8Ff0Bo z#hU>DS&>7I+i?1)LQHx?n5)hVWPMPL^=f_lwmisSuxt7`qG5#~$3Uk0B?_#3Gu|$y zA?h1^nZ=AQf>X3UNbKbly&DRX$~<*(t=e6sVrUKEwRLsOy0h~*>U?9KuOhThsG3Dk z(m}Qa?^ew_=*BVFBySco&{Qr>6mYKiP~W6aPn*$ibOLO*aW4Rb! zH1`)DqVjauEA#DjT0+#QNEN%!10i!VGh9)($2gq%!e+Ak$#MTRbA)JsJNS6 z!dQ7w;L61&A=I{~I3fsbl^4(<(EB6*R}rx{NPv66sFYV&2?zi-rY*Iti%UxnS%Wla zJWzamc&fCV0LSWNi6rm!32AazC_g@@tnoeO4;Uyi4**X&ni0 z!@|b)O0QMjb!?CV>7Br$X{<0f!fZw1WEQU%QKP9wy#PQmV=aWlb)XckcsZ!MKg9b_ zrgzTjPBNzx@IepXqUjwLL9f2INb(l@i+)obL=^nrbGLnA>xBv%{yO~dN9p)ukkLz= zt#h^pA;p(uCihtoFaj{S8U}H*{sV}>za$3iW4Zi>M`Q(C%v4r+GK-usI-d&3K+OSs z)!{?S*`>5QWVHV-KoT=LC!ycSAQ_cM)QtU3jwm=6fO`d;YD7q8Yo#EacGsgGwYlGh z|4FDD2m>FB3d;ptjq{g5Pxkw0=Wy+vR>`f17L@CBG5kdq~8Lo*>RcGMRt7#Q237y zD|mr7V-~-TBv-?Lty9i9Ro10M5=P{L4)D^%l?#m4m?<*+2W-ExQ3~Gq81sd66B^Dt z7o4{k%oP}2OasXZvGlIn8f`qzH% zs0gJIA@_`Tpsl+U$!twrGg*qk z`oNJOxdC$PCyDFz;hynHJkR#j>265mFUgsaXWAzgEn42N1AQ0o;Qm!b{gb5CInGAY zj8Tm5S-vo>ss@(q0%BZ9B_~6e-wh|(Bol-32pwUY)0hG=#(fh^>$L&btbk7t_wW0t zc6B^FgrN8AFj+}N!SYWDMTs`?m;W+Gc5*BYE&4JK2?2~Rakjy113&GLm2|t{jvC6G zqYB_{0A#v5@7_GbjBQ)KGhzTRFsUpa{NyhG^krSR%Z8edP}_AschHegr+w_A0=)GQ zQNgX?psx`(HhfK(J93l`&f49!HYE|;3Y+wq43WIJ(=8Ec_C8^b81a*9LeT< zTfIiq%Eh?|-ie{5ozKc>!@N%t@_cLih>6U2Hk-wCg}KlGwU8O^jP*nuOLdlal@;Z4 z6=o9ooLrc)uv4ilB-_x_6>W3-#uYK>cHXg|iNMl$f+5mKPOlpWGxC6SyAn6vl>M`8 zyb_WU4C}T%bZ|kh7Pu5lRqN{YICM|?O=)+WNe~&XMfRH4ORFNO0<^CoGnGh)A;2P% z=-@5~;f(g3ml!7N8#Kw2@GgJ7gbOI4#JIvLS0F@FIL|R|yvdl9XN=z3U%IrfbGRN& zSm$m$BnSb)pOEN?vSf%%Lxd7FDuWH_s8{^fvgglT+Nn&k^B)$Tf+5LzOG8*J(e{1> zcPX;+Uaai526@f8zA2bdclu)#_jDQ-Hv<;=+OTi9IpJiYEQ^|%zWprvV@sXeCgs*K zX$>pCL(*~A0CH{2Z$9kWyI{1ZB`}q<>kHnd0=5U=lHhe0HZ+Mvhb$zYAkqPcWYc4@ zO#sdX4!Xus6raVi#} z?7SbP&}-?}8_+?B;brcRR7$-svQ8Qe*pC!LJJ--7{M?z?id&iI+N95Ygq4rGaL}jI zzg@1O@CVI+QQD@oEnQE21@d=Aocw!K-9AL3qb_btf0BilGrk8ud_6$QmROJ|WCXx` zU+(RdGK|dkFgNsT(#n`8tvrkF?N>av0Dv`cl(ejja|i3_H1`oa_5J!En2hP6+YL!M zIBhzz;jj3PDtKgr+I^_Kc8W!2*+E_V)T*OBqww_Wid151=!vll3u)HxK(#&)f&YYYsC>SyoUOOF(0Z{V(qivF0#(W6oYjMvR6$S|`s9Jl zS%Ztzb~ouF4Q>Ow7Q3#JZiI&!FXfNmx(1kv!w OnZUwb{V>Dqu1Ux#JH%H2 diff --git a/snap/.travis.yml b/snap/.travis.yml index 3c934b7bd..418407558 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,12 +1,15 @@ language: python before_install: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light snapd python3-venv + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light python3-venv - sudo -E /etc/init.d/nginx stop install: + - sudo /snap/bin/lxd.migrate -yes + - sudo /snap/bin/lxd waitready + - sudo /snap/bin/lxd init --auto - git clone https://github.com/certbot/certbot.git certbot --branch master - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt before_script: - - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" + - sudo snapcraft --use-lxd script: - sudo snap install --dangerous --classic *.snap - sudo bash test.sh @@ -21,3 +24,10 @@ notifications: email: recipients: [robie.basak@canonical.com] on_failure: change +addons: + snaps: + - name: snapcraft + channel: stable + classic: true + - name: lxd + channel: stable diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b7bfab9ca..38d32ed0b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,4 @@ name: certbot -version: script -version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated @@ -17,6 +15,8 @@ description: | - Help you revoke the certificate if that ever becomes necessary. confinement: classic grade: devel +base: core18 +adopt-info: certbot apps: certbot: @@ -47,26 +47,31 @@ parts: plugin: python source: certbot source-subdir: acme - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: plugin: python source: certbot - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [acme] + override-pull: | + snapcraftctl pull + snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//` certbot-apache: plugin: python source: certbot source-subdir: certbot-apache - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] + python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 certbot-nginx: plugin: python source: certbot source-subdir: certbot-nginx - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [certbot] + python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 From 193b44a0fa86355941e4f188f1492a713d05bdf6 Mon Sep 17 00:00:00 2001 From: Sergio Schvezov Date: Thu, 9 Jan 2020 12:41:37 -0300 Subject: [PATCH 06/33] Snap plugin * snap: move snapcraft.yaml to snap directory Signed-off-by: Sergio Schvezov * snap: use a local plugin to get around the delivered plugin Add a plugin to the project which behaves as expected until a version of snapcraft satisfies the project needs. Additional snapcraft.yaml changes were made to accommodate for the snap to build. Signed-off-by: Sergio Schvezov * snap: compile pycache in the last step for the last part Signed-off-by: Sergio Schvezov --- snap/snap/plugins/x_python.py | 499 +++++++++++++++++++++++++++++++++ snap/{ => snap}/snapcraft.yaml | 26 +- 2 files changed, 517 insertions(+), 8 deletions(-) create mode 100644 snap/snap/plugins/x_python.py rename snap/{ => snap}/snapcraft.yaml (79%) diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py new file mode 100644 index 000000000..5c8c3b86a --- /dev/null +++ b/snap/snap/plugins/x_python.py @@ -0,0 +1,499 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# This is an almost verbatim copy of what is in the snapcraft +# tree, can be completely removed once the functionality in +# snapcraft is in place. (LP: #1841861) +# +# Copyright (C) 2016-2020 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The python plugin can be used for python 2 or 3 based parts. + +It can be used for python projects where you would want to do: + + - import python modules with a requirements.txt + - build a python project that has a setup.py + - install packages straight from pip + +This plugin uses the common plugin keywords as well as those for "sources". +For more information check the 'plugins' topic for the former and the +'sources' topic for the latter. + +Additionally, this plugin uses the following plugin-specific keywords: + + - requirements: + (list of strings) + List of paths to requirements files. + - constraints: + (list of strings) + List of paths to constraint files. + - process-dependency-links: + (bool; default: false) + Enable the processing of dependency links in pip, which allow one + project to provide places to look for another project + - python-packages: + (list) + A list of dependencies to get from PyPI + - python-version: + (string; default: python3) + The python version to use. Valid options are: python2 and python3 + +If the plugin finds a python interpreter with a basename that matches +`python-version` in the directory on the following fixed path: +`/usr/bin/` then this interpreter would +be preferred instead and no interpreter would be brought in through +`stage-packages` mechanisms. +""" + +import collections +import contextlib +import os +import re +from shutil import which +from textwrap import dedent +from typing import List, Optional, Set + +import requests + +import snapcraft +from snapcraft.common import isurl +from snapcraft.internal import errors, mangling +from snapcraft.internal.errors import SnapcraftPluginCommandError +from snapcraft.plugins import _python + + +class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): + + fmt = "Unsupported python version: {python_version!r}" + + +class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): + + fmt = ( + "Failed to find the referred {plugin_property} file at the given " + "path: {plugin_property_value!r}.\n" + "Check the property and ensure the file exists." + ) + + def __init__(self, *, plugin_property, plugin_property_value): + super().__init__( + plugin_property=plugin_property, plugin_property_value=plugin_property_value + ) + + +class PythonPlugin(snapcraft.BasePlugin): + @classmethod + def schema(cls): + schema = super().schema() + schema["properties"]["requirements"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["constraints"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["python-packages"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["process-dependency-links"] = { + "type": "boolean", + "default": False, + } + schema["properties"]["python-version"] = { + "type": "string", + "default": "python3", + "enum": ["python2", "python3"], + } + schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] + + return schema + + @classmethod + def get_pull_properties(cls): + # Inform Snapcraft of the properties associated with pulling. If these + # change in the YAML Snapcraft will consider the pull step dirty. + return [ + "requirements", + "constraints", + "python-packages", + "process-dependency-links", + "python-version", + ] + + @property + def plugin_stage_packages(self): + if self.options.python_version == "python2": + python_base = "python" + elif self.options.python_version == "python3": + python_base = "python3" + + if self.project.info.get_build_base() in ("core", "core16", "core18"): + stage_packages = [python_base] + else: + stage_packages = [] + + if self.project.info.get_build_base() == "core18" and python_base == "python3": + stage_packages.append("{}-distutils".format(python_base)) + + return stage_packages + + # ignore mypy error: Read-only property cannot override read-write property + @property # type: ignore + def stage_packages(self): + try: + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingPythonCommandError: + return super().stage_packages + self.plugin_stage_packages + else: + return super().stage_packages + + @property + def _pip(self): + if not self.__pip: + self.__pip = _python.Pip( + python_major_version=self._python_major_version, + part_dir=self.partdir, + install_dir=self.installdir, + stage_dir=self.project.stage_dir, + ) + return self.__pip + + def __init__(self, name, options, project): + super().__init__(name, options, project) + + self._setup_base_tools(project.info.get_build_base()) + + self._manifest = collections.OrderedDict() + + # Pip requires only the major version of python rather than the command + # name like our option requires. + match = re.match(r"python(?P\d).*", self.options.python_version) + if not match: + raise UnsupportedPythonVersionError( + python_version=self.options.python_version + ) + + self._python_major_version = match.group("major_version") + self.__pip = None + + def _setup_base_tools(self, base): + # NOTE: stage-packages are lazily loaded. + if base in ("core", "core16", "core18"): + if self.options.python_version == "python3": + self.build_packages.extend( + [ + "python3-dev", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + ] + ) + elif self.options.python_version == "python2": + self.build_packages.extend( + [ + "python-dev", + "python-pip", + "python-pkg-resources", + "python-setuptools", + ] + ) + else: + raise errors.PluginBaseError(part_name=self.name, base=base) + + def pull(self): + super().pull() + + self._pip.setup() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Download this project, using its setup.py if present. This will + # also download any python-packages requested. + self._download_project() + + def clean_pull(self): + super().clean_pull() + self._pip.clean_packages() + + def build(self): + super().build() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Install the packages that have already been downloaded + installed_pipy_packages = self._install_project() + + requirements = self._get_list_of_packages_from_property( + self.options.requirements + ) + if requirements: + self._manifest["requirements-contents"] = requirements + + constraints = self._get_list_of_packages_from_property(self.options.constraints) + if constraints: + self._manifest["constraints-contents"] = constraints + + self._manifest["python-packages"] = [ + "{}={}".format(name, installed_pipy_packages[name]) + for name in installed_pipy_packages + ] + + try: + _python.generate_sitecustomize( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingUserSitePackagesError as site_error: + print("Part {!r} generated to site-packages: {!s}.".format( + self.name, site_error)) + + def _find_file(self, *, filename: str) -> Optional[str]: + # source-subdir defaults to '' + for basepath in [self.builddir, self.sourcedir]: + if basepath == self.sourcedir: + # This is overwritten in the base plugin + # TODO add consistency + source_subdir = self.options.source_subdir + else: + source_subdir = "" + filepath = os.path.join(basepath, source_subdir, filename) + if os.path.exists(filepath): + return filepath + + return None + + def _get_setup_py_dir(self): + setup_py_dir = None + setup_py_path = self._find_file(filename="setup.py") + if setup_py_path: + setup_py_dir = os.path.dirname(setup_py_path) + + return setup_py_dir + + def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: + """Return a sorted list of all packages found in property.""" + package_list = list() # type: List[str] + for entry in property_list: + contents = self._get_file_contents(entry) + package_list.extend(contents.splitlines()) + return package_list + + def _get_normalized_property_set( + self, property_name, property_list: List[str] + ) -> Set[str]: + """Return a normalized set from a requirements or constraints list.""" + normalized = set() # type: Set[str] + for entry in property_list: + if isurl(entry): + normalized.add(entry) + else: + entry_file = self._find_file(filename=entry) + if not entry_file: + raise SnapcraftPluginPythonFileMissing( + plugin_property=property_name, plugin_property_value=entry + ) + normalized.add(entry_file) + + return normalized + + def _install_wheels(self, wheels): + installed = self._pip.list() + wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] + + # we want to avoid installing what is already provided in + # stage-packages + need_install = [k for k in wheel_names if k not in installed] + self._pip.install( + need_install, + upgrade=True, + install_deps=False, + process_dependency_links=self.options.process_dependency_links, + ) + + def _download_project(self): + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + self._pip.download( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + def _install_project(self): + setup_py_dir = self._get_setup_py_dir() + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + # setup.py is handled in a different step as some projects may + # need to satisfy dependencies for setup.py to be parsed. + wheels = self._pip.wheel( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + if setup_py_dir is not None: + self._pip.download( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + wheels = self._pip.wheel( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + setup_py_path = os.path.join(setup_py_dir, "setup.py") + if os.path.exists(setup_py_path): + self._pip.install( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + process_dependency_links=self.options.process_dependency_links, + upgrade=True, + ) + # pbr and others don't work using `pip install .` + # LP: #1670852 + # There is also a chance that this setup.py is distutils based + # in which case we will rely on the `pip install .` ran before + # this. + with contextlib.suppress(SnapcraftPluginCommandError): + self._setup_tools_install(setup_py_path) + + return self._pip.list() + + def _setup_tools_install(self, setup_file): + command = [ + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ), + os.path.basename(setup_file), + "--no-user-cfg", + "install", + "--single-version-externally-managed", + "--user", + "--record", + "install.txt", + ] + self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) + + # Fix all shebangs to use the in-snap python. The stuff installed from + # pip has already been fixed, but anything done in this step has not. + mangling.rewrite_python_shebangs(self.installdir) + + def _get_file_contents(self, path): + if isurl(path): + return requests.get(path).text + else: + file_path = os.path.join(self.sourcedir, path) + with open(file_path) as _file: + return _file.read() + + def get_manifest(self): + return self._manifest + + def snap_fileset(self): + fileset = super().snap_fileset() + fileset.append("-bin/pip") + fileset.append("-bin/pip2") + fileset.append("-bin/pip3") + fileset.append("-bin/pip2.7") + fileset.append("-bin/pip3.*") + fileset.append("-bin/easy_install*") + fileset.append("-bin/wheel") + # Holds all the .pyc files. It is a major cause of inter part + # conflict. + fileset.append("-**/__pycache__") + fileset.append("-**/*.pyc") + # The RECORD files include hashes useful when uninstalling packages. + # In the snap they will cause conflicts when more than one part uses + # the python plugin. + fileset.append("-lib/python*/site-packages/*/RECORD") + return fileset + + +@contextlib.contextmanager +def simple_env_bzr(bin_dir): + """Create an appropriate environment to run bzr. + + The python plugin sets up PYTHONUSERBASE and PYTHONHOME which + conflicts with bzr when using python3 as those two environment + variables will make bzr look for modules in the wrong location. + """ + os.makedirs(bin_dir, exist_ok=True) + bzr_bin = os.path.join(bin_dir, "bzr") + real_bzr_bin = which("bzr") + if real_bzr_bin: + exec_line = 'exec {} "$@"'.format(real_bzr_bin) + else: + exec_line = "echo bzr needs to be in PATH; exit 1" + with open(bzr_bin, "w") as f: + f.write( + dedent( + """#!/bin/sh + unset PYTHONUSERBASE + unset PYTHONHOME + {} + """.format( + exec_line + ) + ) + ) + os.chmod(bzr_bin, 0o777) + try: + yield + finally: + os.remove(bzr_bin) + if not os.listdir(bin_dir): + os.rmdir(bin_dir) diff --git a/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml similarity index 79% rename from snap/snapcraft.yaml rename to snap/snap/snapcraft.yaml index 38d32ed0b..0e4a12204 100644 --- a/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,19 +38,19 @@ apps: parts: python-augeas: - plugin: python + plugin: x-python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: python + plugin: x-python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: python + plugin: x-python source: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -58,20 +58,30 @@ parts: override-pull: | snapcraftctl pull snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//` + # Workaround for lack of site-packages leading to empty sitecustomize.py + stage: + - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + stage: + # Prefer cffi + - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 - after: [certbot] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + # This is the last step, compile pycache now as there should be no conflicts. + override-prime: | + snapcraftctl prime + ./usr/bin/python3 -m compileall -q . + # After certbot-apache to not rebuild duplicates (essentially sharing what was already staged, + # like zope) + after: [certbot-apache] From c12baf7d8c873a1492088f844fe885a16bf19cd0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 9 Jan 2020 10:26:37 -0800 Subject: [PATCH 07/33] Fix path to apache-conf-test --- snap/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/test.sh b/snap/test.sh index e89743d47..e6aca7efd 100644 --- a/snap/test.sh +++ b/snap/test.sh @@ -7,5 +7,5 @@ venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 # DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! if [ -n "$TRAVIS" ]; then - venv/bin/python certbot/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + venv/bin/python certbot/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules fi From 2bcabe6626ad4aa0073269bcee9719c24da469cb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 18 Feb 2020 13:58:18 +0100 Subject: [PATCH 08/33] Fix certbot part build in snap * Declare properly source-subdir to build the certbot part * Use snapcraft 3.10+, remove the custom python plugin --- snap/.travis.yml | 3 +- snap/snap/plugins/x_python.py | 499 ---------------------------------- snap/snap/snapcraft.yaml | 11 +- 3 files changed, 8 insertions(+), 505 deletions(-) delete mode 100644 snap/snap/plugins/x_python.py diff --git a/snap/.travis.yml b/snap/.travis.yml index 418407558..125893985 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -27,7 +27,8 @@ notifications: addons: snaps: - name: snapcraft - channel: stable + # TODO: move back to stable channel once snapcraft 3.10 has been pushed to it. + channel: candidate classic: true - name: lxd channel: stable diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py deleted file mode 100644 index 5c8c3b86a..000000000 --- a/snap/snap/plugins/x_python.py +++ /dev/null @@ -1,499 +0,0 @@ -# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- -# -# This is an almost verbatim copy of what is in the snapcraft -# tree, can be completely removed once the functionality in -# snapcraft is in place. (LP: #1841861) -# -# Copyright (C) 2016-2020 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""The python plugin can be used for python 2 or 3 based parts. - -It can be used for python projects where you would want to do: - - - import python modules with a requirements.txt - - build a python project that has a setup.py - - install packages straight from pip - -This plugin uses the common plugin keywords as well as those for "sources". -For more information check the 'plugins' topic for the former and the -'sources' topic for the latter. - -Additionally, this plugin uses the following plugin-specific keywords: - - - requirements: - (list of strings) - List of paths to requirements files. - - constraints: - (list of strings) - List of paths to constraint files. - - process-dependency-links: - (bool; default: false) - Enable the processing of dependency links in pip, which allow one - project to provide places to look for another project - - python-packages: - (list) - A list of dependencies to get from PyPI - - python-version: - (string; default: python3) - The python version to use. Valid options are: python2 and python3 - -If the plugin finds a python interpreter with a basename that matches -`python-version` in the directory on the following fixed path: -`/usr/bin/` then this interpreter would -be preferred instead and no interpreter would be brought in through -`stage-packages` mechanisms. -""" - -import collections -import contextlib -import os -import re -from shutil import which -from textwrap import dedent -from typing import List, Optional, Set - -import requests - -import snapcraft -from snapcraft.common import isurl -from snapcraft.internal import errors, mangling -from snapcraft.internal.errors import SnapcraftPluginCommandError -from snapcraft.plugins import _python - - -class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): - - fmt = "Unsupported python version: {python_version!r}" - - -class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): - - fmt = ( - "Failed to find the referred {plugin_property} file at the given " - "path: {plugin_property_value!r}.\n" - "Check the property and ensure the file exists." - ) - - def __init__(self, *, plugin_property, plugin_property_value): - super().__init__( - plugin_property=plugin_property, plugin_property_value=plugin_property_value - ) - - -class PythonPlugin(snapcraft.BasePlugin): - @classmethod - def schema(cls): - schema = super().schema() - schema["properties"]["requirements"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["constraints"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["python-packages"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["process-dependency-links"] = { - "type": "boolean", - "default": False, - } - schema["properties"]["python-version"] = { - "type": "string", - "default": "python3", - "enum": ["python2", "python3"], - } - schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] - - return schema - - @classmethod - def get_pull_properties(cls): - # Inform Snapcraft of the properties associated with pulling. If these - # change in the YAML Snapcraft will consider the pull step dirty. - return [ - "requirements", - "constraints", - "python-packages", - "process-dependency-links", - "python-version", - ] - - @property - def plugin_stage_packages(self): - if self.options.python_version == "python2": - python_base = "python" - elif self.options.python_version == "python3": - python_base = "python3" - - if self.project.info.get_build_base() in ("core", "core16", "core18"): - stage_packages = [python_base] - else: - stage_packages = [] - - if self.project.info.get_build_base() == "core18" and python_base == "python3": - stage_packages.append("{}-distutils".format(python_base)) - - return stage_packages - - # ignore mypy error: Read-only property cannot override read-write property - @property # type: ignore - def stage_packages(self): - try: - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingPythonCommandError: - return super().stage_packages + self.plugin_stage_packages - else: - return super().stage_packages - - @property - def _pip(self): - if not self.__pip: - self.__pip = _python.Pip( - python_major_version=self._python_major_version, - part_dir=self.partdir, - install_dir=self.installdir, - stage_dir=self.project.stage_dir, - ) - return self.__pip - - def __init__(self, name, options, project): - super().__init__(name, options, project) - - self._setup_base_tools(project.info.get_build_base()) - - self._manifest = collections.OrderedDict() - - # Pip requires only the major version of python rather than the command - # name like our option requires. - match = re.match(r"python(?P\d).*", self.options.python_version) - if not match: - raise UnsupportedPythonVersionError( - python_version=self.options.python_version - ) - - self._python_major_version = match.group("major_version") - self.__pip = None - - def _setup_base_tools(self, base): - # NOTE: stage-packages are lazily loaded. - if base in ("core", "core16", "core18"): - if self.options.python_version == "python3": - self.build_packages.extend( - [ - "python3-dev", - "python3-pip", - "python3-pkg-resources", - "python3-setuptools", - ] - ) - elif self.options.python_version == "python2": - self.build_packages.extend( - [ - "python-dev", - "python-pip", - "python-pkg-resources", - "python-setuptools", - ] - ) - else: - raise errors.PluginBaseError(part_name=self.name, base=base) - - def pull(self): - super().pull() - - self._pip.setup() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Download this project, using its setup.py if present. This will - # also download any python-packages requested. - self._download_project() - - def clean_pull(self): - super().clean_pull() - self._pip.clean_packages() - - def build(self): - super().build() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Install the packages that have already been downloaded - installed_pipy_packages = self._install_project() - - requirements = self._get_list_of_packages_from_property( - self.options.requirements - ) - if requirements: - self._manifest["requirements-contents"] = requirements - - constraints = self._get_list_of_packages_from_property(self.options.constraints) - if constraints: - self._manifest["constraints-contents"] = constraints - - self._manifest["python-packages"] = [ - "{}={}".format(name, installed_pipy_packages[name]) - for name in installed_pipy_packages - ] - - try: - _python.generate_sitecustomize( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingUserSitePackagesError as site_error: - print("Part {!r} generated to site-packages: {!s}.".format( - self.name, site_error)) - - def _find_file(self, *, filename: str) -> Optional[str]: - # source-subdir defaults to '' - for basepath in [self.builddir, self.sourcedir]: - if basepath == self.sourcedir: - # This is overwritten in the base plugin - # TODO add consistency - source_subdir = self.options.source_subdir - else: - source_subdir = "" - filepath = os.path.join(basepath, source_subdir, filename) - if os.path.exists(filepath): - return filepath - - return None - - def _get_setup_py_dir(self): - setup_py_dir = None - setup_py_path = self._find_file(filename="setup.py") - if setup_py_path: - setup_py_dir = os.path.dirname(setup_py_path) - - return setup_py_dir - - def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: - """Return a sorted list of all packages found in property.""" - package_list = list() # type: List[str] - for entry in property_list: - contents = self._get_file_contents(entry) - package_list.extend(contents.splitlines()) - return package_list - - def _get_normalized_property_set( - self, property_name, property_list: List[str] - ) -> Set[str]: - """Return a normalized set from a requirements or constraints list.""" - normalized = set() # type: Set[str] - for entry in property_list: - if isurl(entry): - normalized.add(entry) - else: - entry_file = self._find_file(filename=entry) - if not entry_file: - raise SnapcraftPluginPythonFileMissing( - plugin_property=property_name, plugin_property_value=entry - ) - normalized.add(entry_file) - - return normalized - - def _install_wheels(self, wheels): - installed = self._pip.list() - wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] - - # we want to avoid installing what is already provided in - # stage-packages - need_install = [k for k in wheel_names if k not in installed] - self._pip.install( - need_install, - upgrade=True, - install_deps=False, - process_dependency_links=self.options.process_dependency_links, - ) - - def _download_project(self): - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - self._pip.download( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - def _install_project(self): - setup_py_dir = self._get_setup_py_dir() - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - # setup.py is handled in a different step as some projects may - # need to satisfy dependencies for setup.py to be parsed. - wheels = self._pip.wheel( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - if setup_py_dir is not None: - self._pip.download( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - wheels = self._pip.wheel( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - setup_py_path = os.path.join(setup_py_dir, "setup.py") - if os.path.exists(setup_py_path): - self._pip.install( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - process_dependency_links=self.options.process_dependency_links, - upgrade=True, - ) - # pbr and others don't work using `pip install .` - # LP: #1670852 - # There is also a chance that this setup.py is distutils based - # in which case we will rely on the `pip install .` ran before - # this. - with contextlib.suppress(SnapcraftPluginCommandError): - self._setup_tools_install(setup_py_path) - - return self._pip.list() - - def _setup_tools_install(self, setup_file): - command = [ - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ), - os.path.basename(setup_file), - "--no-user-cfg", - "install", - "--single-version-externally-managed", - "--user", - "--record", - "install.txt", - ] - self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) - - # Fix all shebangs to use the in-snap python. The stuff installed from - # pip has already been fixed, but anything done in this step has not. - mangling.rewrite_python_shebangs(self.installdir) - - def _get_file_contents(self, path): - if isurl(path): - return requests.get(path).text - else: - file_path = os.path.join(self.sourcedir, path) - with open(file_path) as _file: - return _file.read() - - def get_manifest(self): - return self._manifest - - def snap_fileset(self): - fileset = super().snap_fileset() - fileset.append("-bin/pip") - fileset.append("-bin/pip2") - fileset.append("-bin/pip3") - fileset.append("-bin/pip2.7") - fileset.append("-bin/pip3.*") - fileset.append("-bin/easy_install*") - fileset.append("-bin/wheel") - # Holds all the .pyc files. It is a major cause of inter part - # conflict. - fileset.append("-**/__pycache__") - fileset.append("-**/*.pyc") - # The RECORD files include hashes useful when uninstalling packages. - # In the snap they will cause conflicts when more than one part uses - # the python plugin. - fileset.append("-lib/python*/site-packages/*/RECORD") - return fileset - - -@contextlib.contextmanager -def simple_env_bzr(bin_dir): - """Create an appropriate environment to run bzr. - - The python plugin sets up PYTHONUSERBASE and PYTHONHOME which - conflicts with bzr when using python3 as those two environment - variables will make bzr look for modules in the wrong location. - """ - os.makedirs(bin_dir, exist_ok=True) - bzr_bin = os.path.join(bin_dir, "bzr") - real_bzr_bin = which("bzr") - if real_bzr_bin: - exec_line = 'exec {} "$@"'.format(real_bzr_bin) - else: - exec_line = "echo bzr needs to be in PATH; exit 1" - with open(bzr_bin, "w") as f: - f.write( - dedent( - """#!/bin/sh - unset PYTHONUSERBASE - unset PYTHONHOME - {} - """.format( - exec_line - ) - ) - ) - os.chmod(bzr_bin, 0o777) - try: - yield - finally: - os.remove(bzr_bin) - if not os.listdir(bin_dir): - os.rmdir(bin_dir) diff --git a/snap/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml index 0e4a12204..53643865a 100644 --- a/snap/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,20 +38,21 @@ apps: parts: python-augeas: - plugin: x-python + plugin: python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: x-python + plugin: python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: x-python + plugin: python source: certbot + source-subdir: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [acme] @@ -62,7 +63,7 @@ parts: stage: - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] @@ -73,7 +74,7 @@ parts: # Prefer cffi - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] From ed9648b4a30a6182a1bb63e5785275c1ff65d513 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 22 Apr 2020 11:30:33 -0700 Subject: [PATCH 09/33] Add snap instructions (#7889) Part of #7671. * Add snap instructions. * Mention Linux+systemd --- certbot/docs/install.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 11994776c..b50540a7d 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -61,6 +61,23 @@ Alternate installation methods If you are offline or your operating system doesn't provide a package, you can use an alternate method for installing ``certbot``. +.. _snap-install: + +Snap +---- + +Most modern Linux distributions (basically any that use systemd) can install +Certbot packaged as a snap. Support for the Certbot snap is currently in its +beta phase and limited to the x86_64 architecture, but it provides an easy way +to ensure you have the latest version of Certbot with features like automated +certificate renewal preconfigured. + +You can find instructions for installing the Certbot snap at +https://certbot.eff.org/instructions by selecting your server software and then +choosing "snapd" in the "System" dropdown menu. (You should select "snapd" +regardless of your operating system, as our instructions are the same across +all systems.) + .. _certbot-auto: Certbot-Auto From 9d94c6c5ef8150fcd3c74319599f5106b9726c4c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 22 Apr 2020 13:10:13 -0700 Subject: [PATCH 10/33] Remove useless before_install lines (#7920) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72cd3a408..5f7bdc21b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,6 @@ matrix: - sudo: required env: TOXENV=apache_compat services: docker - before_install: addons: <<: *not-on-master - sudo: required @@ -84,7 +83,6 @@ matrix: - sudo: required env: TOXENV=nginx_compat services: docker - before_install: addons: <<: *extended-test-suite - python: "3.7" From a06d5ac7a11cb17610389344b6ae8cc10a75a0ef Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 23 Apr 2020 12:49:35 -0700 Subject: [PATCH 11/33] Deprecate certbot-auto on Gentoo, macOS, and FreeBSD (#7926) * Deprecate certbot-auto on Gentoo * Deprecate certbot-auto on macOS * Deprecate certbot-auto on FreeBSD * build le-auto * Update Changelog * Deprecate certbot-auto on Gentoo, FreeBSD, and macOS --- certbot/CHANGELOG.md | 1 + letsencrypt-auto-source/letsencrypt-auto | 15 +++------------ letsencrypt-auto-source/letsencrypt-auto.template | 15 +++------------ 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1b9379de7..9eb7cd9e8 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -20,6 +20,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Stop asking interactively if the user would like to add a redirect. * `mock` dependency is now conditional on Python 2 in all of our packages. +* Deprecate certbot-auto on Gentoo, macOS, and FreeBSD. ### Fixed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index ca0bda2d5..c43e6d6d8 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then } BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION" elif [ -f /etc/gentoo-release ]; then - Bootstrap() { - DeprecationBootstrap "Gentoo" BootstrapGentooCommon - } - BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq FreeBSD ; then - Bootstrap() { - DeprecationBootstrap "FreeBSD" BootstrapFreeBsd - } - BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq Darwin ; then - Bootstrap() { - DeprecationBootstrap "macOS" BootstrapMac - } - BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then Bootstrap() { ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 53e57a498..da8fabfea 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -432,20 +432,11 @@ elif [ -f /etc/manjaro-release ]; then } BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION" elif [ -f /etc/gentoo-release ]; then - Bootstrap() { - DeprecationBootstrap "Gentoo" BootstrapGentooCommon - } - BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq FreeBSD ; then - Bootstrap() { - DeprecationBootstrap "FreeBSD" BootstrapFreeBsd - } - BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq Darwin ; then - Bootstrap() { - DeprecationBootstrap "macOS" BootstrapMac - } - BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then Bootstrap() { ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon From 84b6c3cebb9defecb866d4a89a2ef1f3ae01396d Mon Sep 17 00:00:00 2001 From: taixx046 Date: Thu, 23 Apr 2020 15:20:32 -0500 Subject: [PATCH 12/33] reorganized error message when a user entered an invalid email address --- AUTHORS.md | 1 + certbot/certbot/display/ops.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 21a6e7773..04f5b446f 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -237,6 +237,7 @@ Authors * [Stefan Weil](https://github.com/stweil) * [Steve Desmond](https://github.com/stevedesmond-ca) * [sydneyli](https://github.com/sydneyli) +* [taixx046](https://github.com/taixx046) * [Tan Jay Jun](https://github.com/jayjun) * [Tapple Gao](https://github.com/tapple) * [Telepenin Nikolay](https://github.com/telepenin) diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index 21d169a55..2c3503eab 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -30,7 +30,7 @@ def get_email(invalid=False, optional=True): """ invalid_prefix = "There seem to be problems with that address. " - msg = "Enter email address (used for urgent renewal and security notices)" + msg = "Enter email address (used for urgent renewal and security notices)\n" unsafe_suggestion = ("\n\nIf you really want to skip this, you can run " "the client with --register-unsafely-without-email " "but make sure you then backup your account key from " @@ -64,7 +64,7 @@ def get_email(invalid=False, optional=True): if util.safe_email(email): return email if suggest_unsafe: - msg += unsafe_suggestion + msg = unsafe_suggestion + msg suggest_unsafe = False # add this message at most once invalid = bool(email) From e5cde2c598ede6c93adce4c740046614d056df2f Mon Sep 17 00:00:00 2001 From: taixx046 Date: Thu, 23 Apr 2020 15:29:38 -0500 Subject: [PATCH 13/33] updated changelog.md --- certbot/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 9eb7cd9e8..cd261d166 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed +* Reorganized error message when a user entered an invalid email address. * Stop asking interactively if the user would like to add a redirect. * `mock` dependency is now conditional on Python 2 in all of our packages. * Deprecate certbot-auto on Gentoo, macOS, and FreeBSD. From b06dacdfb50361e239b784ee9dd0e4e81eb68167 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 23 Apr 2020 14:26:34 -0700 Subject: [PATCH 14/33] Add --chain-path to --help install output. (#7937) --- certbot/certbot/_internal/cli/paths_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 4378435d7..62f5e224d 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -38,7 +38,7 @@ def _paths_parser(helpful): default_cp = flag_default("auth_chain_path") add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a full certificate chain (certificate plus chain).") - add("paths", "--chain-path", default=default_cp, type=os.path.abspath, + add(["paths", "install"], "--chain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a certificate chain.") add("paths", "--config-dir", default=flag_default("config_dir"), help=config_help("config_dir")) From d71d3a11448f875d508086c79890db498cc4192a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 23 Apr 2020 14:27:05 -0700 Subject: [PATCH 15/33] dont use venv.py (#7936) --- certbot/docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index c138e4f46..73e681150 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -168,7 +168,7 @@ To do so you need: - Docker installed, and a user with access to the Docker client, - an available `local copy`_ of Certbot. -The virtual environment set up with `python tools/venv.py` contains two commands +The virtual environment set up with `python tools/venv3.py` contains two commands that can be used once the virtual environment is activated: .. code-block:: shell From 4cce3458f3e6a38fbb3f39613aca1c9e1ebd86ca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 23 Apr 2020 14:29:16 -0700 Subject: [PATCH 16/33] Avoid deleting the workspace twice. (#7923) --- certbot-ci/certbot_integration_tests/utils/acme_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index fb202005e..0a6d3ff41 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -86,7 +86,8 @@ class ACMEServer(object): 'alpine', 'rm', '-rf', '/workspace/boulder']) process.wait() finally: - shutil.rmtree(self._workspace) + if os.path.exists(self._workspace): + shutil.rmtree(self._workspace) if self._stdout != sys.stdout: self._stdout.close() print('=> Test infrastructure stopped and cleaned up.') From 335894ab3b0665dcf177ba2121a83422b1053fde Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 9 Apr 2020 13:15:28 -0700 Subject: [PATCH 17/33] Merge snap code into the Certbot repo * merge .gitignore * Move snapcraft.yml up one level. * update source * move test.sh to tox.ini * use new tox.ini in .travis.yml * move snap build code * make script executable * remove unused python3-dev * don't use deprecated classic flag * go back to stable channel * add nginx in snap addons * add deploy steps * Add comments explaining external tox envs. * error if not in CI * don't use --depth * remove old .travis.yml * Add big comment about SNAP_TOKEN. * Set all_branches: true. * Add repo setting. * run travis on tags * Add more documenting comments to .travis.yml. --- .gitignore | 7 ++++ .travis.yml | 67 ++++++++++++++++++++++++++++++--- snap/.gitignore | 8 ---- snap/.travis.yml | 34 ----------------- snap/local/build_and_install.sh | 14 +++++++ snap/{snap => }/snapcraft.yaml | 8 ++-- snap/test.sh | 11 ------ tox.ini | 21 +++++++++-- 8 files changed, 105 insertions(+), 65 deletions(-) delete mode 100644 snap/.gitignore delete mode 100644 snap/.travis.yml create mode 100755 snap/local/build_and_install.sh rename snap/{snap => }/snapcraft.yaml (97%) delete mode 100644 snap/test.sh diff --git a/.gitignore b/.gitignore index 6505e716c..064e7fffe 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,10 @@ tests/letstest/venv3/ .certbot_test_workspace **/assets/pebble* **/assets/challtestsrv* + +# snap files +.snapcraft +parts +prime +stage +*.snap diff --git a/.travis.yml b/.travis.yml index 72cd3a408..3b22429c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,17 +11,23 @@ before_script: # Use Travis retry feature for farm tests since they are flaky - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi' - export TOX_TESTENV_PASSENV=TRAVIS + - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi' # Only build pushes to the master branch, PRs, and branches beginning with -# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x`. This reduces -# the number of simultaneous Travis runs, which speeds turnaround time on -# review since there is a cap of on the number of simultaneous runs. +# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or +# `vdigit(s).digit(s).digit(s)`. As documented at +# https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches, +# this includes tags so pushing tags of the form `vdigit(s).digit(s).digit(s)` +# will also trigger tests. This reduces the number of simultaneous Travis runs, +# which speeds turnaround time on review since there is a cap of on the number +# of simultaneous runs. branches: # When changing these branches, please ensure the documentation under # "Running tests in CI" is still correct. only: - master - - /^\d+\.\d+\.x$/ + - /^\d+\.\d+\.x$/ # this matches our point release branches + - /^v\d+\.\d+\.\d+$/ # this matches our release tags - /^(travis-)?test-.*$/ # Jobs for the main test suite are always executed (including on PRs) except for pushes on master. @@ -36,10 +42,16 @@ extended-test-suite: &extended-test-suite matrix: include: # Main test suite - - python: "2.7" + - stage: "Test" + python: "2.7" env: ACME_SERVER=pebble TOXENV=integration <<: *not-on-master + # As documented at + # https://docs.travis-ci.com/user/build-stages/#how-to-define-build-stages, + # the previous stage will be automatically applied to all subsequent jobs + # until a new stage is defined. + # This job is always executed, including on master - python: "3.8" env: TOXENV=py38-cover FYI="py38 tests + code coverage" @@ -220,6 +232,51 @@ matrix: packages: # don't install nginx and apache - libaugeas0 <<: *extended-test-suite + - stage: "Snap" + sudo: required + env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble + addons: + apt: + packages: + - nginx-light + snaps: + - name: snapcraft + channel: stable + confinement: classic + - name: lxd + channel: stable + git: + # By default, Travis clones the repo to a depth of 50 commits which can + # break the ability to use `git describe` to set the version of the + # snap. This setting removes the --depth flag from git commands solving + # this problem. See + # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth + # for more info. + depth: false + deploy: + # This section relies on credentials stored in a SNAP_TOKEN environment + # variable in Travis. See + # https://docs.travis-ci.com/user/deployment/snaps/ for more info. + # This credential has a maximum lifetime of 1 year and the current + # credential will expire on 4/22/2021. The value of SNAP_TOKEN will + # need to be updated to use a new credential before then to prevent + # automated deploys from breaking. Remembering to do this is also + # tracked by https://github.com/certbot/certbot/issues/7931. + 'on': + # Deploy on release tags or nightly runs from any branch. We only try + # to deploy from the certbot/certbot repo to prevent errors if forks + # of this repo try to run tests. + all_branches: true + condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron + repo: certbot/certbot + provider: snap + snap: certbot_*.snap + channel: edge + # skip_cleanup is needed to prevent Travis from deleting the snaps we + # just built and tested. See + # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. + skip_cleanup: true + <<: *extended-test-suite # container-based infrastructure sudo: false diff --git a/snap/.gitignore b/snap/.gitignore deleted file mode 100644 index 2a339aeaa..000000000 --- a/snap/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.snapcraft -parts -prime -stage -*.snap -certbot -venv -.idea diff --git a/snap/.travis.yml b/snap/.travis.yml deleted file mode 100644 index 125893985..000000000 --- a/snap/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -language: python -before_install: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light python3-venv - - sudo -E /etc/init.d/nginx stop -install: - - sudo /snap/bin/lxd.migrate -yes - - sudo /snap/bin/lxd waitready - - sudo /snap/bin/lxd init --auto - - git clone https://github.com/certbot/certbot.git certbot --branch master - - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt -before_script: - - sudo snapcraft --use-lxd -script: - - sudo snap install --dangerous --classic *.snap - - sudo bash test.sh -deploy: - 'on': - branch: master - provider: snap - snap: certbot_*.snap - channel: edge - skip_cleanup: true -notifications: - email: - recipients: [robie.basak@canonical.com] - on_failure: change -addons: - snaps: - - name: snapcraft - # TODO: move back to stable channel once snapcraft 3.10 has been pushed to it. - channel: candidate - classic: true - - name: lxd - channel: stable diff --git a/snap/local/build_and_install.sh b/snap/local/build_and_install.sh new file mode 100755 index 000000000..4c9754d3e --- /dev/null +++ b/snap/local/build_and_install.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +if [[ -z "$TRAVIS" ]]; then + echo "This script makes global changes to the system it is run on so should only be run in CI." + exit 1 +fi + +sudo /snap/bin/lxd.migrate -yes +sudo /snap/bin/lxd waitready +sudo /snap/bin/lxd init --auto +tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > constraints.txt +sudo snapcraft --use-lxd +sudo snap install --dangerous --classic *.snap diff --git a/snap/snap/snapcraft.yaml b/snap/snapcraft.yaml similarity index 97% rename from snap/snap/snapcraft.yaml rename to snap/snapcraft.yaml index 53643865a..2e4f68c00 100644 --- a/snap/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -45,13 +45,13 @@ parts: build-packages: [libaugeas-dev] acme: plugin: python - source: certbot + source: . source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: plugin: python - source: certbot + source: . source-subdir: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -64,7 +64,7 @@ parts: - -usr/lib/python3.6/sitecustomize.py certbot-apache: plugin: python - source: certbot + source: . source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -75,7 +75,7 @@ parts: - -lib/python3.6/site-packages/augeas.py certbot-nginx: plugin: python - source: certbot + source: . source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 diff --git a/snap/test.sh b/snap/test.sh deleted file mode 100644 index e6aca7efd..000000000 --- a/snap/test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -ex - -python3 -m venv venv -venv/bin/python -m pip install -e certbot/certbot-ci -venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 - -# DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! -if [ -n "$TRAVIS" ]; then - venv/bin/python certbot/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules -fi diff --git a/tox.ini b/tox.ini index 7f5b7bd5a..46e73e32f 100644 --- a/tox.ini +++ b/tox.ini @@ -138,15 +138,22 @@ commands = [testenv:apacheconftest] commands = - {[base]pip_install} acme certbot certbot-apache certbot-compatibility-test + {[base]pip_install} acme certbot certbot-apache {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test --debian-modules passenv = SERVER +[testenv:apacheconftest-external-with-pebble] +# Run apacheconftest with pebble and Certbot outside of tox's virtual +# environment. +commands = + {[base]pip_install} certbot-ci + {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + [testenv:apacheconftest-with-pebble] commands = - {[base]pip_install} acme certbot certbot-apache certbot-ci certbot-compatibility-test - {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + {[base]pip_install} acme certbot certbot-apache + {[testenv:apacheconftest-external-with-pebble]commands} [testenv:nginxroundtrip] commands = @@ -250,6 +257,14 @@ commands = --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=62 +[testenv:integration-external] +# Run integration tests with Certbot outside of tox's virtual environment. +commands = + {[base]pip_install} certbot-ci + pytest certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} +passenv = DOCKER_* + [testenv:integration-certbot-oldest] commands = {[base]pip_install} certbot From db6de76b1108c9c4d04163c9ecc6f9dafac94441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=AA=E3=81=A4=E3=81=8D?= Date: Fri, 24 Apr 2020 17:38:38 -0700 Subject: [PATCH 18/33] Allow existing but empty archive and live dir --- certbot/CHANGELOG.md | 1 + certbot/certbot/_internal/storage.py | 12 ++++++------ certbot/tests/storage_test.py | 20 ++++++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 9eb7cd9e8..af140f6be 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -21,6 +21,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Stop asking interactively if the user would like to add a redirect. * `mock` dependency is now conditional on Python 2 in all of our packages. * Deprecate certbot-auto on Gentoo, macOS, and FreeBSD. +* Allow existing but empty archive and live dir to be used when creating new lineage. ### Fixed diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 2dac163e2..05d9e3a8d 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -1007,18 +1007,18 @@ class RenewableCert(interfaces.RenewableCert): lineagename = lineagename_for_filename(config_filename) archive = full_archive_path(None, cli_config, lineagename) live_dir = _full_live_path(cli_config, lineagename) - if os.path.exists(archive): + if os.path.exists(archive) and (not os.path.isdir(archive) or os.listdir(archive)): config_file.close() raise errors.CertStorageError( "archive directory exists for " + lineagename) - if os.path.exists(live_dir): + if os.path.exists(live_dir) and (not os.path.isdir(live_dir) or os.listdir(live_dir)): config_file.close() raise errors.CertStorageError( "live directory exists for " + lineagename) - filesystem.mkdir(archive) - filesystem.mkdir(live_dir) - logger.debug("Archive directory %s and live " - "directory %s created.", archive, live_dir) + for i in (archive, live_dir): + if not os.path.exists(i): + filesystem.makedirs(i) + logger.debug("Creating directory %s.", i) # Put the data into the appropriate files on disk target = {kind: os.path.join(live_dir, kind + ".pem") for kind in ALL_FOUR} diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 5aa37824d..b67c4cbce 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -610,17 +610,25 @@ class RenewableCertTests(BaseRenewableCertTest): self.config.renewal_configs_dir, "the-lineage.com-0001.conf"))) self.assertTrue(os.path.exists(os.path.join( self.config.live_dir, "the-lineage.com-0001", "README"))) + # Allow write to existing but empty dir + filesystem.mkdir(os.path.join(self.config.default_archive_dir, "the-lineage.com-0002")) + result = storage.RenewableCert.new_lineage( + "the-lineage.com", b"cert3", b"privkey3", b"chain3", self.config) + self.assertTrue(os.path.exists(os.path.join( + self.config.live_dir, "the-lineage.com-0002", "README"))) + self.assertTrue(filesystem.check_mode(result.key_path, 0o600)) # Now trigger the detection of already existing files - filesystem.mkdir(os.path.join( - self.config.live_dir, "the-lineage.com-0002")) + shutil.copytree(os.path.join(self.config.live_dir, "the-lineage.com"), + os.path.join(self.config.live_dir, "the-lineage.com-0003")) self.assertRaises(errors.CertStorageError, storage.RenewableCert.new_lineage, "the-lineage.com", - b"cert3", b"privkey3", b"chain3", self.config) - filesystem.mkdir(os.path.join(self.config.default_archive_dir, "other-example.com")) + b"cert4", b"privkey4", b"chain4", self.config) + shutil.copytree(os.path.join(self.config.live_dir, "the-lineage.com"), + os.path.join(self.config.live_dir, "other-example.com")) self.assertRaises(errors.CertStorageError, storage.RenewableCert.new_lineage, - "other-example.com", b"cert4", - b"privkey4", b"chain4", self.config) + "other-example.com", b"cert5", + b"privkey5", b"chain5", self.config) # Make sure it can accept renewal parameters result = storage.RenewableCert.new_lineage( "the-lineage.com", b"cert2", b"privkey2", b"chain2", self.config) From 9cbb13ef041cb16c668f6644d132fcd738488798 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 27 Apr 2020 18:38:30 +0200 Subject: [PATCH 19/33] Run hooks with Powershell on Windows (#7800) Fixes #7713. As discussed in #7713, providing a Powershell script as hook for Certbot is not working currently. This is because hooks are run in a `cmd` environment, that recognizes only `.bat` files as valid scripts that can be run from their bare name on command line. On the other hand, the Powershell both `.bat` and `.ps1` scripts as valid scripts. This PR makes hooks command be executed by Powershell, instead of `cmd` as `Popen` does by default when `shell=true` is used. It also modifies the tests to handle this new environment, in particular in term of encoding (UTF-16-LE is the default one in Powershell). * Run hooks in powershell on Windows * Fix hook test * Fallback to unittest.mock * In fact, shell_cmd as a list of str could not work. Declare only str as acceptable input for shell_cmd. * Added changelog --- .../certbot_integration_tests/assets/hook.py | 4 +- .../certbot_tests/assertions.py | 4 +- .../certbot_integration_tests/utils/misc.py | 7 ++- certbot/CHANGELOG.md | 2 + certbot/certbot/_internal/hooks.py | 31 +----------- certbot/certbot/_internal/plugins/manual.py | 3 +- certbot/certbot/compat/misc.py | 41 ++++++++++++++++ certbot/tests/compat/misc_test.py | 48 +++++++++++++++++++ certbot/tests/hook_test.py | 44 ++--------------- 9 files changed, 107 insertions(+), 77 deletions(-) create mode 100644 certbot/tests/compat/misc_test.py diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 39aa72ac5..483892a93 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +from __future__ import print_function import os import sys @@ -7,5 +8,4 @@ if hook_script_type == 'deploy' and ('RENEWED_DOMAINS' not in os.environ or 'REN sys.stderr.write('Environment variables not properly set!\n') sys.exit(1) -with open(sys.argv[2], 'a') as file_h: - file_h.write(hook_script_type + '\n') +print(hook_script_type) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py b/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py index 1b5914d1a..53df6b890 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/assertions.py @@ -1,4 +1,5 @@ """This module contains advanced assertions for the certbot integration tests.""" +import io import os try: @@ -21,7 +22,8 @@ def assert_hook_execution(probe_path, probe_content): :param probe_path: path to the file that received the hook output :param probe_content: content expected when the hook is executed """ - with open(probe_path, 'r') as file: + encoding = 'utf-8' if POSIX_MODE else 'utf-16' + with io.open(probe_path, 'rt', encoding=encoding) as file: data = file.read() lines = [line.strip() for line in data.splitlines()] diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index fbb965034..4b2774827 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -140,13 +140,12 @@ def generate_test_file_hooks(config_dir, hook_probe): entrypoint_script = '''\ #!/usr/bin/env bash set -e -"{0}" "{1}" "{2}" "{3}" +"{0}" "{1}" "{2}" >> "{3}" '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe) else: - entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.bat') + entrypoint_script_path = os.path.join(hook_dir, 'entrypoint.ps1') entrypoint_script = '''\ -@echo off -"{0}" "{1}" "{2}" "{3}" +& "{0}" "{1}" "{2}" >> "{3}" '''.format(sys.executable, hook_path, entrypoint_script_path, hook_probe) with open(entrypoint_script_path, 'w') as file_h: diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 9eb7cd9e8..6b5a1055c 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -15,6 +15,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). * Added TLS-ALPN-01 challenge support in the `acme` library. Support of this challenge in the Certbot client is planned to be added in a future release. * Added minimal proxy support for OCSP verification. +* On Windows, hooks are now executed in a Powershell shell instead of a CMD shell, + allowing both `*.ps1` and `*.bat` as valid scripts for Certbot. ### Changed diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 589c59e89..f26c5c76b 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -2,14 +2,13 @@ from __future__ import print_function import logging -from subprocess import PIPE -from subprocess import Popen from acme.magic_typing import List from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem +from certbot.compat import misc from certbot.compat import os from certbot.plugins import util as plug_util @@ -229,36 +228,10 @@ def _run_hook(cmd_name, shell_cmd): :type shell_cmd: `list` of `str` or `str` :returns: stderr if there was any""" - err, _ = execute(cmd_name, shell_cmd) + err, _ = misc.execute_command(cmd_name, shell_cmd) return err -def execute(cmd_name, shell_cmd): - """Run a command. - - :param str cmd_name: the user facing name of the hook being run - :param shell_cmd: shell command to execute - :type shell_cmd: `list` of `str` or `str` - - :returns: `tuple` (`str` stderr, `str` stdout)""" - logger.info("Running %s command: %s", cmd_name, shell_cmd) - - # universal_newlines causes Popen.communicate() - # to return str objects instead of bytes in Python 3 - cmd = Popen(shell_cmd, shell=True, stdout=PIPE, - stderr=PIPE, universal_newlines=True) - out, err = cmd.communicate() - base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) - if out: - logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out) - if cmd.returncode != 0: - logger.error('%s command "%s" returned error code %d', - cmd_name, shell_cmd, cmd.returncode) - if err: - logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err) - return err, out - - def list_hooks(dir_path): """List paths to all hooks found in dir_path in sorted order. diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index b46622796..430059445 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -9,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot import reverter from certbot._internal import hooks +from certbot.compat import misc from certbot.compat import os from certbot.plugins import common @@ -186,4 +187,4 @@ permitted by DNS standards.) self.reverter.recovery_routine() def _execute_hook(self, hook_name): - return hooks.execute(self.option_name(hook_name), self.conf(hook_name)) + return misc.execute_command(self.option_name(hook_name), self.conf(hook_name)) diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index 956c56370..49a0814f4 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -4,12 +4,16 @@ particular category. """ from __future__ import absolute_import +import logging import select +import subprocess import sys from certbot import errors from certbot.compat import os +from acme.magic_typing import Tuple + try: from win32com.shell import shell as shellwin32 POSIX_MODE = False @@ -17,6 +21,7 @@ except ImportError: # pragma: no cover POSIX_MODE = True +logger = logging.getLogger(__name__) # For Linux: define OS specific standard binary directories STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else [] @@ -109,3 +114,39 @@ def underscores_for_unsupported_characters_in_path(path): # Windows specific drive, tail = os.path.splitdrive(path) return drive + tail.replace(':', '_') + + +def execute_command(cmd_name, shell_cmd): + # type: (str, str) -> Tuple[str, str] + """ + Run a command: + - on Linux command will be run by the standard shell selected with Popen(shell=True) + - on Windows command will be run in a Powershell shell + + :param str cmd_name: the user facing name of the hook being run + :param str shell_cmd: shell command to execute + + :returns: `tuple` (`str` stderr, `str` stdout) + """ + logger.info("Running %s command: %s", cmd_name, shell_cmd) + + if POSIX_MODE: + cmd = subprocess.Popen(shell_cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + else: + line = ['powershell.exe', '-Command', shell_cmd] + cmd = subprocess.Popen(line, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + + # universal_newlines causes Popen.communicate() + # to return str objects instead of bytes in Python 3 + out, err = cmd.communicate() + base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) + if out: + logger.info('Output from %s command %s:\n%s', cmd_name, base_cmd, out) + if cmd.returncode != 0: + logger.error('%s command "%s" returned error code %d', + cmd_name, shell_cmd, cmd.returncode) + if err: + logger.error('Error output from %s command %s:\n%s', cmd_name, base_cmd, err) + return err, out diff --git a/certbot/tests/compat/misc_test.py b/certbot/tests/compat/misc_test.py new file mode 100644 index 000000000..642f395ba --- /dev/null +++ b/certbot/tests/compat/misc_test.py @@ -0,0 +1,48 @@ +"""Tests for certbot.compat.misc""" +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore +import unittest + +from certbot.compat import os + + +class ExecuteTest(unittest.TestCase): + """Tests for certbot.compat.misc.execute_command.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.compat.misc import execute_command + return execute_command(*args, **kwargs) + + def test_it(self): + for returncode in range(0, 2): + for stdout in ("", "Hello World!",): + for stderr in ("", "Goodbye Cruel World!"): + self._test_common(returncode, stdout, stderr) + + def _test_common(self, returncode, stdout, stderr): + given_command = "foo" + given_name = "foo-hook" + with mock.patch("certbot.compat.misc.subprocess.Popen") as mock_popen: + mock_popen.return_value.communicate.return_value = (stdout, stderr) + mock_popen.return_value.returncode = returncode + with mock.patch("certbot.compat.misc.logger") as mock_logger: + self.assertEqual(self._call(given_name, given_command), (stderr, stdout)) + + executed_command = mock_popen.call_args[1].get( + "args", mock_popen.call_args[0][0]) + if os.name == 'nt': + expected_command = ['powershell.exe', '-Command', given_command] + else: + expected_command = given_command + self.assertEqual(executed_command, expected_command) + + mock_logger.info.assert_any_call("Running %s command: %s", + given_name, given_command) + if stdout: + mock_logger.info.assert_any_call(mock.ANY, mock.ANY, + mock.ANY, stdout) + if stderr or returncode: + self.assertTrue(mock_logger.error.called) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 32081f9d0..b8fd02461 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -72,13 +72,13 @@ class HookTest(test_util.ConfigTestCase): @classmethod def _call_with_mock_execute(cls, *args, **kwargs): - """Calls self._call after mocking out certbot._internal.hooks.execute. + """Calls self._call after mocking out certbot.compat.misc.execute_command. The mock execute object is returned rather than the return value of self._call. """ - with mock.patch("certbot._internal.hooks.execute") as mock_execute: + with mock.patch("certbot.compat.misc.execute_command") as mock_execute: mock_execute.return_value = ("", "") cls._call(*args, **kwargs) return mock_execute @@ -292,7 +292,7 @@ class RenewalHookTest(HookTest): # pylint: disable=abstract-method def _call_with_mock_execute(self, *args, **kwargs): - """Calls self._call after mocking out certbot._internal.hooks.execute. + """Calls self._call after mocking out certbot.compat.misc.execute_command. The mock execute object is returned rather than the return value of self._call. The mock execute object asserts that environment @@ -313,7 +313,7 @@ class RenewalHookTest(HookTest): self.assertEqual(os.environ["RENEWED_LINEAGE"], lineage) return ("", "") - with mock.patch("certbot._internal.hooks.execute") as mock_execute: + with mock.patch("certbot.compat.misc.execute_command") as mock_execute: mock_execute.side_effect = execute_side_effect self._call(*args, **kwargs) return mock_execute @@ -418,42 +418,6 @@ class RenewHookTest(RenewalHookTest): mock_execute.assert_called_with("deploy-hook", self.config.renew_hook) -class ExecuteTest(unittest.TestCase): - """Tests for certbot._internal.hooks.execute.""" - - @classmethod - def _call(cls, *args, **kwargs): - from certbot._internal.hooks import execute - return execute(*args, **kwargs) - - def test_it(self): - for returncode in range(0, 2): - for stdout in ("", "Hello World!",): - for stderr in ("", "Goodbye Cruel World!"): - self._test_common(returncode, stdout, stderr) - - def _test_common(self, returncode, stdout, stderr): - given_command = "foo" - given_name = "foo-hook" - with mock.patch("certbot._internal.hooks.Popen") as mock_popen: - mock_popen.return_value.communicate.return_value = (stdout, stderr) - mock_popen.return_value.returncode = returncode - with mock.patch("certbot._internal.hooks.logger") as mock_logger: - self.assertEqual(self._call(given_name, given_command), (stderr, stdout)) - - executed_command = mock_popen.call_args[1].get( - "args", mock_popen.call_args[0][0]) - self.assertEqual(executed_command, given_command) - - mock_logger.info.assert_any_call("Running %s command: %s", - given_name, given_command) - if stdout: - mock_logger.info.assert_any_call(mock.ANY, mock.ANY, - mock.ANY, stdout) - if stderr or returncode: - self.assertTrue(mock_logger.error.called) - - class ListHooksTest(test_util.TempDirTestCase): """Tests for certbot._internal.hooks.list_hooks.""" From ac2d691ade1e86595502d6603afc8a7853f4acfe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 4 May 2020 16:54:09 -0700 Subject: [PATCH 20/33] Add warning about ignoring our own warnings (#7971) Coming out of the conversation at #7863 in the linked Google Doc, we should always have at least 1 release between updating one of our plugins to stop using a deprecated acme/certbot API and removing it from acme/certbot. Doing this gives the plugin changes time to propagate rather than potentially having the plugin break because Certbot was updated before the plugin had made the necessary changes. This comment here should help ensure this. * Add pytest warnings warning. * clarify comment --- pytest.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pytest.ini b/pytest.ini index e09813e52..16aa9a193 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,6 +4,13 @@ [pytest] # In general, all warnings are treated as errors. Here are the exceptions: # 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# Warnings being triggered by our plugins using deprecated features in +# acme/certbot should be fixed by having our plugins no longer using the +# deprecated code rather than adding them to the list of ignored warnings here. +# Fixing things in this way prevents us from shipping packages raising our own +# deprecation warnings and gives time for plugins that don't use the deprecated +# API to propagate, especially for plugins packaged as an external snap, before +# we release breaking changes. filterwarnings = error ignore:decodestring:DeprecationWarning From 9154e7965fe5d9137d68b00752ac997c704968c5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 5 May 2020 09:38:20 -0700 Subject: [PATCH 21/33] drop min certbot coverage (#7972) `tox -e cover` fails for me on macOS. This is due to the differences in the code that is run when on Linux vs. other platforms in `certbot.util` and its tests. Diffing my local coverage with Travis, the only difference in lines missing test coverage is: ``` $ diff travis.txt local.txt < certbot/certbot/util.py 238 24 90% 132-134, 210, 275-281, 318, 330-340, 347, 381-382 > certbot/certbot/util.py 239 34 86% 30, 132-134, 210, 275-281, 301, 305, 316-318, 330-340, 347, 367-373, 381-382 < certbot/tests/util_test.py 392 0 100% > certbot/tests/util_test.py 375 26 93% 483-487, 492-502, 507-514, 548-550, 555-557 ``` I think tests on `master` should not be failing locally for people. While there would be other ways to fix this by adding `# pragma: no cover` lines or writing mocked out tests for other platforms, I personally just think dropping the minimum coverage one percentage point is fine at least for now. --- tox.cover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.cover.py b/tox.cover.py index 4848b2740..dc2ce5226 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -14,7 +14,7 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx'] COVER_THRESHOLDS = { - 'certbot': {'linux': 96, 'windows': 96}, + 'certbot': {'linux': 95, 'windows': 96}, 'acme': {'linux': 100, 'windows': 99}, 'certbot_apache': {'linux': 100, 'windows': 100}, 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, From ca60ad52b93457fecd7246bc09dc1e60a30dca25 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 5 May 2020 12:37:33 -0700 Subject: [PATCH 22/33] Update changelog for 1.4.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 35a657bf2..12f4471d3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.4.0 - master +## 1.4.0 - 2020-05-05 ### Added From 5586ae071ad15e79fa6cd0fd3e0243fe80db906c Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 5 May 2020 13:44:21 -0700 Subject: [PATCH 23/33] Release 1.4.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 41 +++++++----------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/local-oldest-requirements.txt | 4 +- certbot-nginx/setup.py | 6 +-- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 22 ++++++---- certbot/local-oldest-requirements.txt | 2 +- certbot/setup.py | 2 +- letsencrypt-auto | 41 +++++++----------- letsencrypt-auto-source/certbot-auto.asc | 16 +++---- letsencrypt-auto-source/letsencrypt-auto | 26 +++++------ letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++----- 29 files changed, 104 insertions(+), 116 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 356410efe..6794744fd 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 25fb920c2..be3ebc8e6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 0ea3275c3..66f3072ca 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0" +LE_AUTO_VERSION="1.4.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then } BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION" elif [ -f /etc/gentoo-release ]; then - Bootstrap() { - DeprecationBootstrap "Gentoo" BootstrapGentooCommon - } - BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq FreeBSD ; then - Bootstrap() { - DeprecationBootstrap "FreeBSD" BootstrapFreeBsd - } - BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq Darwin ; then - Bootstrap() { - DeprecationBootstrap "macOS" BootstrapMac - } - BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then Bootstrap() { ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon @@ -1540,18 +1531,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.3.0 \ - --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ - --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 -acme==1.3.0 \ - --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ - --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d -certbot-apache==1.3.0 \ - --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ - --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 -certbot-nginx==1.3.0 \ - --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ - --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b +certbot==1.4.0 \ + --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \ + --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96 +acme==1.4.0 \ + --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \ + --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69 +certbot-apache==1.4.0 \ + --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \ + --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640 +certbot-nginx==1.4.0 \ + --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \ + --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index ed8e3f861..5a3a0d298 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.4.0.dev0' +version = '1.4.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9ac16bc67..977c4fe91 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index f998027f9..7754809c3 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 7aef67d75..efe1af54d 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 4a3f863f5..2b78a5c01 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -7,7 +7,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2dced23bf..91080f20f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 7c7acc503..0524985af 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 1b51a781e..8a164d4a2 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 860c40079..dd25b3207 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 83932e140..4380e9eb9 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 459c6d752..9c8360f97 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 5823b237e..3f91d5048 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 94cda6f65..a376ae092 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index f140d3f8d..e8d7b9ed8 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index b57e28577..7fd17c99c 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index 1782f15ba..e6d9f26fd 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. --e acme[dev] --e certbot[dev] +acme[dev]==1.4.0 +certbot[dev]==1.4.0 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0e6deceb3..42ea077e9 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0.dev0' +version = '1.4.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=1.4.0.dev0', - 'certbot>=1.4.0.dev0', + 'acme>=1.4.0', + 'certbot>=1.4.0', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support 'setuptools', diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 0ce7ff6b7..3f349ea03 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.4.0.dev0' +__version__ = '1.4.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 3c2289030..1ba2c0b7e 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.3.0 (certbot(-auto); + "". (default: CertbotACMEClient/1.4.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the user agent are: --duplicate, @@ -188,10 +188,12 @@ security: supported setups (Apache version >= 2.3.3 ). (default: False) --redirect Automatically redirect all HTTP traffic to HTTPS for - the newly authenticated vhost. (default: Ask) + the newly authenticated vhost. (default: redirect + enabled for install and run, disabled for enhance) --no-redirect Do not automatically redirect all HTTP traffic to HTTPS for the newly authenticated vhost. (default: - Ask) + redirect enabled for install and run, disabled for + enhance) --hsts Add the Strict-Transport-Security header to every HTTP response. Forcing browser to always use SSL for the domain. Defends against SSL Stripping. (default: None) @@ -213,8 +215,8 @@ testing: --test-cert, --staging Use the staging server to obtain or revoke test - (invalid) certificates; equivalent to --server https - ://acme-staging-v02.api.letsencrypt.org/directory + (invalid) certificates; equivalent to --server + https://acme-staging-v02.api.letsencrypt.org/directory (default: False) --debug Show tracebacks in case of errors, and allow certbot- auto execution on experimental platforms (default: @@ -319,8 +321,8 @@ renew: of renewed certificate domains (for example, "example.com www.example.com" (default: None) --disable-hook-validation - Ordinarily the commands specified for --pre-hook - /--post-hook/--deploy-hook will be checked for + Ordinarily the commands specified for --pre- + hook/--post-hook/--deploy-hook will be checked for validity, to see if the programs being run are in the $PATH, so that mistakes can be caught early, even when the hooks aren't being run just yet. The validation is @@ -669,7 +671,11 @@ manual: requested when performing an HTTP-01 challenge. An additional cleanup script can also be provided and can use the additional variable $CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth - script. + script.For both authenticator and cleanup script, on HTTP-01 and DNS-01 + challenges,$CERTBOT_REMAINING_CHALLENGES will be equal to the number of + challenges that remain after the current one, and $CERTBOT_ALL_DOMAINS + contains a comma-separated list of all domains that are challenged for the + current certificate. --manual-auth-hook MANUAL_AUTH_HOOK Path or command to execute for the authentication diff --git a/certbot/local-oldest-requirements.txt b/certbot/local-oldest-requirements.txt index 0acc68652..bb4a4851f 100644 --- a/certbot/local-oldest-requirements.txt +++ b/certbot/local-oldest-requirements.txt @@ -1,2 +1,2 @@ # Remember to update setup.py to match the package versions below. --e acme[dev] +acme[dev]==1.4.0 diff --git a/certbot/setup.py b/certbot/setup.py index 143e1a10a..54dcea071 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -36,7 +36,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=1.4.0.dev0', + 'acme>=1.4.0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. diff --git a/letsencrypt-auto b/letsencrypt-auto index 0ea3275c3..66f3072ca 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0" +LE_AUTO_VERSION="1.4.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -910,20 +910,11 @@ elif [ -f /etc/manjaro-release ]; then } BOOTSTRAP_VERSION="BootstrapArchCommon $BOOTSTRAP_ARCH_COMMON_VERSION" elif [ -f /etc/gentoo-release ]; then - Bootstrap() { - DeprecationBootstrap "Gentoo" BootstrapGentooCommon - } - BOOTSTRAP_VERSION="BootstrapGentooCommon $BOOTSTRAP_GENTOO_COMMON_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq FreeBSD ; then - Bootstrap() { - DeprecationBootstrap "FreeBSD" BootstrapFreeBsd - } - BOOTSTRAP_VERSION="BootstrapFreeBsd $BOOTSTRAP_FREEBSD_VERSION" + DEPRECATED_OS=1 elif uname | grep -iq Darwin ; then - Bootstrap() { - DeprecationBootstrap "macOS" BootstrapMac - } - BOOTSTRAP_VERSION="BootstrapMac $BOOTSTRAP_MAC_VERSION" + DEPRECATED_OS=1 elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then Bootstrap() { ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon @@ -1540,18 +1531,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.3.0 \ - --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ - --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 -acme==1.3.0 \ - --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ - --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d -certbot-apache==1.3.0 \ - --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ - --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 -certbot-nginx==1.3.0 \ - --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ - --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b +certbot==1.4.0 \ + --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \ + --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96 +acme==1.4.0 \ + --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \ + --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69 +certbot-apache==1.4.0 \ + --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \ + --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640 +certbot-nginx==1.4.0 \ + --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \ + --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 84473dc30..5ba9184af 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl5ewVUACgkQTRfJlc2X -dfJnZAf+KmxYl1YoP/FlTG5Npb64qaDdxm59SeEVJez6fZh15xq71tRPYR+4xszE -XTeyGt7uAxjYqeiBJU5xBvGC1Veprhj5AbflVOTP+5yiBr9iNWC35zmgaE63UlZ/ -V94sfL0pkax7wLngil7a0OuzUjikzK3gXOqrY8LoUdr4mAA9AhSjajWHmyY3tpDR -84GKrVhybIt0sjy/172VuPPbXZKno/clztkKMZHXNrDeL5jgJ15Va4Ts5FK0j9VT -HQvuazbGkYVCuvlp8Np5ESDje69LCJfPZxl34htoa8WNJoVIOsQWZpoXp5B5huSP -vGrh4LabZ5UDsl+k11ikHBRUpO7E5w== -=IgRH +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl6x0CQACgkQTRfJlc2X +dfJMVwf/dQ+Ap/TvIKVkdeIDcAeqycyBpK7CvGGMfkBVD1FPOXvLAaGRu8jpbtKB +HE2SlCiRW5g+o0iD2zJx+6EMm0hDo64/jK7X7AE04Vz5yolhPujrbxqSMF2CZXZX +vh9qfzRU+05kjYmOElP/JZxAE3mZyPPK04Ii6gseIjU8NEaGinQQm3oFBDqnaZq6 +DMGqvczaT3kTt8Rr3r2/9XQzr8aF+zpBAteAg7ou31b8nK/hugiX1gfdQL3xF7Gu +sRPyU14vZeVvoU8n0G0pSWdV//0eV8KmctbQJaU8amrnrFJubM+PKbsRWGSwMtu3 +5PA9aZbXDAB5iXm4huA8sK8IU76FLg== +=eUK2 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c43e6d6d8..66f3072ca 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.4.0.dev0" +LE_AUTO_VERSION="1.4.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1531,18 +1531,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.3.0 \ - --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ - --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 -acme==1.3.0 \ - --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ - --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d -certbot-apache==1.3.0 \ - --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ - --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 -certbot-nginx==1.3.0 \ - --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ - --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b +certbot==1.4.0 \ + --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \ + --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96 +acme==1.4.0 \ + --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \ + --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69 +certbot-apache==1.4.0 \ + --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \ + --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640 +certbot-nginx==1.4.0 \ + --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \ + --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 8c4f52d6e2992d76012ed4c5e8b65c366e53cf8c..408a3ca5812c39b88236e83d274f164b6b3ae353 100644 GIT binary patch literal 256 zcmV+b0ssE0)lP!l8{k0zs793am(^^XLigFJn)6tHVPFjNS&!LccaqX7;QN+G*vr=<3afjqcx!aWow4Vz9!xLOl)%i0c~Z3CwA-_OEO5KT|ObQS|6o&a0V znQy`mxxYDs`xTU3HV)2zmBLgz&K{SZ>u*={gztRy@BaDG$F7WbzcZGiJJWt2>D!`# zL)!|FsmoUs@s;iEK1~DcziP~F)IVY!2jxDWKNMhrIin;Nq%qCn6QYOBCSdlZ2Q6 literal 256 zcmV+b0ssEUOnRo|t9S`PT@Q6pdEx5)TLaw|tXK(Czj^_>8PhYd%g<>HQ?bGidlv-5 zE}9A&SzM6uK*eeB!}iF{EuInepHwl!!N=ypH>Giu0~gNuz3H_G-xK$*4GGCrIi8a} zc|vYH^EA>Vt1;uU?ETR7;K}DFE1nDBiU-xb=ODbJ#yW{#gBQmTy?oXA)L zay!@Hc;qXdB0+{NAC2*m>{YND9$B?_{$Q;DGoOCVmMhh`)kRLNXuPfjru6g4?PpB5 zE=0kjQwl0Y1m+t}(iNS8e1OH8)e$OZ#|vb^J4)NEd(wxnV%*phX+ZJIy2kBPS7%B+ G&t7>eJc8N) diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index a8e275a45..4b6c4dddb 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==1.3.0 \ - --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ - --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 -acme==1.3.0 \ - --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ - --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d -certbot-apache==1.3.0 \ - --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ - --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 -certbot-nginx==1.3.0 \ - --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ - --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b +certbot==1.4.0 \ + --hash=sha256:5f8be1e6087d2f1f742caf0048b0f46bac8d3a655d038d5355abd1638523d87e \ + --hash=sha256:69b5b7925de0d3b693b00a40bf109d85afb24c7199bf616339d74d59a80d8d96 +acme==1.4.0 \ + --hash=sha256:d2f6799f7fce2414fc1a6753ced91c0ccdf1d0b2cee892c509851db45402fb5b \ + --hash=sha256:f12cb59762e0b833911b87e95cb16e85a162517ba4aa3440594bdf3b8126fc69 +certbot-apache==1.4.0 \ + --hash=sha256:1be1a38cb73e950c5cbff941719d326bfd2d1b4fff17b39b7a27377067cd90a6 \ + --hash=sha256:6067f537deb7f70b979d11ed19846712dbf5c484ca927841805e78d8797b4640 +certbot-nginx==1.4.0 \ + --hash=sha256:8ee1c7201b40bde7d476894fb06bf8ab0c0cd0ba03c0510bc568e8713e801ccc \ + --hash=sha256:44a9f74dee7e2f8a32aafaf793280e8fcd4d50a9ffb7c5ed47a0bc591ce6ecca From 1a72cdecf2ce102251f03ae1abfaade7421cb130 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 5 May 2020 13:44:23 -0700 Subject: [PATCH 24/33] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 12f4471d3..e46d134fb 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.5.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.4.0 - 2020-05-05 ### Added From b224b499869d7022c18fa460547b298d31abe76e Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 5 May 2020 13:44:23 -0700 Subject: [PATCH 25/33] Bump version to 1.5.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 6794744fd..02b81c81c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index be3ebc8e6..d3095153a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 5a3a0d298..95bc276f4 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -5,7 +5,7 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.4.0' +version = '1.5.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 977c4fe91..45ea83880 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 7754809c3..b5b883693 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index efe1af54d..9c4de22fe 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 2b78a5c01..05c2ac90d 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -7,7 +7,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 91080f20f..9c50d5e4d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 0524985af..d06cf7bcb 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 8a164d4a2..a088388ae 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index dd25b3207..9ae74eff8 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 4380e9eb9..4539185ec 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 9c8360f97..48caefe45 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 3f91d5048..f14994708 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index a376ae092..526d64a52 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index e8d7b9ed8..6178d7cbb 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 7fd17c99c..e5b510e83 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 42ea077e9..8f2c9f0e3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.4.0' +version = '1.5.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 3f349ea03..a642b220c 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.4.0' +__version__ = '1.5.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 66f3072ca..4a42a71fd 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.4.0" +LE_AUTO_VERSION="1.5.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From bb6a660b21d94dc4818bdbff7379b3c9cf30c232 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 6 May 2020 21:33:50 +0200 Subject: [PATCH 26/33] Platform-agnostic method to copy owner and permissions between files (#7968) Related to #7649 since @joohoi needs a method to copy owner and permissions together from a source file to a destination file. This PR creates the method `copy_ownership_and_mode()` in `certbot.compat.filesystem` module to achieve this goal. Its behavior is consistent across Linux and Windows in respect to the security model that have been defined for Certbot on Windows. The method behaves globally the same than `copy_ownership_and_apply_mode`, but this time the permissions are extracted from the source file. For Windows it means that the DACL is copied from the source to the destination with the same content. * Create copy_ownership_and_mode to copy both owner and mode from src to dst * Update certbot/tests/compat/filesystem_test.py Co-authored-by: Brad Warren * Fix docstring Co-authored-by: Brad Warren --- certbot/certbot/compat/filesystem.py | 45 +++++++++++++++++++++++++ certbot/tests/compat/filesystem_test.py | 32 ++++++++++++++---- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index b9b6e5cc6..e44e1b32a 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -78,6 +78,35 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): chmod(dst, mode) +# Quite similar to copy_ownership_and_apply_mode, but this time the DACL is copied from +# the source file on Windows. The DACL stays consistent with the dynamic rights of the +# equivalent POSIX mode, because ownership and mode are copied altogether on the destination +# file, so no recomputing of the DACL against the new owner is needed, as it would be +# for a copy_ownership alone method. +def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): + # type: (str, str, bool, bool) -> None + """ + Copy ownership (user and optionally group on Linux) and mode/DACL + from the source to the destination. + :param str src: Path of the source file + :param str dst: Path of the destination file + :param bool copy_user: Copy user if `True` + :param bool copy_group: Copy group if `True` on Linux (has no effect on Windows) + """ + if POSIX_MODE: + # On Linux, we just delegate to chown and chmod. + stats = os.stat(src) + user_id = stats.st_uid if copy_user else -1 + group_id = stats.st_gid if copy_group else -1 + os.chown(dst, user_id, group_id) + chmod(dst, stats.st_mode) + else: + if copy_user: + # There is no group handling in Windows + _copy_win_ownership(src, dst) + _copy_win_mode(src, dst) + + def check_mode(file_path, mode): # type: (str, int) -> bool """ @@ -515,6 +544,9 @@ def _analyze_mode(mode): def _copy_win_ownership(src, dst): + # Resolve symbolic links + src = realpath(src) + security_src = win32security.GetFileSecurity(src, win32security.OWNER_SECURITY_INFORMATION) user_src = security_src.GetSecurityDescriptorOwner() @@ -526,6 +558,19 @@ def _copy_win_ownership(src, dst): win32security.SetFileSecurity(dst, win32security.OWNER_SECURITY_INFORMATION, security_dst) +def _copy_win_mode(src, dst): + # Resolve symbolic links + src = realpath(src) + + # Copy the DACL from src to dst. + security_src = win32security.GetFileSecurity(src, win32security.DACL_SECURITY_INFORMATION) + dacl = security_src.GetSecurityDescriptorDacl() + + security_dst = win32security.GetFileSecurity(dst, win32security.DACL_SECURITY_INFORMATION) + security_dst.SetSecurityDescriptorDacl(1, dacl, 0) + win32security.SetFileSecurity(dst, win32security.DACL_SECURITY_INFORMATION, security_dst) + + def _generate_windows_flags(rights_desc): # Some notes about how each POSIX right is interpreted. # diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 1c2d2df0d..3f6a5ec1d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -280,14 +280,14 @@ class WindowsMkdirTests(test_util.TempDirTestCase): self.assertEqual(original_mkdir, std_os.mkdir) -class OwnershipTest(test_util.TempDirTestCase): - """Tests about copy_ownership_and_apply_mode and has_same_ownership""" +class CopyOwnershipAndModeTest(test_util.TempDirTestCase): + """Tests about copy_ownership_and_apply_mode, copy_ownership_and_mode and has_same_ownership""" def setUp(self): - super(OwnershipTest, self).setUp() + super(CopyOwnershipAndModeTest, self).setUp() self.probe_path = _create_probe(self.tempdir) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') - def test_copy_ownership_windows(self): + def test_copy_ownership_and_apply_mode_windows(self): system = win32security.ConvertStringSidToSid(SYSTEM_SID) security = win32security.SECURITY_ATTRIBUTES().SECURITY_DESCRIPTOR security.SetSecurityDescriptorOwner(system, False) @@ -313,7 +313,7 @@ class OwnershipTest(test_util.TempDirTestCase): if dacl.GetAce(index)[2] == everybody]) @unittest.skipUnless(POSIX_MODE, reason='Test specific to Linux security') - def test_copy_ownership_linux(self): + def test_copy_ownership_and_apply_mode_linux(self): with mock.patch('os.chown') as mock_chown: with mock.patch('os.chmod') as mock_chmod: with mock.patch('os.stat') as mock_stat: @@ -334,6 +334,24 @@ class OwnershipTest(test_util.TempDirTestCase): self.assertTrue(filesystem.has_same_ownership(path1, path2)) + @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') + def test_copy_ownership_and_mode_windows(self): + src = self.probe_path + dst = _create_probe(self.tempdir, name='dst') + + filesystem.chmod(src, 0o700) + self.assertTrue(filesystem.check_mode(src, 0o700)) + self.assertTrue(filesystem.check_mode(dst, 0o744)) + + # Checking an actual change of owner is tricky during a unit test, since we do not know + # if any user exists beside the current one. So we mock _copy_win_ownership. It's behavior + # have been checked theoretically with test_copy_ownership_and_apply_mode_windows. + with mock.patch('certbot.compat.filesystem._copy_win_ownership') as mock_copy_owner: + filesystem.copy_ownership_and_mode(src, dst) + + mock_copy_owner.assert_called_once_with(src, dst) + self.assertTrue(filesystem.check_mode(dst, 0o700)) + class CheckPermissionsTest(test_util.TempDirTestCase): """Tests relative to functions that check modes.""" @@ -537,9 +555,9 @@ def _set_owner(target, security_owner, user): target, win32security.OWNER_SECURITY_INFORMATION, security_owner) -def _create_probe(tempdir): +def _create_probe(tempdir, name='probe'): filesystem.chmod(tempdir, 0o744) - probe_path = os.path.join(tempdir, 'probe') + probe_path = os.path.join(tempdir, name) util.safe_open(probe_path, 'w', chmod=0o744).close() return probe_path From 71e3d82e47696daa24c5f07bd2cb8c553b40babd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 6 May 2020 14:28:15 -0700 Subject: [PATCH 27/33] Remove passthrough because its no longer needed (#7956) --- snap/snapcraft.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2e4f68c00..3b5d98f2d 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -32,9 +32,8 @@ apps: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" - passthrough: - # Run approximately twice a day with randomization - timer: 00:00~24:00/2 + # Run approximately twice a day with randomization + timer: 00:00~24:00/2 parts: python-augeas: From bcf33c665932160c2fe446e46e42c15b57c05a81 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Thu, 14 May 2020 08:55:35 +1000 Subject: [PATCH 28/33] ocsp: add support for public key hash ResponderIDs (#7989) For both cases where the the response is signed by the issuer, or by a delegated OCSP signer. Resolves #7986 --- certbot/CHANGELOG.md | 3 ++- certbot/certbot/ocsp.py | 9 +++++++-- certbot/tests/ocsp_test.py | 31 +++++++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index e46d134fb..fcb68bc66 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -14,7 +14,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Add support for OCSP responses which use a public key hash ResponderID, fixing + interoperability with Sectigo CAs. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 863c5f163..51ada012a 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -256,7 +256,11 @@ def _check_ocsp_response(response_ocsp, request_ocsp, issuer_cert, cert_path): def _check_ocsp_response_signature(response_ocsp, issuer_cert, cert_path): """Verify an OCSP response signature against certificate issuer or responder""" - if response_ocsp.responder_name == issuer_cert.subject: + def _key_hash(cert): + return x509.SubjectKeyIdentifier.from_public_key(cert.public_key()).digest + + if response_ocsp.responder_name == issuer_cert.subject or \ + response_ocsp.responder_key_hash == _key_hash(issuer_cert): # Case where the OCSP responder is also the certificate issuer logger.debug('OCSP response for certificate %s is signed by the certificate\'s issuer.', cert_path) @@ -267,7 +271,8 @@ def _check_ocsp_response_signature(response_ocsp, issuer_cert, cert_path): cert_path) responder_certs = [cert for cert in response_ocsp.certificates - if cert.subject == response_ocsp.responder_name] + if response_ocsp.responder_name == cert.subject or \ + response_ocsp.responder_key_hash == _key_hash(cert)] if not responder_certs: raise AssertionError('no matching responder certificate could be found') diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index af54844cf..fe89fff9f 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -182,13 +182,23 @@ class OSCPTestCryptography(unittest.TestCase): with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks: + # OCSP response with ResponseID as Name mocks['mock_response'].return_value.responder_name = issuer.subject + mocks['mock_response'].return_value.responder_key_hash = None self.checker.ocsp_revoked(self.cert_obj) + # OCSP response with ResponseID as KeyHash + key_hash = x509.SubjectKeyIdentifier.from_public_key(issuer.public_key()).digest + mocks['mock_response'].return_value.responder_name = None + mocks['mock_response'].return_value.responder_key_hash = key_hash + self.checker.ocsp_revoked(self.cert_obj) + # Here responder and issuer are the same. So only the signature of the OCSP # response is checked (using the issuer/responder public key). - self.assertEqual(mocks['mock_check'].call_count, 1) - self.assertEqual(mocks['mock_check'].call_args[0][0].public_numbers(), - issuer.public_key().public_numbers()) + self.assertEqual(mocks['mock_check'].call_count, 2) + self.assertEqual(mocks['mock_check'].call_args_list[0][0][0].public_numbers(), + issuer.public_key().public_numbers()) + self.assertEqual(mocks['mock_check'].call_args_list[1][0][0].public_numbers(), + issuer.public_key().public_numbers()) def test_responder_is_authorized_delegate(self): issuer = x509.load_pem_x509_certificate( @@ -198,15 +208,28 @@ class OSCPTestCryptography(unittest.TestCase): with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks: + # OCSP response with ResponseID as Name + mocks['mock_response'].return_value.responder_name = responder.subject + mocks['mock_response'].return_value.responder_key_hash = None self.checker.ocsp_revoked(self.cert_obj) + # OCSP response with ResponseID as KeyHash + key_hash = x509.SubjectKeyIdentifier.from_public_key(responder.public_key()).digest + mocks['mock_response'].return_value.responder_name = None + mocks['mock_response'].return_value.responder_key_hash = key_hash + self.checker.ocsp_revoked(self.cert_obj) + # Here responder and issuer are not the same. Two signatures will be checked then, # first to verify the responder cert (using the issuer public key), second to # to verify the OCSP response itself (using the responder public key). - self.assertEqual(mocks['mock_check'].call_count, 2) + self.assertEqual(mocks['mock_check'].call_count, 4) self.assertEqual(mocks['mock_check'].call_args_list[0][0][0].public_numbers(), issuer.public_key().public_numbers()) self.assertEqual(mocks['mock_check'].call_args_list[1][0][0].public_numbers(), responder.public_key().public_numbers()) + self.assertEqual(mocks['mock_check'].call_args_list[2][0][0].public_numbers(), + issuer.public_key().public_numbers()) + self.assertEqual(mocks['mock_check'].call_args_list[3][0][0].public_numbers(), + responder.public_key().public_numbers()) def test_revoke_resiliency(self): # Server return an invalid HTTP response From d230dcafebf1ade14bc5f9bf5954282df42fca71 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 18 May 2020 09:31:15 -0700 Subject: [PATCH 29/33] Print cause of exit in red text. --- certbot/certbot/_internal/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 0a492ba55..21996548d 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -323,7 +323,10 @@ def post_arg_parse_except_hook(exc_type, exc_value, trace, debug, log_path): else: logger.debug('Exiting abnormally:', exc_info=exc_info) if issubclass(exc_type, errors.Error): - sys.exit(exc_value) + # Use logger to print the error message to take advantage of + # our logger printing warnings and errors in red text. + logger.error(exc_value) + sys.exit(1) logger.error('An unexpected error occurred:') if messages.is_acme_error(exc_value): # Remove the ACME error prefix from the exception From 0e59c6ba1baf2c32f229a9b8c68fd457ba136617 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 18 May 2020 10:17:32 -0700 Subject: [PATCH 30/33] handle more cases --- certbot/certbot/_internal/log.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 21996548d..3b87f9bca 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -322,10 +322,10 @@ def post_arg_parse_except_hook(exc_type, exc_value, trace, debug, log_path): logger.error('Exiting abnormally:', exc_info=exc_info) else: logger.debug('Exiting abnormally:', exc_info=exc_info) + # Use logger to print the error message to take advantage of + # our logger printing warnings and errors in red text. if issubclass(exc_type, errors.Error): - # Use logger to print the error message to take advantage of - # our logger printing warnings and errors in red text. - logger.error(exc_value) + logger.error(str(exc_value)) sys.exit(1) logger.error('An unexpected error occurred:') if messages.is_acme_error(exc_value): @@ -333,7 +333,12 @@ def post_arg_parse_except_hook(exc_type, exc_value, trace, debug, log_path): _, _, exc_str = str(exc_value).partition(':: ') logger.error(exc_str) else: - traceback.print_exception(exc_type, exc_value, None) + output = traceback.format_exception_only(exc_type, exc_value) + # format_exception_only returns a list of strings each + # terminated by a newline. We combine them into one string + # and remove the final newline before passing it to + # logger.error. + logger.error(''.join(output).rstrip()) exit_with_log_path(log_path) From 96e003d1a37e445775795bb9b91f455231cc7d77 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 May 2020 14:28:41 -0700 Subject: [PATCH 31/33] mention python3-venv in docs (#8006) The error message from `python3 -m venv` when you don't have `python3-venv` installed is pretty good, but lets skip the failure and make sure it is installed the first time. --- tests/letstest/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/letstest/README.md b/tests/letstest/README.md index 5bd326e2a..4cf6c83c3 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -15,9 +15,10 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you -have Python 3 installed, you can use requirements.txt to create a virtual -environment with a known set of dependencies by running: +These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're +on a Debian based system, make sure you also have the python3-venv package +installed. If you have Python 3 installed, you can use requirements.txt to +create a virtual environment with a known set of dependencies by running: ``` python3 -m venv venv3 . ./venv3/bin/activate From 4eb9a71a4c078d0ffc49eb2fa1fa06bf3b41f82f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 19 May 2020 14:29:28 -0700 Subject: [PATCH 32/33] remove quay cruft (#8003) Our README still has links to our old quay.io builds which we shutdown a while ago. See #4343. This PR simply removes the old stray links. --- certbot/README.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/certbot/README.rst b/certbot/README.rst index 5ed74f247..39da06c8a 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/ ACME working area in github: https://github.com/ietf-wg-acme/acme -|build-status| |container| +|build-status| .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status -.. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status - :target: https://quay.io/repository/letsencrypt/letsencrypt - :alt: Docker Repository on Quay.io - .. Do not modify this comment unless you know what you're doing. tag:links-end System Requirements From 3ea5170647fa9a1f6d5644a86811680e82ca05fa Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 19 May 2020 15:34:21 -0700 Subject: [PATCH 33/33] Error out earlier in apache installer when mod_ssl is not available (#7984) * Error out in apache installer when mod_ssl is not available * Update to MisconfigurationError and add/fix tests * Remove error cases we no longer hit and associated test * mock out function to have consistent error across machines * improve changelog message * only check key in modules list, not value --- .../certbot_apache/_internal/configurator.py | 20 +++------ certbot-apache/tests/configurator_test.py | 45 +++++-------------- certbot/CHANGELOG.md | 2 +- 3 files changed, 16 insertions(+), 51 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index dbfc15468..10b3c52d9 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -595,6 +595,11 @@ class ApacheConfigurator(common.Installer): # cert_key... can all be parsed appropriately self.prepare_server_https("443") + # If we haven't managed to enable mod_ssl by this point, error out + if "ssl_module" not in self.parser.modules: + raise errors.MisconfigurationError("Could not find ssl_module; " + "not installing certificate.") + # Add directives and remove duplicates self._add_dummy_ssl_directives(vhost.path) self._clean_vhost(vhost) @@ -609,21 +614,6 @@ class ApacheConfigurator(common.Installer): path["chain_path"] = self.parser.find_dir( "SSLCertificateChainFile", None, vhost.path) - # Handle errors when certificate/key directives cannot be found - if not path["cert_path"]: - logger.warning( - "Cannot find an SSLCertificateFile directive in %s. " - "VirtualHost was not modified", vhost.path) - raise errors.PluginError( - "Unable to find an SSLCertificateFile directive") - elif not path["cert_key"]: - logger.warning( - "Cannot find an SSLCertificateKeyFile directive for " - "certificate in %s. VirtualHost was not modified", vhost.path) - raise errors.PluginError( - "Unable to find an SSLCertificateKeyFile directive for " - "certificate") - logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 8fd3cb750..b5dcc464d 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -455,41 +455,6 @@ class MultipleVhostsTest(util.ApacheTest): "SSLCertificateChainFile", "two/cert_chain.pem", self.vh_truth[1].path)) - def test_deploy_cert_invalid_vhost(self): - """For test cases where the `ApacheConfigurator` class' `_deploy_cert` - method is called with an invalid vhost parameter. Currently this tests - that a PluginError is appropriately raised when important directives - are missing in an SSL module.""" - self.config.parser.modules["ssl_module"] = None - self.config.parser.modules["mod_ssl.c"] = None - self.config.parser.modules["socache_shmcb_module"] = None - - def side_effect(*args): - """Mocks case where an SSLCertificateFile directive can be found - but an SSLCertificateKeyFile directive is missing.""" - if "SSLCertificateFile" in args: - return ["example/cert.pem"] - return [] - - mock_find_dir = mock.MagicMock(return_value=[]) - mock_find_dir.side_effect = side_effect - - self.config.parser.find_dir = mock_find_dir - - # Get the default 443 vhost - self.config.assoc["random.demo"] = self.vh_truth[1] - - self.assertRaises( - errors.PluginError, self.config.deploy_cert, "random.demo", - "example/cert.pem", "example/key.pem", "example/cert_chain.pem") - - # Remove side_effect to mock case where both SSLCertificateFile - # and SSLCertificateKeyFile directives are missing - self.config.parser.find_dir.side_effect = None - self.assertRaises( - errors.PluginError, self.config.deploy_cert, "random.demo", - "example/cert.pem", "example/key.pem", "example/cert_chain.pem") - def test_is_name_vhost(self): addr = obj.Addr.fromstring("*:80") self.assertTrue(self.config.is_name_vhost(addr)) @@ -1349,6 +1314,16 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue(mock_add.called) shutil.rmtree(tmp_path) + def test_deploy_cert_no_mod_ssl(self): + # Create + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) + self.config.parser.modules["socache_shmcb_module"] = None + self.config.prepare_server_https = mock.Mock() + + self.assertRaises(errors.MisconfigurationError, self.config.deploy_cert, + "encryption-example.demo", "example/cert.pem", "example/key.pem", + "example/cert_chain.pem", "example/fullchain.pem") + @mock.patch("certbot_apache._internal.parser.ApacheParser.parsed_in_original") def test_choose_vhost_and_servername_addition_parsed(self, mock_parsed): ret_vh = self.vh_truth[8] diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ad8a04e07..9f5ce6226 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -10,7 +10,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* Improved error message in apache installer when mod_ssl is not available. ### Fixed