From c43f4fe518b48465b5cb886f2b1df1f3aed17616 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Feb 2021 13:20:04 -0800 Subject: [PATCH 01/99] upgrade to 3.8.8 (#8682) Fixes https://github.com/certbot/certbot/issues/8681. https://python-security.readthedocs.io/vuln/ctypes-buffer-overflow-pycarg_repr.html is the best resource I found linking to the original Python bug, when each Python branch was fixed, etc. --- windows-installer/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 0684b3c25..60834e7e5 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -9,7 +9,7 @@ import sys import tempfile import time -PYTHON_VERSION = (3, 8, 6) +PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' From c3d6fca3eb1f24616bb784865662adfa048f9bf9 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 24 Feb 2021 00:29:52 +0100 Subject: [PATCH 02/99] Make certbot constraint file independent from certbot-auto + update cryptography (#8649) * Refactor to not depend on certbot-auto dependencies pinning anymore * Update constraints * Replaces references * Upgrade AWS dependencies pinning * Fix script * Fix Windows installer builds * Fixing sdists letstest script * Pin cryptography on 3.1.1 specifically for RHEL/CentOS 7 to avoid build failures during test_sdists test. * Finish fix * Fix VERSION_ID in RHEL 7 --- snap/snapcraft.yaml | 6 +- tests/letstest/scripts/test_sdists.sh | 22 +- tools/certbot_constraints.txt | 262 ++++++++++++++++++ tools/dev_constraints.txt | 6 +- tools/docker/core/Dockerfile | 4 - tools/pip_install.py | 6 +- .../rebuild_certbot_constraints.py | 52 ++-- tools/snap/generate_dnsplugins_all.sh | 2 +- windows-installer/construct.py | 18 +- 9 files changed, 325 insertions(+), 53 deletions(-) create mode 100644 tools/certbot_constraints.txt rename letsencrypt-auto-source/rebuild_dependencies.py => tools/rebuild_certbot_constraints.py (84%) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c9061ecb3..d53fba88b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -73,10 +73,10 @@ parts: build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade - # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the + # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is # used. This is done to let these constraints be applied not only on the certbot package - # build, but also on any isolated build that pip could trigger when building wheels for + # build, but also on any isolated build that pip could trigger when building wheels for # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt override-build: | @@ -85,7 +85,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/letsencrypt-auto-source/pieces/dependency-requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index aa12d5610..becdd6d9a 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,13 +12,21 @@ sudo $BOOTSTRAP_SCRIPT # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed # packages makes pip error out. -python3 tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > requirements.txt -# We also strip out the requirement for enum34 because it cannot be installed -# in newer versions of Python 3, tools/strip_hashes.py removes the environment -# marker that'd normally prevent it from being installed, and this package is -# not needed for any OS tested here. -sed -i '/enum34/d' requirements.txt -CERTBOT_PIP_NO_BINARY=:all: tools/venv.py --requirement requirements.txt +python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt +python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt + +# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been +# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then + sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt + sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt +fi + +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt new file mode 100644 index 000000000..77bfef9db --- /dev/null +++ b/tools/certbot_constraints.txt @@ -0,0 +1,262 @@ +# This is the flattened list of pinned packages to build certbot deployable artifacts. +# To generate this, do (with docker and package hashin installed): +# ``` +# tools/rebuild_certbot_contraints.py \ +# tools/certbot_constraints.txt +# ``` +# If you want to update a single dependency, run commands similar to these: +# ``` +# pip install hashin +# hashin -r dependency-requirements.txt cryptography==1.5.2 +# ``` +ConfigArgParse==1.2.3 \ + --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc +certifi==2020.12.5 \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +cffi==1.14.4 \ + --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ + --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ + --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ + --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ + --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ + --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ + --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ + --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ + --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ + --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ + --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ + --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ + --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ + --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ + --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ + --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ + --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ + --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ + --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ + --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ + --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ + --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ + --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ + --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ + --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ + --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ + --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ + --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ + --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ + --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ + --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ + --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ + --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ + --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ + --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ + --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ + --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f +chardet==4.0.0 \ + --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ + --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +configobj==5.0.6 \ + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==3.3.2 \ + --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ + --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ + --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ + --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ + --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ + --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ + --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ + --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ + --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ + --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ + --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ + --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ + --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ + --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da +distro==1.5.0 \ + --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ + --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +josepy==1.6.0 \ + --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ + --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 +parsedatetime==2.6 \ + --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ + --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b +pyOpenSSL==20.0.1 \ + --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ + --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b +pyRFC3339==1.1 \ + --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ + --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a +pycparser==2.20 \ + --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ + --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b +python-augeas==0.5.0 \ + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +pytz==2021.1 \ + --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ + --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 +requests==2.25.1 \ + --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ + --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +six==1.15.0 \ + --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ + --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +urllib3==1.26.3 \ + --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ + --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 +zope.component==4.6.2 \ + --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ + --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 +zope.deferredimport==4.3.1 \ + --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ + --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a +zope.deprecation==4.4.0 \ + --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ + --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 +zope.event==4.5.0 \ + --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ + --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 +zope.hookable==5.0.1 \ + --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ + --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ + --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ + --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ + --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ + --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ + --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ + --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ + --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ + --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ + --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ + --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ + --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ + --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ + --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ + --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ + --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ + --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ + --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ + --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ + --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ + --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ + --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ + --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ + --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ + --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ + --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ + --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ + --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ + --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ + --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ + --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ + --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ + --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ + --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ + --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ + --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ + --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ + --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ + --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc +zope.interface==5.2.0 \ + --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ + --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ + --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ + --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ + --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ + --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ + --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ + --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ + --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ + --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ + --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ + --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ + --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ + --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ + --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ + --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ + --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ + --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ + --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ + --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ + --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ + --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ + --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ + --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ + --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ + --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ + --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ + --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ + --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ + --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ + --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ + --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ + --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ + --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ + --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ + --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ + --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ + --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ + --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ + --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ + --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ + --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ + --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ + --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ + --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ + --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ + --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ + --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ + --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ + --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ + --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ + --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 +zope.proxy==4.3.5 \ + --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ + --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ + --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ + --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ + --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ + --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ + --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ + --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ + --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ + --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ + --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ + --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ + --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ + --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ + --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ + --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ + --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ + --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ + --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ + --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ + --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ + --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ + --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ + --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ + --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ + --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ + --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ + --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ + --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ + --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ + --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ + --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ + --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ + --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ + --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ + --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ + --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ + --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ + --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ + --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f5140f9c7..10308bd39 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). +# files during tests (eg. tools/certbot_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 @@ -16,8 +16,8 @@ backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.1.6 -boto3==1.11.7 -botocore==1.14.7 +boto3==1.17.4 +botocore==1.20.4 cached-property==1.5.1 cloudflare==2.3.1 configparser==3.7.4 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 0d3626853..d2ebe3537 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,10 +14,6 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -# We keep the relative path to the requirements file the same because, as of -# writing this, tools/pip_install.py is used in the Dockerfile for Certbot -# plugins and this script expects to find the requirements file there. -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot diff --git a/tools/pip_install.py b/tools/pip_install.py index c1c81482b..e06650ff2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -57,7 +57,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + repo_path, 'tools/certbot_constraints.txt')) with open(certbot_requirements, 'r') as fd: certbot_reqs = fd.readlines() with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: @@ -76,8 +76,7 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Here is the order by increasing priority: # 1) The general development constraints (tools/dev_constraints.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot-auto's dependency-requirements.txt + pipstrap's constraints - # for the normal processing) + # certbot_constraints.txt + pipstrap's constraints for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] if requirements: @@ -134,6 +133,7 @@ def main(args): pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' .format(requirements)) + print(' '.join(args)) pip_install_with_print(' '.join(args), env=env) diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/tools/rebuild_certbot_constraints.py similarity index 84% rename from letsencrypt-auto-source/rebuild_dependencies.py rename to tools/rebuild_certbot_constraints.py index 864394661..f5e5d3ca7 100755 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/tools/rebuild_certbot_constraints.py @@ -4,12 +4,12 @@ Gather and consolidate the up-to-date dependencies available and required to ins on various Linux distributions. It generates a requirements file contained the pinned and hashed versions, ready to be used by pip to install the certbot dependencies. -This script is typically used to update the certbot-requirements.txt file of certbot-auto. +This script is typically used to update the certbot_constraints.txt file. To achieve its purpose, this script will start a certbot installation with unpinned dependencies, then gather them, on various distributions started as Docker containers. -Usage: letsencrypt-auto-source/rebuild_dependencies new_requirements.txt +Usage: tools/rebuild_certbot_constraints.py new_requirements.txt NB1: Docker must be installed on the machine running this script. NB2: Python library 'hashin' must be installed on the machine running this script. @@ -26,52 +26,41 @@ import argparse # The list of docker distributions to test dependencies against with. DISTRIBUTION_LIST = [ - 'ubuntu:18.04', 'ubuntu:16.04', - 'debian:stretch', - 'centos:7', 'centos:6', - 'opensuse/leap:15', - 'fedora:29', + 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', + 'centos:8', 'centos:7', 'fedora:29', ] # These constraints will be added while gathering dependencies on each distribution. # It can be used because a particular version for a package is required for any reason, # or to solve a version conflict between two distributions requirements. AUTHORITATIVE_CONSTRAINTS = { - # Using an older version of mock here prevents regressions of #5276. - 'mock': '1.3.0', # Too touchy to move to a new version. And will be removed soon # in favor of pure python parser for Apache. 'python-augeas': '0.5.0', - # Package enum34 needs to be explicitly limited to Python2.x, in order to avoid - # certbot-auto failures on Python 3.6+ which enum34 doesn't support. See #5456. - 'enum34': '1.1.10; python_version < \'3.4\'', - # Cryptography 2.9+ drops support for OpenSSL 1.0.1, but we still want to support it - # for officially supported non-x86_64 ancient distributions like RHEL 6. - 'cryptography': '2.8', - # Parsedatetime 2.6 is broken on Python 2.7, see https://github.com/bear/parsedatetime/issues/246 - 'parsedatetime': '2.5', + # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and + # this needs some work on the snap builds. + 'cryptography': '3.3.2', } -# ./certbot/letsencrypt-auto-source/rebuild_dependencies.py (2 levels from certbot root path) +# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) # The script will be used to gather dependencies for a given distribution. -# - certbot-auto is used to install relevant OS packages, and set up an initial venv +# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv # - then this venv is used to consistently construct an empty new venv -# - once pipstraped, this new venv pip-installs certbot runtime (including apache/nginx), +# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), # without pinned dependencies, and respecting input authoritative requirements # - `certbot plugins` is called to check we have a healthy environment # - finally current set of dependencies is extracted out of the docker using pip freeze SCRIPT = r"""#!/bin/sh -set -e +set -ex cd /tmp/certbot -letsencrypt-auto-source/letsencrypt-auto --install-only -n -PYVER=`/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` +tests/letstest/scripts/bootstrap_os_packages.sh -/opt/eff.org/certbot/venv/bin/python letsencrypt-auto-source/pieces/create_venv.py /tmp/venv "$PYVER" 1 +python3 -m venv /tmp/venv -/tmp/venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py +/tmp/venv/bin/python tools/pipstrap.py /tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt /tmp/venv/bin/certbot plugins /tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt @@ -109,6 +98,7 @@ def _requirements_from_one_distribution(distribution, verbose): '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, + '--network=host', '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), '-v', '{0}:/tmp/workspace'.format(workspace), '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), @@ -158,7 +148,7 @@ def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, dis """ for line in requirements_file_lines: match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and match: + if not line.startswith('-e') and not line.startswith('#') and match: package, version = match.groups() if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: dependencies_map.setdefault(package, []).append((version, distribution)) @@ -215,11 +205,11 @@ def _write_requirements(dest_file, requirements, conflicts): print('===> Calculating hashes for the requirement file.') _write_to(dest_file, '''\ -# This is the flattened list of packages certbot-auto installs. +# This is the flattened list of pinned packages to build certbot deployable artifacts. # To generate this, do (with docker and package hashin installed): # ``` -# letsencrypt-auto-source/rebuild_dependencies.py \\ -# letsencrypt-auto-source/pieces/dependency-requirements.txt +# tools/rebuild_certbot_contraints.py \\ +# tools/certbot_constraints.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` @@ -264,8 +254,8 @@ def _gather_dependencies(dest_file, verbose): if __name__ == '__main__': parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot-auto, ' - 'validated against several OS distributions using Docker.')) + description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' + ' artifacts, validated against several OS distributions using Docker.')) parser.add_argument('requirements_path', help='path for the generated requirements file') parser.add_argument('--verbose', '-v', action='store_true', diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 40404bf9b..976b0dd7b 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -10,7 +10,7 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index 60834e7e5..eb199a7e1 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -2,6 +2,7 @@ import contextlib import ctypes import os +import re import shutil import struct import subprocess @@ -52,6 +53,21 @@ def _compile_wheels(repo_path, build_path, venv_python): command.extend(wheels_project) subprocess.check_call(command, env=env) + # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where + # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible + # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable + # Application Binary Interface, this naming convention makes pynsist falsely think that the + # wheel is compatible with Python 3.6 only. + # Let's trick pynsist by renaming the wheel until this is fixed upstream. + for file in os.listdir(wheels_path): + # Given that our Python version is 3.8, this rename files like + # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl + renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' + .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) + print(renamed) + if renamed != file: + os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) + def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') @@ -63,7 +79,7 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'dependency-requirements.txt') + reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') constraints_certbot = subprocess.check_output( [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], From ae3ed200c09f2d1913ec69c33a2604ae1fe32eb5 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Feb 2021 14:51:57 -0800 Subject: [PATCH 03/99] Remove check for 'fake' in issuer name when renewing certs (#8685) Fixes #8680. We seem to have no existing testing code anywhere in this vicinity, so figured I'd get this up quickly then work on that. Manual tests (renew staging certificate, should allow it; renew non-staging cert as staging, should error) passed. * Remove check for 'fake' in issuer name when renewing certs * Change fake issuer name to make sure we're not relying on it anywhere --- certbot/certbot/_internal/renewal.py | 5 +---- certbot/tests/main_test.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 9fe9cb546..7533c8c6b 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -312,12 +312,9 @@ def _avoid_invalidating_lineage(config, lineage, original_server): contents = the_file.read() latest_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, contents) - # all our test certificates are from happy hacker fake CA, though maybe one day - # we should test more methodically - now_valid = "fake" not in repr(latest_cert.get_issuer()).lower() if util.is_staging(config.server): - if not util.is_staging(original_server) or now_valid: + if not util.is_staging(original_server): if not config.break_my_certs: names = ", ".join(lineage.names()) raise errors.Error( diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index ddd911c8d..785433585 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1053,7 +1053,7 @@ class MainTest(test_util.ConfigTestCase): mock_get_utility().notification.side_effect = write_msg with mock.patch('certbot._internal.main.renewal.OpenSSL') as mock_ssl: mock_latest = mock.MagicMock() - mock_latest.get_issuer.return_value = "Fake fake" + mock_latest.get_issuer.return_value = "Artificial pretend" mock_ssl.crypto.load_certificate.return_value = mock_latest with mock.patch('certbot._internal.main.renewal.crypto_util') \ as mock_crypto_util: From 025eb16c7a07b5ed3286a17c7a53bb00020b657c Mon Sep 17 00:00:00 2001 From: alexzorin Date: Fri, 26 Feb 2021 05:22:40 +1100 Subject: [PATCH 04/99] docs: rewrite "Revoking certificates" (#8657) * docs: rewrite "Revoking certificates" - `--cert-name` is supported since a long time ago - `--delete-after-revoke` is default - Mention that non-default `--server` must be specified - Document difference between acme key/cert key revocation methods - Reshuffle text to keep more important things earlier * minor edits * remove revocation note * remove "preauthorization" revocation method * rewrite deletion note --- certbot/docs/using.rst | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index ab8d64d79..1d97caecc 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -474,29 +474,37 @@ like Revoking certificates --------------------- -If your account key has been compromised or you otherwise need to revoke a certificate, -use the ``revoke`` command to do so. Note that the ``revoke`` command takes the certificate path -(ending in ``cert.pem``), not a certificate name or domain. Example:: +If you need to revoke a certificate, use the ``revoke`` subcommand to do so. - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem +A certificate may be revoked by providing its name (see ``certbot certificates``) or by providing +its path directly:: + + certbot revoke --cert-name example.com + + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem + +If the certificate being revoked was obtained via the ``--staging``, ``--test-cert`` or a non-default ``--server`` flag, +that flag must be passed to the ``revoke`` subcommand. + +.. note:: After revocation, Certbot will (by default) ask whether you want to **delete** the certificate. + Unless deleted, Certbot will try to renew revoked certificates the next time ``certbot renew`` runs. You can also specify the reason for revoking your certificate by using the ``reason`` flag. Reasons include ``unspecified`` which is the default, as well as ``keycompromise``, ``affiliationchanged``, ``superseded``, and ``cessationofoperation``:: - certbot revoke --cert-path /etc/letsencrypt/live/CERTNAME/cert.pem --reason keycompromise + certbot revoke --cert-name example.com --reason keycompromise -Additionally, if a certificate -is a test certificate obtained via the ``--staging`` or ``--test-cert`` flag, that flag must be passed to the -``revoke`` subcommand. -Once a certificate is revoked (or for other certificate management tasks), all of a certificate's -relevant files can be removed from the system with the ``delete`` subcommand:: +Revoking by account key or certificate private key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - certbot delete --cert-name example.com +By default, Certbot will try revoke the certificate using your ACME account key. If the certificate was created from +the same ACME account, the revocation will be successful. -.. note:: If you don't use ``delete`` to remove the certificate completely, it will be renewed automatically at the next renewal event. +If you instead have the corresponding private key file to the certificate you wish to revoke, use ``--key-path`` to perform the +revocation from any ACME account:: -.. note:: Revoking a certificate will have no effect on the rate limit imposed by the Let's Encrypt server. + certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem --key-path /etc/letsencrypt/live/example.com/privkey.pem .. _renewal: From f71298f6614078454532f82aeafbe5d06d3a7cdb Mon Sep 17 00:00:00 2001 From: alexzorin Date: Fri, 26 Feb 2021 06:32:21 +1100 Subject: [PATCH 05/99] cli: make key_path and cert_path always be a str (#8687) There is some code in [`_paths_parser`](https://github.com/certbot/certbot/blob/ae3ed200c09f2d1913ec69c33a2604ae1fe32eb5/certbot/certbot/_internal/cli/paths_parser.py#L17-L34) which has the effect of varying the value type of `config.cert_path` and `config.key_path` based on the CLI verb. When the verb is `revoke`, the type is a tuple `(path: str, contents: bytes)`, otherwise it is a single `str` representing the file path. (I wasn't able to find a written reason as to why it works this way). This commit removes that special `revoke` case and ensures it is always a `str`. Why change it now? I am trying to write some changes and there's some code in `cert_manager` which only works if the verb is `revoke`, you hack `config.cert_path` to be a tuple beforehand, or you [(not actually in `master`) try sniff for the value type](https://github.com/certbot/certbot/blob/49911afaa62ade1fca4c4ec407f817720cb354e3/certbot/certbot/_internal/cert_manager.py#L224-L227). I have a bad feeling about such workarounds. I would prefer to just make these variables simpler to use, but I'm open to opinions. In addition to the test suites, I've manually tested `revoke` (including by `--key-path`) and `install`. Are there other places I may have missed? Unblocks #8636 and #8671. --- certbot/certbot/_internal/cert_manager.py | 4 ++-- certbot/certbot/_internal/cli/paths_parser.py | 20 +++++++++---------- certbot/certbot/_internal/main.py | 15 ++++++++------ certbot/certbot/_internal/storage.py | 7 ++----- certbot/tests/cert_manager_test.py | 16 +++++++-------- certbot/tests/main_test.py | 7 ++----- certbot/tests/storage_test.py | 4 ++-- 7 files changed, 34 insertions(+), 39 deletions(-) diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index dfbe4b538..ee2bd6254 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -223,7 +223,7 @@ def cert_path_to_lineage(cli_config): """ acceptable_matches = _acceptable_matches() match = match_and_check_overlaps(cli_config, acceptable_matches, - lambda x: cli_config.cert_path[0], lambda x: x.lineagename) + lambda x: cli_config.cert_path, lambda x: x.lineagename) return match[0] @@ -254,7 +254,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func matched = _search_lineages(cli_config, find_matches, [], acceptable_matches) if not matched: - raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path[0])) + raise errors.Error("No match found for cert-path {0}!".format(cli_config.cert_path)) elif len(matched) > 1: raise errors.OverlappingMatchFound() return matched diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 62f5e224d..04b3725b9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -2,7 +2,6 @@ paths for certificates""" from certbot.compat import os from certbot._internal.cli import ( - read_file, flag_default, config_help ) @@ -14,22 +13,21 @@ def _paths_parser(helpful): if verb == "help": verb = helpful.help_arg - cph = "Path to where certificate is saved (with auth --csr), installed from, or revoked." - sections = ["paths", "install", "revoke", "certonly", "manage"] + cpkwargs = { + "type": os.path.abspath, + "help": "Path to where certificate is saved (with certonly --csr), installed " + "from, or revoked" + } if verb == "certonly": - add(sections, "--cert-path", type=os.path.abspath, - default=flag_default("auth_cert_path"), help=cph) + cpkwargs["default"] = flag_default("auth_cert_path") elif verb == "revoke": - add(sections, "--cert-path", type=read_file, required=False, help=cph) - else: - add(sections, "--cert-path", type=os.path.abspath, help=cph) + cpkwargs["required"] = False + add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" if verb in ("install", "revoke"): section = verb - # revoke --key-path reads a file, install --key-path takes a string - add(section, "--key-path", - type=((verb == "revoke" and read_file) or os.path.abspath), + add(section, "--key-path", type=os.path.abspath, help="Path to private key for certificate installation " "or revocation (if account key is missing)") diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index b9b6b16f6..36ebb1a50 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1098,15 +1098,18 @@ def revoke(config, unused_plugins): if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using certificate key %s", - config.cert_path[0], config.key_path[0]) - crypto_util.verify_cert_matches_priv_key(config.cert_path[0], config.key_path[0]) - key = jose.JWK.load(config.key_path[1]) + config.cert_path, config.key_path) + crypto_util.verify_cert_matches_priv_key(config.cert_path, config.key_path) + with open(config.key_path, 'rb') as f: + key = jose.JWK.load(f.read()) acme = client.acme_from_config_key(config, key) else: # revocation by account key - logger.debug("Revoking %s using Account Key", config.cert_path[0]) + logger.debug("Revoking %s using Account Key", config.cert_path) acc, _ = _determine_account(config) acme = client.acme_from_config_key(config, acc.key, acc.regr) - cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0] + + with open(config.cert_path, 'rb') as f: + cert = crypto_util.pyopenssl_load_certificate(f.read())[0] logger.debug("Reason code for revocation: %s", config.reason) try: acme.revoke(jose.ComparableX509(cert), config.reason) @@ -1114,7 +1117,7 @@ def revoke(config, unused_plugins): except acme_errors.ClientError as e: return str(e) - display_ops.success_revocation(config.cert_path[0]) + display_ops.success_revocation(config.cert_path) return None diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 690567a17..d04bc5774 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -58,7 +58,7 @@ def renewal_file_for_certname(config, certname): return path -def cert_path_for_cert_name(config, cert_name): +def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ If `--cert-name` was specified, but you need a value for `--cert-path`. :param `configuration.NamespaceConfig` config: parsed command line arguments @@ -66,10 +66,7 @@ def cert_path_for_cert_name(config, cert_name): """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - fullchain_path = configobj.ConfigObj(cert_name_implied_conf)["fullchain"] - with open(fullchain_path) as f: - cert_path = (fullchain_path, f.read()) - return cert_path + return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] def config_with_defaults(config=None): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index b26c1f624..ba6cfddc3 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -526,7 +526,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config): from certbot._internal.cert_manager import cert_path_to_lineage @@ -556,21 +556,21 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): mock_acceptable_matches.return_value = [lambda x: x.cert_path] test_cert_path = os.path.join(self.config.config_dir, 'live', 'example.org', 'cert.pem') - self.config.cert_path = (test_cert_path, '') + self.config.cert_path = test_cert_path self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_cert(self, mock_acceptable_matches): # Also this and the next test check that the regex of _archive_files is working. - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', 'example.org', - 'cert11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', 'example.org', + 'cert11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'cert')] self.assertEqual('example.org', self._call(self.config)) @mock.patch('certbot._internal.cert_manager._acceptable_matches') def test_options_archive_fullchain(self, mock_acceptable_matches): - self.config.cert_path = (os.path.join(self.config.config_dir, 'archive', - 'example.org', 'fullchain11.pem'), '') + self.config.cert_path = os.path.join(self.config.config_dir, 'archive', + 'example.org', 'fullchain11.pem') mock_acceptable_matches.return_value = [lambda x: self._archive_files(x, 'fullchain')] self.assertEqual('example.org', self._call(self.config)) @@ -586,7 +586,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, acceptable_matches, match_func, rv_func): from certbot._internal.cert_manager import match_and_check_overlaps @@ -595,7 +595,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): def test_basic_match(self): from certbot._internal.cert_manager import _acceptable_matches self.assertEqual(['example.org'], self._call(self.config, _acceptable_matches(), - lambda x: self.config.cert_path[0], lambda x: x.lineagename)) + lambda x: self.config.cert_path, lambda x: x.lineagename)) @mock.patch('certbot._internal.cert_manager._search_lineages') def test_no_matches(self, mock_search_lineages): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 785433585..2d5d88947 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -287,10 +287,7 @@ class RevokeTest(test_util.TempDirTestCase): super(RevokeTest, self).setUp() shutil.copy(CERT_PATH, self.tempdir) - self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, - 'cert_512.pem')) - with open(self.tmp_cert_path, 'r') as f: - self.tmp_cert = (self.tmp_cert_path, f.read()) + self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) patches = [ mock.patch('acme.client.BackwardsCompatibleClientV2'), @@ -349,7 +346,7 @@ class RevokeTest(test_util.TempDirTestCase): def test_revoke_by_certname(self, mock_cert_path_for_cert_name, mock_delete_if_appropriate): args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert + mock_cert_path_for_cert_name.return_value = self.tmp_cert_path mock_delete_if_appropriate.return_value = False self._call(args) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index abd496c8d..0f696bc34 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -934,14 +934,14 @@ class CertPathForCertNameTest(BaseRenewableCertTest): self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', 'fullchain.pem') - self.config.cert_path = (self.fullchain, '') + self.config.cert_path = self.fullchain def _call(self, cli_config, certname): from certbot._internal.storage import cert_path_for_cert_name return cert_path_for_cert_name(cli_config, certname) def test_simple_cert_name(self): - self.assertEqual(self._call(self.config, 'example.org'), (self.fullchain, 'fullchain')) + self.assertEqual(self._call(self.config, 'example.org'), self.fullchain) def test_no_such_cert_name(self): self.assertRaises(errors.CertStorageError, self._call, self.config, 'fake-example.org') From e742cfaa21176f1f5a3e43c8022b9c899bc5c205 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Feb 2021 13:39:55 -0800 Subject: [PATCH 06/99] dont set required to False (#8689) --- certbot/certbot/_internal/cli/paths_parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 04b3725b9..43ee41c88 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -20,8 +20,6 @@ def _paths_parser(helpful): } if verb == "certonly": cpkwargs["default"] = flag_default("auth_cert_path") - elif verb == "revoke": - cpkwargs["required"] = False add(["paths", "install", "revoke", "certonly", "manage"], "--cert-path", **cpkwargs) section = "paths" From 135187f03e18eebcb32b05c4ecb76a4552aed883 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:50:54 +0100 Subject: [PATCH 07/99] Python 3 obsoletes explicit __ne__ methods (#8676) This shouldn't be needed as of Python 3+. https://stackoverflow.com/questions/4352244/should-ne-be-implemented-as-the-negation-of-eq-in-python#30676267 --- acme/acme/messages.py | 3 --- certbot-apache/certbot_apache/_internal/obj.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 44ecb143c..61fd89dfd 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -150,9 +150,6 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore def __hash__(self): return hash((self.__class__, self.name)) - def __ne__(self, other): - return not self == other - class Status(_Constant): """ACME "status" field.""" diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 498766744..3cd5f0ff2 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -20,9 +20,6 @@ class Addr(common.Addr): self.is_wildcard() and other.is_wildcard())) return False - def __ne__(self, other): - return not self.__eq__(other) - def __repr__(self): return "certbot_apache._internal.obj.Addr(" + repr(self.tup) + ")" @@ -191,9 +188,6 @@ class VirtualHost(object): return False - def __ne__(self, other): - return not self.__eq__(other) - def __hash__(self): return hash((self.filep, self.path, tuple(self.addrs), tuple(self.get_names()), From 67c2b27af7d7dc04c246be4f9a1d4cd29a3099ca Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:59:00 +0100 Subject: [PATCH 08/99] Stop inheriting from object. It's unneeded on Python 3+. (#8675) --- acme/acme/client.py | 6 +++--- acme/acme/crypto_util.py | 6 +++--- acme/acme/magic_typing.py | 2 +- acme/acme/messages.py | 2 +- acme/acme/mixins.py | 2 +- acme/acme/standalone.py | 2 +- certbot-apache/certbot_apache/_internal/dualparser.py | 2 +- certbot-apache/certbot_apache/_internal/obj.py | 2 +- certbot-apache/certbot_apache/_internal/parser.py | 2 +- .../certbot_integration_tests/certbot_tests/context.py | 2 +- certbot-ci/certbot_integration_tests/utils/acme_server.py | 2 +- certbot-ci/certbot_integration_tests/utils/dns_server.py | 2 +- .../certbot_compatibility_test/configurators/common.py | 2 +- .../certbot_compatibility_test/validator.py | 2 +- .../certbot_dns_cloudflare/_internal/dns_cloudflare.py | 2 +- .../certbot_dns_digitalocean/_internal/dns_digitalocean.py | 2 +- .../certbot_dns_google/_internal/dns_google.py | 2 +- certbot-dns-google/tests/dns_google_test.py | 2 +- .../certbot_dns_rfc2136/_internal/dns_rfc2136.py | 2 +- certbot-nginx/certbot_nginx/_internal/nginxparser.py | 4 ++-- certbot-nginx/certbot_nginx/_internal/obj.py | 2 +- certbot-nginx/certbot_nginx/_internal/parser.py | 2 +- certbot-nginx/certbot_nginx/_internal/parser_obj.py | 2 +- certbot/certbot/_internal/account.py | 2 +- certbot/certbot/_internal/auth_handler.py | 2 +- certbot/certbot/_internal/cli/cli_utils.py | 4 ++-- certbot/certbot/_internal/cli/helpful.py | 2 +- certbot/certbot/_internal/client.py | 4 ++-- certbot/certbot/_internal/configuration.py | 2 +- certbot/certbot/_internal/display/completer.py | 2 +- certbot/certbot/_internal/error_handler.py | 2 +- certbot/certbot/_internal/lock.py | 4 ++-- certbot/certbot/_internal/plugins/disco.py | 2 +- certbot/certbot/_internal/plugins/standalone.py | 2 +- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/display/util.py | 4 ++-- certbot/certbot/ocsp.py | 2 +- certbot/certbot/plugins/common.py | 6 +++--- certbot/certbot/plugins/dns_common.py | 2 +- certbot/certbot/plugins/dns_common_lexicon.py | 2 +- certbot/certbot/plugins/dns_test_common.py | 2 +- certbot/certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/certbot/plugins/storage.py | 2 +- certbot/certbot/reverter.py | 2 +- certbot/certbot/tests/util.py | 2 +- certbot/tests/plugins/dns_common_test.py | 2 +- tests/letstest/multitester.py | 2 +- 47 files changed, 58 insertions(+), 58 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3f8c550f..33e515124 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -33,7 +33,7 @@ DEFAULT_NETWORK_TIMEOUT = 45 DER_CONTENT_TYPE = 'application/pkix-cert' -class ClientBase(object): +class ClientBase: """ACME client base object. :ivar messages.Directory directory: @@ -795,7 +795,7 @@ class ClientV2(ClientBase): if 'rel' in l and 'url' in l and l['rel'] == relation_type] -class BackwardsCompatibleClientV2(object): +class BackwardsCompatibleClientV2: """ACME client wrapper that tends towards V2-style calls, but supports V1 servers. @@ -938,7 +938,7 @@ class BackwardsCompatibleClientV2(object): return self.client.external_account_required() -class ClientNetwork(object): +class ClientNetwork: """Wrapper around requests that signs POSTs for authentication. Also adds user agent, and handles Content-Type. diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 4b58db328..a14737053 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class _DefaultCertSelection(object): +class _DefaultCertSelection: def __init__(self, certs): self.certs = certs @@ -36,7 +36,7 @@ class _DefaultCertSelection(object): return self.certs.get(server_name, None) -class SSLSocket(object): # pylint: disable=too-few-public-methods +class SSLSocket: # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. @@ -93,7 +93,7 @@ class SSLSocket(object): # pylint: disable=too-few-public-methods new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) - class FakeConnection(object): + class FakeConnection: """Fake OpenSSL.SSL.Connection.""" # pylint: disable=missing-function-docstring diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 388fc4a58..0e60d3203 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -2,7 +2,7 @@ import sys -class TypingClass(object): +class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): return None diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 61fd89dfd..0d73037ae 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -274,7 +274,7 @@ class ResourceBody(jose.JSONObjectWithFields): """ACME Resource Body.""" -class ExternalAccountBinding(object): +class ExternalAccountBinding: """ACME External Account Binding""" @classmethod diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 1cd050ccc..0e1e0977c 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -1,7 +1,7 @@ """Useful mixins for Challenge and Resource objects""" -class VersionedLEACMEMixin(object): +class VersionedLEACMEMixin: """This mixin stores the version of Let's Encrypt's endpoint being used.""" @property def le_acme_version(self): diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 94397f0de..f5bc548b6 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -53,7 +53,7 @@ class ACMEServerMixin: allow_reuse_address = True -class BaseDualNetworkedServers(object): +class BaseDualNetworkedServers: """Base class for a pair of IPv6 and IPv4 servers that tries to do everything it's asked for both servers, but where failures in one server don't affect the other. diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index eef8f2a0e..1ba23e92f 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -4,7 +4,7 @@ from certbot_apache._internal import augeasparser from certbot_apache._internal import apacheparser -class DualNodeBase(object): +class DualNodeBase: """ Dual parser interface for in development testing. This is used as the base class for dual parser interface classes. This class handles runtime attribute value assertions.""" diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 3cd5f0ff2..e2fe48cf8 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -95,7 +95,7 @@ class Addr(common.Addr): return self.get_addr_obj(port) -class VirtualHost(object): +class VirtualHost: """Represents an Apache Virtualhost. :ivar str filep: file path of VH diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 75be0833f..fdef167bc 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -16,7 +16,7 @@ from certbot_apache._internal import constants logger = logging.getLogger(__name__) -class ApacheParser(object): +class ApacheParser: """Class handles the fine details of parsing the Apache Configuration. .. todo:: Make parsing general... remove sites-available etc... diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b9854b402..b052e375d 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -7,7 +7,7 @@ import tempfile from certbot_integration_tests.utils import certbot_call -class IntegrationTestsContext(object): +class IntegrationTestsContext: """General fixture describing a certbot integration tests context""" def __init__(self, request): self.request = request diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index bbbdd196b..846e2d071 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -23,7 +23,7 @@ from certbot_integration_tests.utils import proxy from certbot_integration_tests.utils.constants import * -class ACMEServer(object): +class ACMEServer: """ ACMEServer configures and handles the lifecycle of an ACME CA server and an HTTP reverse proxy instance, to allow parallel execution of integration tests against the unique http-01 port diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 416f6567e..62a58275e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -21,7 +21,7 @@ BIND_BIND_ADDRESS = ("127.0.0.1", 45953) BIND_TEST_QUERY = bytearray.fromhex("0011cb37000000010000000000000000010003") -class DNSServer(object): +class DNSServer: """ DNSServer configures and handles the lifetime of an RFC2136-capable server. DNServer provides access to the dns_xdist parameter, listing the address and port diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 34aa9133f..bf768f8f8 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -11,7 +11,7 @@ from certbot_compatibility_test import util logger = logging.getLogger(__name__) -class Proxy(object): +class Proxy: """A common base for compatibility test configurators""" @classmethod diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator.py b/certbot-compatibility-test/certbot_compatibility_test/validator.py index 9a082523a..226b8585b 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator.py @@ -10,7 +10,7 @@ from acme import errors as acme_errors logger = logging.getLogger(__name__) -class Validator(object): +class Validator: """Collection of functions to test a live webserver's configuration""" def certificate(self, cert, name, alt_host=None, port=443): diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 5978af85c..c896a6e3a 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -85,7 +85,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) -class _CloudflareClient(object): +class _CloudflareClient: """ Encapsulates all communication with the Cloudflare API. """ diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index e0c9561a2..5893df9c2 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -54,7 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _DigitalOceanClient(self.credentials.conf('token')) -class _DigitalOceanClient(object): +class _DigitalOceanClient: """ Encapsulates all communication with the DigitalOcean API. """ diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 4b0d91463..363c5e079 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -76,7 +76,7 @@ class Authenticator(dns_common.DNSAuthenticator): return _GoogleClient(self.conf('credentials')) -class _GoogleClient(object): +class _GoogleClient: """ Encapsulates all communication with the Google Cloud DNS API. """ diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 396a6c8bd..7de5f1d67 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -401,7 +401,7 @@ class GoogleClientTest(unittest.TestCase): self.assertRaises(ServerNotFoundError, _GoogleClient.get_project_id) -class DummyResponse(object): +class DummyResponse: """ Dummy object to create a fake HTTPResponse (the actual one requires a socket and we only need the status attribute) diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index 57e9506f2..adebdd963 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -88,7 +88,7 @@ class Authenticator(dns_common.DNSAuthenticator): dns.tsig.HMAC_MD5)) -class _RFC2136Client(object): +class _RFC2136Client: """ Encapsulates all communication with the target DNS server. """ diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index f043b0e4d..a51302fae 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -19,7 +19,7 @@ from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) -class RawNginxParser(object): +class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -69,7 +69,7 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() -class RawNginxDumper(object): +class RawNginxDumper: """A class that dumps nginx configuration from the provided tree.""" def __init__(self, blocks): self.blocks = blocks diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 2dd02f180..1511cba6d 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -143,7 +143,7 @@ class Addr(common.Addr): return False -class VirtualHost(object): +class VirtualHost: """Represents an Nginx Virtualhost. :ivar str filep: file path of VH diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index fe5d7bb31..17a8c652a 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -21,7 +21,7 @@ from certbot_nginx._internal import obj logger = logging.getLogger(__name__) -class NginxParser(object): +class NginxParser: """Class handles the fine details of parsing the Nginx Configuration. :ivar str root: Normalized absolute path to the server root diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index d616a1a99..e55d48dc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -13,7 +13,7 @@ COMMENT = " managed by Certbot" COMMENT_BLOCK = ["#", COMMENT] -class Parsable(object): +class Parsable: """ Abstract base class for "Parsable" objects whose underlying representation is a tree of lists. diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index dbe111fbc..b2d50297e 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -24,7 +24,7 @@ from certbot.compat import filesystem logger = logging.getLogger(__name__) -class Account(object): +class Account: """ACME protocol registration. :ivar .RegistrationResource regr: Registration Resource diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 7ea2a1de8..17bf75225 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -19,7 +19,7 @@ from certbot._internal import error_handler logger = logging.getLogger(__name__) -class AuthHandler(object): +class AuthHandler: """ACME Authorization Handler for a client. :ivar auth: Authenticator capable of solving diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index a0ddce38f..5bdbbe02c 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -12,7 +12,7 @@ from certbot.compat import os from certbot._internal import constants -class _Default(object): +class _Default: """A class to use as a default to detect if a value is set by a user""" def __bool__(self): @@ -66,7 +66,7 @@ def config_help(name, hidden=False): return field.__doc__ -class HelpfulArgumentGroup(object): +class HelpfulArgumentGroup: """Emulates an argparse group for use with HelpfulArgumentParser. This class is used in the add_group method of HelpfulArgumentParser. diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 80afe5db4..5eaec978b 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -41,7 +41,7 @@ from certbot._internal.cli import ( ) -class HelpfulArgumentParser(object): +class HelpfulArgumentParser: """Argparse Wrapper. This class wraps argparse, adding the ability to make --help less diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 5dc62580e..f2fc06937 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -93,7 +93,7 @@ def ua_flags(config): flags.append("hook") return " ".join(flags) -class DummyConfig(object): +class DummyConfig: "Shim for computing a sample user agent." def __init__(self): self.authenticator = "XXX" @@ -227,7 +227,7 @@ def perform_registration(acme, config, tos_cb): raise -class Client(object): +class Client: """Certbot's client. :ivar .IConfig config: Client configuration. diff --git a/certbot/certbot/_internal/configuration.py b/certbot/certbot/_internal/configuration.py index 1b5cf5da7..aee0022b8 100644 --- a/certbot/certbot/_internal/configuration.py +++ b/certbot/certbot/_internal/configuration.py @@ -13,7 +13,7 @@ from certbot.compat import os @zope.interface.implementer(interfaces.IConfig) -class NamespaceConfig(object): +class NamespaceConfig: """Configuration wrapper around :class:`argparse.Namespace`. For more documentation, including available attributes, please see diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index 03719862b..a6c984195 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -8,7 +8,7 @@ except ImportError: import certbot._internal.display.dummy_readline as readline # type: ignore -class Completer(object): +class Completer: """Provides Tab completion when prompting users for a path. This class is meant to be used with readline to provide Tab diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 60fb287a6..05af9d837 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -45,7 +45,7 @@ else: _SIGNALS = [] -class ErrorHandler(object): +class ErrorHandler: """Context manager for running code that must be cleaned up on failure. The context manager allows you to register functions that will be called diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index aa0b80eaa..32142fe44 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -38,7 +38,7 @@ def lock_dir(dir_path): return LockFile(os.path.join(dir_path, '.certbot.lock')) -class LockFile(object): +class LockFile: """ Platform independent file lock system. LockFile accepts a parameter, the path to a file acting as a lock. Once the LockFile, @@ -97,7 +97,7 @@ class LockFile(object): return self._lock_mechanism.is_locked() -class _BaseLockMechanism(object): +class _BaseLockMechanism: def __init__(self, path): # type: (str) -> None """ diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d19fdd3ef..8adf6d4d2 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -39,7 +39,7 @@ PREFIX_FREE_DISTRIBUTIONS = [ """Distributions for which prefix will be omitted.""" -class PluginEntryPoint(object): +class PluginEntryPoint: """Plugin entry point.""" # this object is mutable, don't allow it to be hashed! diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 60c9558cb..d5d9fd2ec 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: Set[achallenges.KeyAuthorizationAnnotatedChallenge] ] -class ServerManager(object): +class ServerManager: """Standalone servers manager. Manager for `ACMEServer` and `ACMETLSServer` instances. diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 64c0fbd6d..f43f0e7af 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IReporter) -class Reporter(object): +class Reporter: """Collects and displays information to the user. :ivar `queue.PriorityQueue` messages: Messages to be displayed to diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 9da981892..f26ec468f 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -111,7 +111,7 @@ def notify(msg): @zope.interface.implementer(interfaces.IDisplay) -class FileDisplay(object): +class FileDisplay: """File-based display.""" # see https://github.com/certbot/certbot/issues/3915 @@ -478,7 +478,7 @@ def assert_valid_call(prompt, default, cli_flag, force_interactive): @zope.interface.implementer(interfaces.IDisplay) -class NoninteractiveDisplay(object): +class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index b63338e2e..dcbf0381a 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -36,7 +36,7 @@ except (ImportError, AttributeError): # pragma: no cover logger = logging.getLogger(__name__) -class RevocationChecker(object): +class RevocationChecker: """This class figures out OCSP checking on this system, and performs it.""" def __init__(self, enforce_openssl_binary_usage=False): diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index d3fb5b7dc..489c75a3d 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -40,7 +40,7 @@ hostname_regex = re.compile( @zope.interface.implementer(interfaces.IPlugin) -class Plugin(object): +class Plugin: """Generic plugin.""" # provider is not inherited, subclasses must define it on their own # @zope.interface.provider(interfaces.IPluginFactory) @@ -201,7 +201,7 @@ class Installer(Plugin): constants.ALL_SSL_DHPARAMS_HASHES) -class Addr(object): +class Addr: r"""Represents an virtual host address. :param str addr: addr part of vhost address @@ -299,7 +299,7 @@ class Addr(object): return result -class ChallengePerformer(object): +class ChallengePerformer: """Abstract base for challenge performers. :ivar configurator: Authenticator and installer plugin diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 245b7dc05..5c0fbcba9 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -233,7 +233,7 @@ class DNSAuthenticator(common.Plugin): raise errors.PluginError('{0} required to proceed.'.format(label)) -class CredentialsConfiguration(object): +class CredentialsConfiguration: """Represents a user-supplied filed which stores API credentials.""" def __init__(self, filename, mapper=lambda x: x): diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index c3d80ca29..a29509b79 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -23,7 +23,7 @@ except ImportError: logger = logging.getLogger(__name__) -class LexiconClient(object): +class LexiconClient: """ Encapsulates all communication with a DNS provider via Lexicon. """ diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index a3a26a7a1..1affcae4e 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -17,7 +17,7 @@ DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) -class BaseAuthenticatorTest(object): +class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 1bef06042..adeb1a268 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -34,7 +34,7 @@ class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): self.assertEqual(expected, self.mock_client.mock_calls) -class BaseLexiconClientTest(object): +class BaseLexiconClientTest: DOMAIN_NOT_FOUND = Exception('No domain found') GENERIC_ERROR = RequestException LOGIN_ERROR = HTTPError('400 Client Error: ...') diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index f3ed14dce..abef534f9 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -11,7 +11,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class PluginStorage(object): +class PluginStorage: """Class implementing storage functionality for plugins""" def __init__(self, config, classkey): diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index be9d78a11..363215dd0 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -17,7 +17,7 @@ from certbot.compat import os logger = logging.getLogger(__name__) -class Reverter(object): +class Reverter: """Reverter Class - save and revert configuration checkpoints. This class can be used by the plugins, especially Installers, to diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 558bfbed3..78236fa06 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -185,7 +185,7 @@ def patch_get_utility_with_stdout(target='zope.component.getUtility', return mock.patch(target, new=freezable_mock) -class FreezableMock(object): +class FreezableMock: """Mock object with the ability to freeze attributes. This class works like a regular mock.MagicMock object, except diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 993f3b461..31761e986 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -29,7 +29,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen def more_info(self): # pylint: disable=missing-docstring,no-self-use return 'A fake authenticator for testing.' - class _FakeConfig(object): + class _FakeConfig: fake_propagation_seconds = 0 fake_config_key = 1 fake_other_key = None diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 83a92e6dc..75dd06ad6 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -95,7 +95,7 @@ SECURITY_GROUP_NAME = 'certbot-security-group' SENTINEL = None #queue kill signal SUBNET_NAME = 'certbot-subnet' -class Status(object): +class Status: """Possible statuses of client tests.""" PASS = 'pass' FAIL = 'fail' From b0e35c694e3741cb7f9c581ec25f2be1c406b53a Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 25 Feb 2021 23:59:11 +0100 Subject: [PATCH 09/99] Remove import fallback of urllib2 in tests/modification-check. (#8677) --- tests/modification-check.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/modification-check.py b/tests/modification-check.py index 7a69fb1db..357b25350 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -7,16 +7,13 @@ import shutil import subprocess import sys import tempfile +from urllib.request import urlretrieve -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve def find_repo_path(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines +# We do not use filecmp.cmp to take advantage of universal newlines # handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. # As a consequence, this function will not work correctly if executed by Python 2.x on Windows. # But it will work correctly on Linux for any version, because every file tested will be LF. @@ -50,10 +47,10 @@ def validate_scripts_content(repo_path, temp_cwd): errors = True else: shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'certbot-auto'), os.path.join(temp_cwd, 'local-auto')) shutil.copy(os.path.normpath(os.path.join( - repo_path, + repo_path, 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) # Compare file against current version in the target branch @@ -72,7 +69,7 @@ def validate_scripts_content(repo_path, temp_cwd): latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', + [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( os.path.join(temp_cwd, 'letsencrypt-auto'), From 540fd6db93db317ffbeba6d210b92dd830255715 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Fri, 26 Feb 2021 00:41:05 +0100 Subject: [PATCH 10/99] Dictionaries are ordered by insert by default on Python 3.6. (#8678) --- certbot/certbot/_internal/plugins/disco.py | 3 +-- certbot/certbot/util.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 8adf6d4d2..1e1e00982 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,5 +1,4 @@ """Utilities for plugins discovery and selection.""" -import collections import itertools import logging import sys @@ -209,7 +208,7 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - self._plugins = collections.OrderedDict(sorted(plugins.items())) + self._plugins = dict(sorted(plugins.items())) @classmethod def find_all(cls): diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 7bbc02f5f..ffd3a65f2 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -4,7 +4,6 @@ import argparse import atexit import collections -from collections import OrderedDict import distutils.version import errno import logging @@ -16,6 +15,7 @@ import sys import configargparse +from acme.magic_typing import Dict from acme.magic_typing import Text from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -58,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = OrderedDict() # type: OrderedDict[str, lock.LockFile] +_LOCKS = {} # type: Dict[str, lock.LockFile] def env_no_snap_for_external_calls(): From d3ca6af9824d4715f4312724e32a26c270696db8 Mon Sep 17 00:00:00 2001 From: Yuma Mihira Date: Fri, 26 Feb 2021 09:30:48 +0900 Subject: [PATCH 11/99] Insert new line before "More details about these changes can be found on our GitHub repo." (#8645) Fixing #8634. It's my first time contributing to this repository, if there's something wrong please let me know. Before this fix ``` $ python3 extract_changelog.py 1.12.0 ... ### Fixed * Fixed the apache component on openSUSE Tumbleweed which no longer provides an apache2ctl symlink and uses apachectl instead. * Fixed a typo in `certbot/crypto_util.py` causing an error upon attempting `secp521r1` key generation More details about these changes can be found on our GitHub repo. ``` After this fix ``` $ python3 extract_changelog.py 1.12.0 ... ### Fixed * Fixed the apache component on openSUSE Tumbleweed which no longer provides an apache2ctl symlink and uses apachectl instead. * Fixed a typo in `certbot/crypto_util.py` causing an error upon attempting `secp521r1` key generation More details about these changes can be found on our GitHub repo. ``` --- tools/extract_changelog.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py index fb0b849aa..bd718191a 100755 --- a/tools/extract_changelog.py +++ b/tools/extract_changelog.py @@ -24,7 +24,7 @@ def main(): i = 0 while i < len(lines): if section_pattern.match(lines[i]): - i = i + 1 + i = i + 2 while i < len(lines): if NEW_SECTION_PATTERN.match(lines[i]): break @@ -32,8 +32,6 @@ def main(): i = i + 1 i = i + 1 - changelog = [entry for entry in changelog if entry] - print('\n'.join(changelog)) From fab9bfd878952baf98c5e3098db5f975399e0479 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Sat, 27 Feb 2021 08:43:22 +1100 Subject: [PATCH 12/99] nginx: authenticate all matching vhosts for HTTP01 (#8663) * nginx: authenticate all matching vhosts for HTTP01 Previously, the nginx authenticator would set up the HTTP-01 challenge response on a single HTTP vhost which matched the challenge domain. The nginx authenticator will now set the challenge response on every vhost which matches the challenge domain, including duplicates and HTTPS vhosts. This makes the authenticator usable behind a CDN where all origin traffic is performed over HTTPS and also makes the authenticator work more reliably against "invalid" nginx configurations, such as those where there are duplicate vhosts. * some typos * dont authenticate the same vhost twice One vhost may appear in both the HTTP and HTTPS vhost lists. Use a set() to avoid trying to mod the same vhost twice. * fix type annotations * rewrite changelog entry --- .../certbot_nginx/_internal/configurator.py | 94 ++++++++++++++----- .../certbot_nginx/_internal/http_01.py | 54 ++++++----- .../certbot_nginx/_internal/parser.py | 6 +- certbot-nginx/tests/configurator_test.py | 18 +++- certbot-nginx/tests/http_01_test.py | 45 ++++++++- certbot-nginx/tests/parser_test.py | 5 +- .../testdata/etc_nginx/sites-enabled/both.com | 32 +++++++ certbot/CHANGELOG.md | 2 + 8 files changed, 197 insertions(+), 59 deletions(-) create mode 100644 certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 15fbe61f7..8a3b8078f 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines """Nginx Configuration""" from distutils.version import LooseVersion import logging @@ -15,8 +16,10 @@ from acme import challenges from acme import crypto_util as acme_crypto_util from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Text +from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -105,7 +108,7 @@ class NginxConfigurator(common.Installer): self.save_notes = "" # For creating new vhosts if no names match - self.new_vhost = None + self.new_vhost: Optional[obj.VirtualHost] = None # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() @@ -116,7 +119,7 @@ class NginxConfigurator(common.Installer): self._chall_out = 0 # These will be set in the prepare function - self.parser = None + self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -377,10 +380,13 @@ class NginxConfigurator(common.Installer): ipv6only_present = True return (ipv6_active, ipv6only_present) - def _vhost_from_duplicated_default(self, domain, allow_port_mismatch, port): + def _vhost_from_duplicated_default(self, domain: str, allow_port_mismatch: bool, port: str + ) -> obj.VirtualHost: """if allow_port_mismatch is False, only server blocks with matching ports will be used as a default server block template. """ + assert self.parser is not None # prepare should already have been called here + if self.new_vhost is None: default_vhost = self._get_default_vhost(domain, allow_port_mismatch, port) self.new_vhost = self.parser.duplicate_vhost(default_vhost, @@ -509,7 +515,7 @@ class NginxConfigurator(common.Installer): match['rank'] += NO_SSL_MODIFIER return sorted(matches, key=lambda x: x['rank']) - def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False): + def choose_redirect_vhosts(self, target_name: str, port: str) -> List[obj.VirtualHost]: """Chooses a single virtual host for redirect enhancement. Chooses the vhost most closely matching target_name that is @@ -523,9 +529,6 @@ class NginxConfigurator(common.Installer): :param str target_name: domain name :param str port: port number - :param bool create_if_no_match: If we should create a new vhost from default - when there is no match found. If we can't choose a default, raise a - MisconfigurationError. :returns: vhosts associated with name :rtype: list of :class:`~certbot_nginx._internal.obj.VirtualHost` @@ -538,32 +541,75 @@ class NginxConfigurator(common.Installer): else: matches = self._get_redirect_ranked_matches(target_name, port) vhosts = [x for x in [self._select_best_name_match(matches)]if x is not None] - if not vhosts and create_if_no_match: - vhosts = [self._vhost_from_duplicated_default(target_name, False, port)] return vhosts - def _port_matches(self, test_port, matching_port): + def choose_auth_vhosts(self, target_name: str) -> Tuple[List[obj.VirtualHost], + List[obj.VirtualHost]]: + """Returns a list of HTTP and HTTPS vhosts with a server_name matching target_name. + + If no HTTP vhost exists, one will be cloned from the default vhost. If that fails, no HTTP + vhost will be returned. + + :param str target_name: non-wildcard domain name + + :returns: tuple of HTTP and HTTPS virtualhosts + :rtype: tuple of :class:`~certbot_nginx._internal.obj.VirtualHost` + + """ + vhosts = [m['vhost'] for m in self._get_ranked_matches(target_name) if m and 'vhost' in m] + http_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.http01_port), False)] + https_vhosts = [vh for vh in vhosts if + self._vhost_listening(vh, str(self.config.https_port), True)] + + # If no HTTP vhost matches, try create one from the default_server on http01_port. + if not http_vhosts: + try: + http_vhosts = [self._vhost_from_duplicated_default(target_name, False, + str(self.config.http01_port))] + except errors.MisconfigurationError: + http_vhosts = [] + + return http_vhosts, https_vhosts + + def _port_matches(self, test_port: str, matching_port: str) -> bool: # test_port is a number, matching is a number or "" or None if matching_port == "" or matching_port is None: # if no port is specified, Nginx defaults to listening on port 80. return test_port == self.DEFAULT_LISTEN_PORT return test_port == matching_port - def _vhost_listening_on_port_no_ssl(self, vhost, port): - found_matching_port = False - if not vhost.addrs: - # if there are no listen directives at all, Nginx defaults to - # listening on port 80. - found_matching_port = (port == self.DEFAULT_LISTEN_PORT) - else: - for addr in vhost.addrs: - if self._port_matches(port, addr.get_port()) and not addr.ssl: - found_matching_port = True + def _vhost_listening(self, vhost: obj.VirtualHost, port: str, ssl: bool) -> bool: + """Tests whether a vhost has an address listening on a port with SSL enabled or disabled. - if found_matching_port: - # make sure we don't have an 'ssl on' directive - return not self.parser.has_ssl_on_directive(vhost) - return False + :param `obj.VirtualHost` vhost: The vhost whose addresses will be tested + :param port str: The port number as a string that the address should be bound to + :param bool ssl: Whether SSL should be enabled or disabled on the address + + :returns: Whether the vhost has an address listening on the port and protocol. + :rtype: bool + + """ + assert self.parser is not None # prepare should already have been called here + + # if the 'ssl on' directive is present on the vhost, all its addresses have SSL enabled + all_addrs_are_ssl = self.parser.has_ssl_on_directive(vhost) + + # if we want ssl vhosts: either 'ssl on' or 'addr.ssl' should be enabled + # if we want plaintext vhosts: neither 'ssl on' nor 'addr.ssl' should be enabled + _ssl_matches = lambda addr: addr.ssl or all_addrs_are_ssl if ssl else \ + not addr.ssl and not all_addrs_are_ssl + + # if there are no listen directives at all, Nginx defaults to + # listening on port 80. + if not vhost.addrs: + return port == self.DEFAULT_LISTEN_PORT and ssl == all_addrs_are_ssl + + return any(self._port_matches(port, addr.get_port()) and _ssl_matches(addr) + for addr in vhost.addrs) + + def _vhost_listening_on_port_no_ssl(self, vhost: obj.VirtualHost, port: str) -> bool: + return self._vhost_listening(vhost, port, False) def _get_redirect_ranked_matches(self, target_name, port): """Gets a ranked list of plaintextish port-listening vhosts matching target_name diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 40f994988..896760fc4 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -5,6 +5,8 @@ import logging from acme import challenges from acme.magic_typing import List +from acme.magic_typing import Optional +from certbot import achallenges from certbot import errors from certbot.compat import os from certbot.plugins import common @@ -138,13 +140,12 @@ class NginxHttp01(common.ChallengePerformer): def _get_validation_path(self, achall): return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token")) - def _make_server_block(self, achall): + def _make_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge) -> List: """Creates a server block for a challenge. + :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` - :param list addrs: addresses of challenged domain - :class:`list` of type :class:`~nginx.obj.Addr` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :returns: server block for the challenge host :rtype: list """ @@ -172,34 +173,35 @@ class NginxHttp01(common.ChallengePerformer): return location_directive - def _make_or_mod_server_block(self, achall): - """Modifies a server block to respond to a challenge. + def _make_or_mod_server_block(self, achall: achallenges.KeyAuthorizationAnnotatedChallenge + ) -> Optional[List]: + """Modifies server blocks to respond to a challenge. Returns a new HTTP server block + to add to the configuration if an existing one can't be found. :param achall: Annotated HTTP-01 challenge - :type achall: - :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + :type achall: :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge` + + :returns: new server block to be added, if any + :rtype: list """ - try: - vhosts = self.configurator.choose_redirect_vhosts(achall.domain, - '%i' % self.configurator.config.http01_port, create_if_no_match=True) - except errors.MisconfigurationError: + http_vhosts, https_vhosts = self.configurator.choose_auth_vhosts(achall.domain) + + new_vhost: Optional[list] = None + if not http_vhosts: # Couldn't find either a matching name+port server block # or a port+default_server block, so create a dummy block - return self._make_server_block(achall) + new_vhost = self._make_server_block(achall) - # len is max 1 because Nginx doesn't authenticate wildcards - # if len were or vhosts None, we would have errored - vhost = vhosts[0] + # Modify any existing server blocks + for vhost in set(http_vhosts + https_vhosts): + location_directive = [self._location_directive_for_achall(achall)] - # Modify existing server block - location_directive = [self._location_directive_for_achall(achall)] + self.configurator.parser.add_server_directives(vhost, location_directive) - self.configurator.parser.add_server_directives(vhost, - location_directive) + rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', + ' ', '$1', ' ', 'break']] + self.configurator.parser.add_server_directives(vhost, + rewrite_directive, insert_at_top=True) - rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)', - ' ', '$1', ' ', 'break']] - self.configurator.parser.add_server_directives(vhost, - rewrite_directive, insert_at_top=True) - return None + return new_vhost diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 17a8c652a..db9530104 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -10,6 +10,7 @@ import pyparsing from acme.magic_typing import Dict from acme.magic_typing import List +from acme.magic_typing import Optional from acme.magic_typing import Set from acme.magic_typing import Tuple from acme.magic_typing import Union @@ -361,8 +362,9 @@ class NginxParser: except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) - def duplicate_vhost(self, vhost_template, remove_singleton_listen_params=False, - only_directives=None): + def duplicate_vhost(self, vhost_template: obj.VirtualHost, + remove_singleton_listen_params: bool = False, + only_directives: Optional[List] = None) -> obj.VirtualHost: """Duplicate the vhost in the configuration files. :param :class:`~certbot_nginx._internal.obj.VirtualHost` vhost_template: The vhost diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 5e788e394..9ccc3fc9e 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -39,7 +39,7 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) - self.assertEqual(12, len(self.config.parser.parsed)) + self.assertEqual(13, len(self.config.parser.parsed)) @mock.patch("certbot_nginx._internal.configurator.util.exe_exists") @mock.patch("certbot_nginx._internal.configurator.subprocess.Popen") @@ -89,7 +89,7 @@ class NginxConfiguratorTest(util.NginxTest): "155.225.50.69.nephoscale.net", "www.example.org", "another.alias", "migration.com", "summer.com", "geese.com", "sslon.com", "globalssl.com", "globalsslsetssl.com", "ipv6.com", "ipv6ssl.com", - "headers.com", "example.net"}) + "headers.com", "example.net", "ssl.both.com"}) def test_supported_enhancements(self): self.assertEqual(['redirect', 'ensure-http-header', 'staple-ocsp'], @@ -935,7 +935,19 @@ class NginxConfiguratorTest(util.NginxTest): prefer_ssl=False, no_ssl_filter_port='80') # Check that the dialog was called with only port 80 vhosts - self.assertEqual(len(mock_select_vhs.call_args[0][0]), 6) + self.assertEqual(len(mock_select_vhs.call_args[0][0]), 8) + + def test_choose_auth_vhosts(self): + """choose_auth_vhosts correctly selects duplicative and HTTP/HTTPS vhosts""" + http, https = self.config.choose_auth_vhosts('ssl.both.com') + self.assertEqual(len(http), 4) + self.assertEqual(len(https), 2) + self.assertEqual(http[0].names, {'ssl.both.com'}) + self.assertEqual(http[1].names, {'ssl.both.com'}) + self.assertEqual(http[2].names, {'ssl.both.com'}) + self.assertEqual(http[3].names, {'*.both.com'}) + self.assertEqual(https[0].names, {'ssl.both.com'}) + self.assertEqual(https[1].names, {'*.both.com'}) class InstallSslOptionsConfTest(util.NginxTest): diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 2947b099d..d0e84fc83 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -44,6 +44,10 @@ class HttpPerformTest(util.NginxTest): challb=acme_util.chall_to_challb( challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), domain="migration.com", account_key=account_key), + achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ipv6ssl.com", account_key=account_key), ] def setUp(self): @@ -77,8 +81,8 @@ class HttpPerformTest(util.NginxTest): http_responses = self.http01.perform() - self.assertEqual(len(http_responses), 4) - for i in range(4): + self.assertEqual(len(http_responses), 5) + for i in range(5): self.assertEqual(http_responses[i], acme_responses[i]) def test_mod_config(self): @@ -105,6 +109,43 @@ class HttpPerformTest(util.NginxTest): # self.assertEqual(vhost.addrs, set(v_addr2_print)) # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')])) + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_http_and_https(self, mock_add_server_directives): + """A server_name with both HTTP and HTTPS vhosts should get modded in both vhosts""" + self.configuration.https_port = 443 + self.http01.add_chall(self.achalls[3]) # migration.com + self.http01._mod_config() # pylint: disable=protected-access + + # Domain has an HTTP and HTTPS vhost + # 2 * 'rewrite' + 2 * 'return 200 keyauthz' = 4 + self.assertEqual(mock_add_server_directives.call_count, 4) + + @mock.patch('certbot_nginx._internal.parser.nginxparser.dump') + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_only_https(self, mock_add_server_directives, mock_dump): + """A server_name with only an HTTPS vhost should get modded""" + self.http01.add_chall(self.achalls[4]) # ipv6ssl.com + self.http01._mod_config() # pylint: disable=protected-access + + # It should modify the existing HTTPS vhost + self.assertEqual(mock_add_server_directives.call_count, 2) + # since there was no suitable HTTP vhost or default HTTP vhost, a non-empty one + # should have been created and written to the challenge conf file + self.assertNotEqual(mock_dump.call_args[0][0], []) + + @mock.patch('certbot_nginx._internal.parser.NginxParser.add_server_directives') + def test_mod_config_deduplicate(self, mock_add_server_directives): + """A vhost that appears in both HTTP and HTTPS vhosts only gets modded once""" + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.chall_to_challb( + challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"), + domain="ssl.both.com", account_key=AUTH_KEY) + self.http01.add_chall(achall) + self.http01._mod_config() # pylint: disable=protected-access + + # Should only get called 5 times, rather than 6, because two vhosts are the same + self.assertEqual(mock_add_server_directives.call_count, 5*2) + @mock.patch("certbot_nginx._internal.configurator.NginxConfigurator.ipv6_info") def test_default_listen_addresses_no_memoization(self, ipv6_info): # pylint: disable=protected-access diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 0083c2448..23fe390ad 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -51,6 +51,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual({nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', 'sites-enabled/default', + 'sites-enabled/both.com', 'sites-enabled/example.com', 'sites-enabled/headers.com', 'sites-enabled/migration.com', @@ -88,7 +89,7 @@ class NginxParserTest(util.NginxTest): parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) self.assertEqual(3, len(glob.glob(nparser.abs_path('*.test')))) - self.assertEqual(9, len( + self.assertEqual(10, len( glob.glob(nparser.abs_path('sites-enabled/*.test')))) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -171,7 +172,7 @@ class NginxParserTest(util.NginxTest): '*.www.example.com'}, [], [2, 1, 0]) - self.assertEqual(14, len(vhosts)) + self.assertEqual(19, len(vhosts)) example_com = [x for x in vhosts if 'example.com' in x.filep][0] self.assertEqual(vhost3, example_com) default = [x for x in vhosts if 'default' in x.filep][0] diff --git a/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com new file mode 100644 index 000000000..23a660df3 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/sites-enabled/both.com @@ -0,0 +1,32 @@ +server { + server_name ssl.both.com; +} + +# a duplicate vhost +server { + server_name ssl.both.com; +} + +# a duplicate by means of wildcard +server { + server_name *.both.com; +} + +# combined HTTP and HTTPS +server { + server_name ssl.both.com; + listen 80; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} + +# HTTPS, duplicate by means of wildcard +server { + server_name *.both.com; + listen 5001 ssl; + + ssl_certificate cert.pem; + ssl_certificate_key cert.key; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 1067c3b92..ea9c35e1d 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -21,6 +21,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). Python 2. * Certbot and all of its components no longer depend on the library `six`. * The update of certbot-auto itself is now disabled on all RHEL-like systems. +* The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 + challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. ### Fixed From 496a4ced2560686bf5dc6769c90406ea66fea4e5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 26 Feb 2021 16:05:34 -0800 Subject: [PATCH 13/99] Remove broken test for typing import failure (#8692) * remove broken test * fix coverage * don't worry about getattr test --- acme/acme/magic_typing.py | 20 +++++++++----------- acme/tests/magic_typing_test.py | 13 ------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 0e60d3203..91308fef6 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -1,16 +1,14 @@ -"""Shim class to not have to depend on typing module in prod.""" -import sys +"""Simple shim around the typing module. + +This was useful when this code supported Python 2 and typing wasn't always +available. This code is being kept for now for backwards compatibility. + +""" +from typing import * # pylint: disable=wildcard-import, unused-wildcard-import +from typing import Collection, IO # type: ignore class TypingClass: """Ignore import errors by getting anything""" def __getattr__(self, name): - return None - -try: - # mypy doesn't respect modifying sys.modules - from typing import * # pylint: disable=wildcard-import, unused-wildcard-import - from typing import Collection, IO # type: ignore -except ImportError: - # mypy complains because TypingClass is not a module - sys.modules[__name__] = TypingClass() # type: ignore + return None # pragma: no cover diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 048995916..257164d77 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -22,19 +22,6 @@ class MagicTypingTest(unittest.TestCase): del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing - def test_import_failure(self): - try: - import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover - sys.modules['typing'] = None - if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text - self.assertTrue(Text is None) - del sys.modules['acme.magic_typing'] - sys.modules['typing'] = temp_typing - if __name__ == '__main__': unittest.main() # pragma: no cover From 1e307230038ba0e92dfe06ecfef4e22d135370d4 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 2 Mar 2021 07:56:22 +1100 Subject: [PATCH 14/99] revoke: try determine the server automatically (#8691) * revoke: try determine the server automatically When revoking via --cert-name, use the server from the lineage (unless overriden by the CLI). * RenewableCert.storage might be None * guard against an empty lineage server --- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/main.py | 11 +++++--- certbot/certbot/_internal/storage.py | 13 ++++++--- certbot/tests/main_test.py | 40 +++++++++++++++++++++++++--- certbot/tests/storage_test.py | 7 +++++ 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ea9c35e1d..49bd2f222 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -21,6 +21,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). Python 2. * Certbot and all of its components no longer depend on the library `six`. * The update of certbot-auto itself is now disabled on all RHEL-like systems. +* When revoking a certificate by `--cert-name`, it is no longer necessary to specify the `--server` + if the certificate was obtained from a non-default ACME server. * The nginx authenticator now configures all matching HTTP and HTTPS vhosts for the HTTP-01 challenge. It is now compatible with external HTTPS redirection by a CDN or load balancer. diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 36ebb1a50..f7455db96 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1072,8 +1072,7 @@ def certificates(config, unused_plugins): cert_manager.certificates(config) -# TODO: coop with renewal config -def revoke(config, unused_plugins): +def revoke(config, unused_plugins: plugins_disco.PluginsRegistry) -> Optional[str]: """Revoke a previously obtained certificate. :param config: Configuration object @@ -1090,7 +1089,13 @@ def revoke(config, unused_plugins): config.installer = config.authenticator = None if config.cert_path is None and config.certname: - config.cert_path = storage.cert_path_for_cert_name(config, config.certname) + # When revoking via --cert-name, take the cert path and server from renewalparams + lineage = storage.RenewableCert( + storage.renewal_file_for_certname(config, config.certname), config) + config.cert_path = lineage.cert_path + # --server takes priority over lineage.server + if lineage.server and not cli.set_by_cli("server"): + config.server = lineage.server elif not config.cert_path or (config.cert_path and config.certname): # intentionally not supporting --cert-path & --cert-name together, # to avoid dealing with mismatched values diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index d04bc5774..218a5dd92 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -6,6 +6,7 @@ import re import shutil import stat +from typing import Optional import configobj import parsedatetime import pytz @@ -518,11 +519,15 @@ class RenewableCert(interfaces.RenewableCert): return _relpath_from_file(self.archive_dir, from_file) @property - def is_test_cert(self): + def server(self) -> Optional[str]: + """Returns the ACME server associated with this certificate""" + return self.configuration["renewalparams"].get("server", None) + + @property + def is_test_cert(self) -> bool: """Returns true if this is a test cert from a staging server.""" - server = self.configuration["renewalparams"].get("server", None) - if server: - return util.is_staging(server) + if self.server: + return util.is_staging(self.server) return False def _check_symlinks(self): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 2d5d88947..5ad11b120 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -317,6 +317,7 @@ class RevokeTest(test_util.TempDirTestCase): if not args: args = 'revoke --cert-path={0} ' args = args.format(self.tmp_cert_path).split() + cli.set_by_cli.detector = None # required to reset set_by_cli state plugins = disco.PluginsRegistry.find_all() config = configuration.NamespaceConfig( cli.prepare_and_parse_args(plugins, args)) @@ -342,13 +343,44 @@ class RevokeTest(test_util.TempDirTestCase): self.assertEqual(expected, mock_revoke.call_args_list) @mock.patch('certbot._internal.main._delete_if_appropriate') - @mock.patch('certbot._internal.storage.cert_path_for_cert_name') - def test_revoke_by_certname(self, mock_cert_path_for_cert_name, - mock_delete_if_appropriate): + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") args = 'revoke --cert-name=example.com'.split() - mock_cert_path_for_cert_name.return_value = self.tmp_cert_path mock_delete_if_appropriate.return_value = False self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://acme.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_and_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --server should use the server from the CLI""" + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, + server="https://acme.example") + args = 'revoke --cert-name=example.com --server https://other.example'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with(mock.ANY, mock.ANY, 'https://other.example') + self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) + + @mock.patch('certbot._internal.main._delete_if_appropriate') + @mock.patch('certbot._internal.storage.RenewableCert') + @mock.patch('certbot._internal.storage.renewal_file_for_certname') + def test_revoke_by_certname_empty_server(self, unused_mock_renewal_file_for_certname, + mock_cert, mock_delete_if_appropriate): + """Revoking with --cert-name where the lineage server is empty shouldn't crash """ + mock_cert.return_value = mock.MagicMock(cert_path=self.tmp_cert_path, server=None) + args = 'revoke --cert-name=example.com'.split() + mock_delete_if_appropriate.return_value = False + self._call(args) + self.mock_acme_client.assert_called_once_with( + mock.ANY, mock.ANY, constants.CLI_DEFAULTS['server']) self.mock_success_revoke.assert_called_once_with(self.tmp_cert_path) @mock.patch('certbot._internal.main._delete_if_appropriate') diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 0f696bc34..1e3a1ffd8 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -765,6 +765,13 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(storage.add_time_interval(base_time, interval), excepted) + def test_server(self): + self.test_rc.configuration["renewalparams"] = {} + self.assertEqual(self.test_rc.server, None) + rp = self.test_rc.configuration["renewalparams"] + rp["server"] = "https://acme.example/dir" + self.assertEqual(self.test_rc.server, "https://acme.example/dir") + def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] From 976068b5a09a4856d95dc9e408d5578b6058b963 Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:37:04 -0800 Subject: [PATCH 15/99] Update changelog for 1.13.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 49bd2f222..d007166b8 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.13.0 - master +## 1.13.0 - 2021-03-02 ### Added From 92a66454b6ece097a987621031b16b1fb2f490fd Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:49:58 -0800 Subject: [PATCH 16/99] Release 1.13.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 27 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 19 ++---------- letsencrypt-auto | 27 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 26 files changed, 83 insertions(+), 94 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 847ca9299..deaf56b59 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index f129343b3..b63ed1d08 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 002fd5ffc..24af0b4c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index af19b126e..94bde0ae2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index eab6cdb70..86d4a7855 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 83513ef7c..6fa117a15 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 6e60444cf..9570bd000 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index f1fcfd11d..6a3ff1235 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 185048a2d..ebde007c6 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 0ae9c1bf7..f8bb147da 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b16d014c6..45599b4c0 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 21ccf9d42..e6c5db942 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 2312d6fcc..831f4d0ad 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 658027b9a..4da5dbdf0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b4f73ddb4..b65548ec4 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index ce74611cd..a90a6ce2c 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 8def9a702..27663ed28 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 6f4f8e506..aa106037f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 385f4cc17..5c2a8ba12 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0.dev0' +version = '1.13.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index be06b5803..5fb51f98e 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0.dev0' +__version__ = '1.13.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 4482ea439..27511777b 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,7 +118,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.12.0 + "". (default: CertbotACMEClient/1.13.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -167,19 +167,6 @@ automation: --duplicate Allow making a certificate lineage that duplicates an existing one (both can be renewed in parallel) (default: False) - --os-packages-only (certbot-auto only) install OS package dependencies - and then stop (default: False) - --no-self-upgrade (certbot-auto only) prevent the certbot-auto script - from upgrading itself to newer released versions - (default: Upgrade automatically) - --no-bootstrap (certbot-auto only) prevent the certbot-auto script - from installing OS-level dependencies (default: Prompt - to install OS-wide dependencies, but exit if the user - says 'No') - --no-permissions-check - (certbot-auto only) skip the check on the file system - permissions of the certbot-auto script (default: - False) -q, --quiet Silence all output except errors. Useful for automation via cron. Implies --non-interactive. (default: False) @@ -254,8 +241,8 @@ paths: Flags for changing execution paths & servers --cert-path CERT_PATH - Path to where certificate is saved (with auth --csr), - installed from, or revoked. (default: None) + Path to where certificate is saved (with certonly + --csr), installed from, or revoked (default: None) --key-path KEY_PATH Path to private key for certificate installation or revocation (if account key is missing) (default: None) --fullchain-path FULLCHAIN_PATH diff --git a/letsencrypt-auto b/letsencrypt-auto index 002fd5ffc..24af0b4c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.12.0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -806,6 +806,7 @@ elif [ -f /etc/mageia-release ]; then NO_SELF_UPGRADE=1 elif [ -f /etc/redhat-release ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 # Run DeterminePythonVersion to decide on the basis of available Python versions # whether to use 2.x or 3.x on RedHat-like systems. # Then, revert LE_PYTHON to its previous state. @@ -1487,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index aba5f1140..fbd16f2b0 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmAZorcACgkQTRfJlc2X -dfI6Ogf+LFASyH9sgTV1k9hs1zbmO3CxyE9QQs1JLXpoKOQ1tKv+v+kpt+lJ005g -rielyRSssXtZSyfLchCSBh6qaEBodoOcz8RS2z7rDnR9jKOJv252Buh2oSa3KPmn -WPjRmB3zVXnhq/XmPKQTnoflUlBg+MtZuZXt0Fvu8rvQB+RY3AUfB5Xs83nxJNj4 -W9qNpZYl0sJWWiydr23bEk35MJSt62sKDvyqIVjUfgDfXHmauOpg0foz2xS6XP8i -Ke66GUKaQ1ap2BTucwVT0hieXiQZpxx1PitUeEOjOH9PUfrAxyFlQ0XQaVlqoBhc -YM3nzJw9yf12b+XCUvMzHyQmDA5vdQ== -=AUGt +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X +dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR +PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 +hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy +qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 +wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ +RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== +=jYnL -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 224abaf01..24af0b4c3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0.dev0" +LE_AUTO_VERSION="1.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1488,18 +1488,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.12.0 \ - --hash=sha256:f4bb3da5391e4a28e9a2e52ab54986171c0864feff17eaaaca6729a1d4c433a6 \ - --hash=sha256:5ee738773479bcb7794e43fedd2415acc0969b75bdd2a21f451e3bff9d99df59 -acme==1.12.0 \ - --hash=sha256:ca4ad044429f1b8b670b958e5c7ea38159def9d601f4af2359355993918c3317 \ - --hash=sha256:aa363474d50e9fdda27acb8b1aa7efb26fecc5650e02039a0de3a3f0e696c2f2 -certbot-apache==1.12.0 \ - --hash=sha256:38899f6fa08799de9535795d919acf968f288d7208909baf7733f9a763c15227 \ - --hash=sha256:e5679b40d99bd241f4fcd9fe44b73e6e25ccc969a617131ff6ebc90d562a49f2 -certbot-nginx==1.12.0 \ - --hash=sha256:332cd70067bbcf6db52a002650ffa4844d0bd9780279d662aa6725b43f776c14 \ - --hash=sha256:3fb6a55290d37ad466681a89a85ceca4c4026fdd8702f3010b87a74266a6fe7b +certbot==1.13.0 \ + --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ + --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 +acme==1.13.0 \ + --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ + --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 +certbot-apache==1.13.0 \ + --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ + --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 +certbot-nginx==1.13.0 \ + --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ + --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index ac143de5133fe67f7e2603a9be2f33665e78fa2c..3516f77b0476e8fcd96d806f6cb7a0714311bd12 100644 GIT binary patch literal 256 zcmV+b0ssDA%!<&T2#4iP{>&0k}(o_PX)s>Y- zrBV%mfP)is0sH4KmfDZW=Yt9<@Ql0GR>?6n=DPyOQX*xB(d{5QtnX4nS95FE%`Jww z=bnPK2Z=HKU`Gn4;8|WeAKEMOKcX;|~NR_4j^#_pW5KEftEPHR&7d6>tM z25sS8!q>0X2i1!j^>r7H3#_3h+9Hd?a0>T0w3bDqFtv1&?YEiLe&nlJm$Hh)Kyg6) zgqN!!#FtpuwhEdp+myusw`M`ny<6n)L%{I6g57Roqh20pjWc`Ewg6gn`Ye-p$Q(?^ zbC^pt$wV}Dv&JQ!aJ+c?Nd9eta%CF`q+j!a9ju|3PdE}u_D5)b8cm|&VKzf+ Date: Tue, 2 Mar 2021 13:50:03 -0800 Subject: [PATCH 17/99] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index d007166b8..7e877574f 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.14.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.13.0 - 2021-03-02 ### Added From 9d97be3a84c963c50f144cfbfc6e174212343e0a Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Tue, 2 Mar 2021 13:50:04 -0800 Subject: [PATCH 18/99] Bump version to 1.14.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index deaf56b59..90ab5c9f3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b63ed1d08..43165a9dd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 94bde0ae2..3e9e6a7c2 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 86d4a7855..2e5215f9c 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6fa117a15..44bada6bc 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9570bd000..763961e79 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 6a3ff1235..99a3c92a3 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index ebde007c6..b126b4e8f 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index f8bb147da..cada7a85e 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 45599b4c0..e8b26eceb 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e6c5db942..63da31df2 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 831f4d0ad..61b9dfa78 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4da5dbdf0..d60412f27 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b65548ec4..4d6131ff7 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index a90a6ce2c..18d006732 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 27663ed28..fdc931bbe 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index aa106037f..55f140d76 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 5c2a8ba12..8b638f28e 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.13.0' +version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 5fb51f98e..790443bcc 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.13.0' +__version__ = '1.14.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 24af0b4c3..7f1987439 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From a3abcc001a1dff41a6ecd90f779cf038e2f32cc9 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Wed, 3 Mar 2021 22:10:56 +0100 Subject: [PATCH 19/99] Removed a Python 2 fallback in certbot.Reverter. (#8694) * Removed a Python 2 fallback in certbot.Reverter. * Removed a Python < 3.6 fallback in certbot-apache._internal.parser. --- certbot-apache/certbot_apache/_internal/parser.py | 5 ----- certbot/certbot/reverter.py | 8 +++----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index fdef167bc..a1ea02b18 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,8 +3,6 @@ import copy import fnmatch import logging import re -import sys - from acme.magic_typing import Dict from acme.magic_typing import List @@ -737,9 +735,6 @@ class ApacheParser: :rtype: str """ - if sys.version_info < (3, 6): - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index 363215dd0..e6777a7da 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -3,7 +3,6 @@ import csv import glob import logging import shutil -import sys import time import traceback @@ -251,11 +250,10 @@ class Reverter: def _run_undo_commands(self, filepath): """Run all commands in a file.""" - # NOTE: csv module uses native strings. That is, bytes on Python 2 and - # unicode on Python 3 + # NOTE: csv module uses native strings. That is unicode on Python 3 # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} with open(filepath, 'r', **kwargs) as csvfile: # type: ignore csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): @@ -354,7 +352,7 @@ class Reverter: command_file = None # It is strongly advised to set newline = '' on Python 3 with CSV, # and it fixes problems on Windows. - kwargs = {'newline': ''} if sys.version_info[0] > 2 else {} + kwargs = {'newline': ''} try: if os.path.isfile(commands_fp): command_file = open(commands_fp, "a", **kwargs) # type: ignore From 94dc6936e73114ac4adc5f766d06f6c3cc9f67e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 Mar 2021 14:14:32 -0800 Subject: [PATCH 20/99] Final update to certbot-auto (#8706) Fixes https://github.com/certbot/certbot/issues/8690. After this PR, we'll let the release script make its automated changes to certbot-auto as part of the 1.14.0 release and then never make any code changes to certbot-auto ever again! * disable upgrades on debian * update test_leauto_upgrades.sh * update changelog --- certbot/CHANGELOG.md | 2 +- letsencrypt-auto-source/letsencrypt-auto | 1 + .../letsencrypt-auto.template | 1 + tests/letstest/scripts/test_leauto_upgrades.sh | 17 ++++------------- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7e877574f..c316ec376 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -10,7 +10,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* certbot-auto no longer checks for updates on any operating system. ### Fixed diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7f1987439..b0b25759f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 70b75176e..72876bb59 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -322,6 +322,7 @@ DeterminePythonVersion() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index 407a865f2..3964364e1 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -153,17 +153,8 @@ if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive exit 1 fi -# Finally, we check if our local server received more requests. Over time, -# we'll move more and more OSes into this case until it this is the expected -# behavior on all systems. -if [ -f /etc/redhat-release ]; then - if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 - fi -else - if diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server did not receive the requests we expected - exit 1 - fi +# Finally, we check if our local server received more requests. +if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then + echo our local server received unexpected requests + exit 1 fi From c02b2d30f2535a254020184912a94c6ed19b5026 Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Sat, 6 Mar 2021 01:53:20 +0100 Subject: [PATCH 21/99] Removed Python legacy __future__ imports (#8697) There are still some left, but the `modification_check` test fails. Some are still in `tools`, and they can probably be removed as well. `with_statement` was introduced officially in Python 2.5, so there's really old stuff in the code base. --- certbot-ci/certbot_integration_tests/assets/hook.py | 1 - .../certbot_integration_tests/certbot_tests/test_main.py | 1 - certbot-ci/certbot_integration_tests/conftest.py | 1 - certbot-ci/certbot_integration_tests/utils/acme_server.py | 1 - certbot-ci/certbot_integration_tests/utils/certbot_call.py | 1 - certbot-ci/certbot_integration_tests/utils/dns_server.py | 2 -- certbot-ci/certbot_integration_tests/utils/misc.py | 2 +- certbot-ci/windows_installer_integration_tests/conftest.py | 2 +- certbot/certbot/_internal/cli/__init__.py | 1 - certbot/certbot/_internal/cli/helpful.py | 2 +- certbot/certbot/_internal/hooks.py | 1 - certbot/certbot/_internal/log.py | 2 +- certbot/certbot/_internal/main.py | 1 - certbot/certbot/_internal/plugins/selection.py | 1 - certbot/certbot/_internal/renewal.py | 1 - certbot/certbot/_internal/reporter.py | 2 -- certbot/tests/main_test.py | 4 +--- certbot/tests/plugins/manual_test.py | 2 +- tests/letstest/multitester.py | 4 ---- tests/lock_test.py | 2 -- tools/install_and_test.py | 2 -- tools/merge_requirements.py | 2 -- tools/pip_install_editable.py | 3 --- tools/pipstrap.py | 1 - tools/readlink.py | 2 -- 25 files changed, 6 insertions(+), 38 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/assets/hook.py b/certbot-ci/certbot_integration_tests/assets/hook.py index 483892a93..c11704f47 100755 --- a/certbot-ci/certbot_integration_tests/assets/hook.py +++ b/certbot-ci/certbot_integration_tests/assets/hook.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import os import sys diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 2d3d93669..0b649acdc 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -1,5 +1,4 @@ """Module executing integration tests against certbot core.""" -from __future__ import print_function import os from os.path import exists diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 230fb0eda..6fc77480d 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -6,7 +6,6 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function import contextlib import subprocess import sys diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 846e2d071..b16aa80af 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to setup an ACME CA server environment able to run multiple tests in parallel""" -from __future__ import print_function import argparse import errno diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index c9e46cdc7..b319eca4c 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Module to call certbot in test mode""" -from __future__ import absolute_import import os import subprocess diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 62a58275e..eaa601f1b 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -1,7 +1,5 @@ #!/usr/bin/env python """Module to setup an RFC2136-capable DNS server""" -from __future__ import print_function - import os import os.path import shutil diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index e6b9f0c88..cf2e7ff09 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -311,7 +311,7 @@ def echo(keyword, path=None): if not re.match(r'^\w+$', keyword): raise ValueError('Error, keyword `{0}` is not a single keyword.' .format(keyword)) - return '{0} -c "from __future__ import print_function; print(\'{1}\')"{2}'.format( + return '{0} -c "print(\'{1}\')"{2}'.format( os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py index c6a89c323..8a9de057f 100644 --- a/certbot-ci/windows_installer_integration_tests/conftest.py +++ b/certbot-ci/windows_installer_integration_tests/conftest.py @@ -6,7 +6,7 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ -from __future__ import print_function + import os ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 8d2f7c329..688efeb7a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,6 +1,5 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines -from __future__ import print_function import logging import logging.handlers import argparse diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 5eaec978b..2505c24c6 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -1,5 +1,5 @@ """Certbot command line argument parser""" -from __future__ import print_function + import argparse import copy import functools diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 5526b21c4..256fce532 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,5 +1,4 @@ """Facilities for implementing hooks that call shell commands.""" -from __future__ import print_function import logging diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index 90dc2cda1..d8a55e27d 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -19,7 +19,7 @@ The preferred method to display important information to the user is to use `certbot.display.util` and `certbot.display.ops`. """ -from __future__ import print_function + import functools import logging diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index f7455db96..9b2141b5a 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1,6 +1,5 @@ """Certbot main entry point.""" # pylint: disable=too-many-lines -from __future__ import print_function import functools import logging.handlers diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index e5c311efe..bf7e505b9 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -1,5 +1,4 @@ """Decide which plugins to use for authentication & installation""" -from __future__ import print_function import logging diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 7533c8c6b..0b99ae3d1 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -1,5 +1,4 @@ """Functionality for autorenewal and associated juggling of configurations""" -from __future__ import print_function import copy import itertools diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index f43f0e7af..3093ef8ca 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -1,6 +1,4 @@ """Collects and displays information to the user.""" -from __future__ import print_function - import collections import logging import queue diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 5ad11b120..7668a1b6a 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -1,8 +1,6 @@ # coding=utf-8 """Tests for certbot._internal.main.""" # pylint: disable=too-many-lines -from __future__ import print_function - import datetime from importlib import reload as reload_module import io @@ -1353,7 +1351,7 @@ class MainTest(test_util.ConfigTestCase): _, _, stdout = self._test_renewal_common( due_for_renewal=False, extra_args=None, should_renew=False, args=['renew', '--post-hook', - '{0} -c "from __future__ import print_function; print(\'hello world\');"' + '{0} -c "print(\'hello world\');"' .format(sys.executable)]) self.assertTrue('No hooks were run.' in stdout.getvalue()) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index f3c580517..0e552e0c5 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -60,7 +60,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): def test_script_perform(self): self.config.manual_auth_hook = ( - '{0} -c "from __future__ import print_function;' + '{0} -c "' 'from certbot.compat import os;' 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 75dd06ad6..1907995c2 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -27,10 +27,6 @@ see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html """ - -from __future__ import print_function -from __future__ import with_statement - import argparse import multiprocessing as mp from multiprocessing import Manager diff --git a/tests/lock_test.py b/tests/lock_test.py index 56399c874..f310b5753 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,6 +1,4 @@ """Tests to ensure the lock order is preserved.""" -from __future__ import print_function - import atexit import datetime import functools diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 0b47fa5f8..e7a34286c 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -5,8 +5,6 @@ # set to 1, packages are installed using pinned versions of all of our # dependencies. See pip_install.py for more information on the versions pinned # to. -from __future__ import print_function - import os import re import subprocess diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index bbcb38051..44a61249b 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -6,8 +6,6 @@ Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are currently supported. """ -from __future__ import print_function - import sys diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index abfe9f214..de2a0ff57 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -6,9 +6,6 @@ # https://github.com/pyca/cryptography/blob/a02fdd60d98273ca34427235c4ca96687a12b239/.travis/downstream.d/certbot.sh#L8-L9. # We should try to remember to keep their repo updated if we make any changes # to this script which may break things for them. - -from __future__ import absolute_import - import sys import pip_install diff --git a/tools/pipstrap.py b/tools/pipstrap.py index e6b746916..df3e46003 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -1,6 +1,5 @@ #!/usr/bin/env python """Uses pip to upgrade Python packaging tools to pinned versions.""" -from __future__ import absolute_import import os import pip_install diff --git a/tools/readlink.py b/tools/readlink.py index 446c8ebdc..c1e0c2e61 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -6,8 +6,6 @@ useful as there are often differences in readlink on different platforms. """ -from __future__ import print_function - import os import sys From 76895457c9f5956ec97ccadc3e0d7e894db17094 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Wed, 10 Mar 2021 10:56:51 +1100 Subject: [PATCH 22/99] dns-digitalocean: use a 30 second TTL (#8693) Fixes an issue with long TTLs and caching behavior on DigitalOcean's DNS hosting service. --- .../_internal/dns_digitalocean.py | 13 +++++++++---- certbot-dns-digitalocean/setup.py | 2 +- .../tests/dns_digitalocean_test.py | 15 +++++++++------ certbot/CHANGELOG.md | 1 + tools/dev_constraints.txt | 3 ++- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 5893df9c2..cb5012fb7 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -21,6 +21,7 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are ' + \ 'using DigitalOcean for DNS).' + ttl = 30 def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) @@ -45,7 +46,8 @@ class Authenticator(dns_common.DNSAuthenticator): ) def _perform(self, domain, validation_name, validation): - self._get_digitalocean_client().add_txt_record(domain, validation_name, validation) + self._get_digitalocean_client().add_txt_record(domain, validation_name, validation, + self.ttl) def _cleanup(self, domain, validation_name, validation): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) @@ -62,13 +64,15 @@ class _DigitalOceanClient: def __init__(self, token): self.manager = digitalocean.Manager(token=token) - def add_txt_record(self, domain_name, record_name, record_content): + def add_txt_record(self, domain_name: str, record_name: str, record_content: str, + record_ttl: int): """ Add a TXT record using the supplied information. :param str domain_name: The domain to use to associate the record with. :param str record_name: The record name (typically beginning with '_acme-challenge.'). :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL. :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean API """ @@ -89,7 +93,8 @@ class _DigitalOceanClient: result = domain.create_new_domain_record( type='TXT', name=self._compute_record_name(domain, record_name), - data=record_content) + data=record_content, + ttl=record_ttl) # ttl kwarg is only effective starting python-digitalocean 1.15.0 record_id = result['domain_record']['id'] @@ -99,7 +104,7 @@ class _DigitalOceanClient: raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}' .format(e)) - def del_txt_record(self, domain_name, record_name, record_content): + def del_txt_record(self, domain_name: str, record_name: str, record_content: str): """ Delete a TXT record using the supplied information. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 763961e79..106eb3cac 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -9,7 +9,7 @@ version = '1.14.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'python-digitalocean>=1.11', + 'python-digitalocean>=1.11', # 1.15.0 or newer is recommended for TTL support 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index a752f52d0..84bb35b1f 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -40,7 +40,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def test_perform(self): self.auth.perform([self.achall]) - expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, 30)] self.assertEqual(expected, self.mock_client.mock_calls) def test_cleanup(self): @@ -58,6 +58,7 @@ class DigitalOceanClientTest(unittest.TestCase): record_prefix = "_acme-challenge" record_name = record_prefix + "." + DOMAIN record_content = "bar" + record_ttl = 60 def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import _DigitalOceanClient @@ -78,25 +79,27 @@ class DigitalOceanClientTest(unittest.TestCase): self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] - self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content) + self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content, + self.record_ttl) domain_mock.create_new_domain_record.assert_called_with(type='TXT', name=self.record_prefix, - data=self.record_content) + data=self.record_content, + ttl=self.record_ttl) def test_add_txt_record_fail_to_find_domain(self): self.manager.get_all_domains.return_value = [] self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_finding_domain(self): self.manager.get_all_domains.side_effect = API_ERROR self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_add_txt_record_error_creating_record(self): domain_mock = mock.MagicMock() @@ -107,7 +110,7 @@ class DigitalOceanClientTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.digitalocean_client.add_txt_record, - DOMAIN, self.record_name, self.record_content) + DOMAIN, self.record_name, self.record_content, self.record_ttl) def test_del_txt_record(self): first_record_mock = mock.MagicMock() diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index c316ec376..ccbe6a43b 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot-auto no longer checks for updates on any operating system. +* The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. ### Fixed diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 10308bd39..f4059a3f9 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,6 +51,7 @@ jedi==0.17.1 Jinja2==2.9.6 jmespath==0.9.4 josepy==1.1.0 +jsonpickle==2.0.0 jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 @@ -93,7 +94,7 @@ pytest-xdist==1.22.5 pytest-sugar==0.9.2 pytest-rerunfailures==4.2 python-dateutil==2.8.1 -python-digitalocean==1.11 +python-digitalocean==1.15.0 python-dotenv==0.14.0 pywin32==300 PyYAML==5.3.1 From 67b65bb2c0b5fee3b97ae2e94649e8ef28fcbd74 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 10 Mar 2021 01:12:32 +0100 Subject: [PATCH 23/99] Deprecate acme.typing_magic module, stop using it in certbot (#8643) * Deprecate acme.magic_typing, stop to use it in certbot * Isort * Add a changelog entry Co-authored-by: Brad Warren --- acme/acme/challenges.py | 7 ++++--- acme/acme/client.py | 8 ++++---- acme/acme/crypto_util.py | 6 +++--- acme/acme/magic_typing.py | 3 +++ acme/acme/standalone.py | 2 +- acme/tests/magic_typing_test.py | 11 +++++++---- .../certbot_apache/_internal/apache_util.py | 1 - .../certbot_apache/_internal/assertions.py | 1 - .../certbot_apache/_internal/augeasparser.py | 4 ++-- .../certbot_apache/_internal/configurator.py | 13 +++++++------ .../certbot_apache/_internal/dualparser.py | 2 +- .../certbot_apache/_internal/http_01.py | 6 +++--- .../certbot_apache/_internal/interfaces.py | 1 - certbot-apache/certbot_apache/_internal/obj.py | 2 +- .../certbot_apache/_internal/override_centos.py | 2 +- certbot-apache/certbot_apache/_internal/parser.py | 5 +++-- .../nginx_tests/test_main.py | 2 +- .../rfc2136_tests/context.py | 2 +- .../utils/acme_server.py | 4 ++-- .../certbot_integration_tests/utils/misc.py | 4 ++-- .../utils/pebble_artifacts.py | 3 ++- .../snap_integration_tests/dns_tests/test_main.py | 5 +++-- .../test_main.py | 4 ++-- .../configurators/nginx/common.py | 2 +- .../certbot_compatibility_test/test_driver.py | 4 ++-- .../_internal/dns_cloudflare.py | 7 +++---- .../certbot_dns_route53/_internal/dns_route53.py | 6 +++--- .../certbot_nginx/_internal/configurator.py | 12 ++++++------ .../certbot_nginx/_internal/constants.py | 5 ++--- certbot-nginx/certbot_nginx/_internal/http_01.py | 4 ++-- .../certbot_nginx/_internal/nginxparser.py | 3 ++- certbot-nginx/certbot_nginx/_internal/parser.py | 12 ++++++------ .../certbot_nginx/_internal/parser_obj.py | 2 +- certbot/CHANGELOG.md | 2 ++ certbot/certbot/_internal/account.py | 2 +- certbot/certbot/_internal/auth_handler.py | 6 +++--- certbot/certbot/_internal/cert_manager.py | 2 +- certbot/certbot/_internal/cli/__init__.py | 4 ++-- certbot/certbot/_internal/cli/cli_utils.py | 4 ++-- certbot/certbot/_internal/cli/helpful.py | 4 ++-- certbot/certbot/_internal/client.py | 4 ++-- certbot/certbot/_internal/eff.py | 5 ++--- certbot/certbot/_internal/error_handler.py | 10 +++++----- certbot/certbot/_internal/hooks.py | 4 ++-- certbot/certbot/_internal/lock.py | 2 +- certbot/certbot/_internal/main.py | 6 +++++- certbot/certbot/_internal/plugins/disco.py | 2 +- certbot/certbot/_internal/plugins/manual.py | 3 ++- certbot/certbot/_internal/plugins/standalone.py | 10 +++++----- certbot/certbot/_internal/plugins/webroot.py | 8 ++++---- certbot/certbot/_internal/renewal.py | 9 +++++---- certbot/certbot/_internal/snap_config.py | 2 +- certbot/certbot/compat/filesystem.py | 3 +-- certbot/certbot/compat/misc.py | 4 ++-- certbot/certbot/crypto_util.py | 15 +++++++++------ certbot/certbot/display/util.py | 2 +- certbot/certbot/ocsp.py | 4 ++-- certbot/certbot/plugins/common.py | 2 +- certbot/certbot/plugins/dns_common_lexicon.py | 6 +++--- certbot/certbot/plugins/enhancements.py | 6 +++--- certbot/certbot/plugins/storage.py | 4 ++-- certbot/certbot/util.py | 8 ++++---- tests/letstest/multitester.py | 1 - tools/pipstrap.py | 1 - tools/snap/build_remote.py | 10 ++++++++-- 65 files changed, 165 insertions(+), 145 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 41a2aa258..58b457e0d 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -8,14 +8,15 @@ import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose -import requests -from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from OpenSSL import crypto +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +import requests from acme import crypto_util from acme import errors from acme import fields -from acme.mixins import ResourceMixin, TypeMixin +from acme.mixins import ResourceMixin +from acme.mixins import TypeMixin logger = logging.getLogger(__name__) diff --git a/acme/acme/client.py b/acme/acme/client.py index 33e515124..b1b2089cf 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -8,6 +8,10 @@ import http.client as http_client import logging import re import time +from typing import Dict +from typing import List +from typing import Set +from typing import Text import josepy as jose import OpenSSL @@ -20,10 +24,6 @@ from acme import crypto_util from acme import errors from acme import jws from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Text from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index a14737053..04faf6912 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -5,15 +5,15 @@ import logging import os import re import socket +from typing import Callable +from typing import Tuple +from typing import Union import josepy as jose from OpenSSL import crypto from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 from acme import errors -from acme.magic_typing import Callable -from acme.magic_typing import Tuple -from acme.magic_typing import Union logger = logging.getLogger(__name__) diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index 91308fef6..8190fa552 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -4,9 +4,12 @@ This was useful when this code supported Python 2 and typing wasn't always available. This code is being kept for now for backwards compatibility. """ +import warnings from typing import * # pylint: disable=wildcard-import, unused-wildcard-import from typing import Collection, IO # type: ignore +warnings.warn("acme.magic_typing is deprecated and will be removed in a future release.", + DeprecationWarning) class TypingClass: """Ignore import errors by getting anything""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index f5bc548b6..123899470 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -7,10 +7,10 @@ import logging import socket import socketserver import threading +from typing import List from acme import challenges from acme import crypto_util -from acme.magic_typing import List logger = logging.getLogger(__name__) diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 257164d77..d470337bd 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -1,6 +1,7 @@ """Tests for acme.magic_typing.""" import sys import unittest +import warnings from unittest import mock @@ -9,15 +10,17 @@ class MagicTypingTest(unittest.TestCase): def test_import_success(self): try: import typing as temp_typing - except ImportError: # pragma: no cover - temp_typing = None # pragma: no cover + except ImportError: # pragma: no cover + temp_typing = None # pragma: no cover typing_class_mock = mock.MagicMock() text_mock = mock.MagicMock() typing_class_mock.Text = text_mock sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: - del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text + del sys.modules['acme.magic_typing'] # pragma: no cover + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 93612f424..9b9855a45 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -9,7 +9,6 @@ import pkg_resources from certbot import errors from certbot import util - from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index 2b2ce4f50..53603c526 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -3,7 +3,6 @@ import fnmatch from certbot_apache._internal import interfaces - PASS = "CERTBOT_PASS_ASSERT" diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 3b2ce40d8..461b5cb7b 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,10 +64,10 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set +from typing import Set + from certbot import errors from certbot.compat import os - from certbot_apache._internal import apache_util from certbot_apache._internal import assertions from certbot_apache._internal import interfaces diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 16def1998..ecd5c46de 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,13 +1,18 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict -from distutils.version import LooseVersion import copy +from distutils.version import LooseVersion import fnmatch import logging import re import socket import time +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set +from typing import Union import zope.component import zope.interface @@ -18,11 +23,6 @@ except ImportError: # pragma: no cover HAS_APACHECONFIG = False from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set -from acme.magic_typing import Union from certbot import errors from certbot import interfaces from certbot import util @@ -41,6 +41,7 @@ from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser + logger = logging.getLogger(__name__) diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index 1ba23e92f..c89ff95be 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -1,7 +1,7 @@ """ Dual ParserNode implementation """ +from certbot_apache._internal import apacheparser from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser -from certbot_apache._internal import apacheparser class DualNodeBase: diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 5ef44fa2e..4bd935d2d 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,9 +1,9 @@ """A class that performs HTTP-01 challenges for Apache""" -import logging import errno +import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 77daa5b56..38c21b785 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc - class ParserNode(object, metaclass=abc.ABCMeta): """ ParserNode is the basic building block of the tree of such nodes, diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index e2fe48cf8..5e4680e1b 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -1,7 +1,7 @@ """Module contains classes used by the Apache Configurator.""" import re +from typing import Set -from acme.magic_typing import Set from certbot.plugins import common diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 9b2ee54c9..4cde65d57 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,9 +1,9 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import List import zope.interface -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index a1ea02b18..eb8f267f9 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -3,9 +3,10 @@ import copy import fnmatch import logging import re +from typing import Dict +from typing import List + -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 8a2d48a50..4c746e1ee 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -1,8 +1,8 @@ """Module executing integration tests against certbot with nginx plugin.""" import os import ssl - from typing import List + import pytest from certbot_integration_tests.nginx_tests import context as nginx_context diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index bdedee1fe..c8024b21d 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -1,7 +1,7 @@ """Module to handle the context of RFC2136 integration tests.""" -import tempfile from contextlib import contextmanager +import tempfile from pkg_resources import resource_filename from pytest import skip diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index b16aa80af..706c3ebae 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -11,14 +11,14 @@ import subprocess import sys import tempfile import time - from typing import List + import requests +# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts from certbot_integration_tests.utils import proxy -# pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils.constants import * diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index cf2e7ff09..2fac494f2 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -26,8 +26,8 @@ from OpenSSL import crypto import pkg_resources import requests -from certbot_integration_tests.utils.constants import \ - PEBBLE_ALTERNATE_ROOTS, PEBBLE_MANAGEMENT_URL +from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS +from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL RSA_KEY_TYPE = 'rsa' ECDSA_KEY_TYPE = 'ecdsa' diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index cd62e1a7f..918a5fd04 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,8 @@ import stat import pkg_resources import requests -from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT, MOCK_OCSP_SERVER_PORT +from certbot_integration_tests.utils.constants import DEFAULT_HTTP_01_PORT +from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 016355334..721352c04 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import pytest -import subprocess import glob import os import re +import subprocess + +import pytest @pytest.fixture(autouse=True, scope="module") diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py index c8c347aa8..f699b736a 100644 --- a/certbot-ci/windows_installer_integration_tests/test_main.py +++ b/certbot-ci/windows_installer_integration_tests/test_main.py @@ -1,8 +1,8 @@ import os +import re +import subprocess import time import unittest -import subprocess -import re @unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.') diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 7cb4e9722..58b72c205 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -2,10 +2,10 @@ import os import shutil import subprocess +from typing import Set import zope.interface -from acme.magic_typing import Set from certbot._internal import configuration from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index f11b9fdf8..fe634ba9f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -8,6 +8,8 @@ import shutil import sys import tempfile import time +from typing import List +from typing import Tuple import OpenSSL from urllib3.util import connection @@ -15,8 +17,6 @@ from urllib3.util import connection from acme import challenges from acme import crypto_util from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors as le_errors from certbot.tests import acme_util diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index c896a6e3a..583b41410 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -1,13 +1,12 @@ """DNS Authenticator for Cloudflare.""" import logging +from typing import Any +from typing import Dict +from typing import List import CloudFlare import zope.interface -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List - from certbot import errors from certbot import interfaces from certbot.plugins import dns_common diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 6250d2274..4837e2036 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -2,15 +2,15 @@ import collections import logging import time +from typing import DefaultDict +from typing import Dict +from typing import List import boto3 from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError import zope.interface -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot.plugins import dns_common diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 8a3b8078f..33baf076a 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -7,6 +7,12 @@ import socket import subprocess import tempfile import time +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Text +from typing import Tuple import OpenSSL import pkg_resources @@ -14,12 +20,6 @@ import zope.interface from acme import challenges from acme import crypto_util as acme_crypto_util -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Optional -from acme.magic_typing import Set -from acme.magic_typing import Text -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 1f058e7ef..7b4111577 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -1,8 +1,7 @@ """nginx plugin constants.""" import platform - -from acme.magic_typing import Any -from acme.magic_typing import Dict +from typing import Any +from typing import Dict FREEBSD_DARWIN_SERVER_ROOT = "/usr/local/etc/nginx" LINUX_SERVER_ROOT = "/etc/nginx" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index 896760fc4..ec610f15b 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -2,10 +2,10 @@ import io import logging +from typing import List +from typing import Optional from acme import challenges -from acme.magic_typing import List -from acme.magic_typing import Optional from certbot import achallenges from certbot import errors from certbot.compat import os diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index a51302fae..1aee1f329 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,6 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging +from typing import Any # pylint: disable=unused-import +from typing import IO # pylint: disable=unused-import from pyparsing import Combine from pyparsing import Forward @@ -15,7 +17,6 @@ from pyparsing import restOfLine from pyparsing import stringEnd from pyparsing import White from pyparsing import ZeroOrMore -from acme.magic_typing import IO, Any # pylint: disable=unused-import logger = logging.getLogger(__name__) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index db9530104..ed7e211d5 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -5,15 +5,15 @@ import glob import io import logging import re +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple +from typing import Union import pyparsing -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Optional -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index e55d48dc4..9efd6651a 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -3,9 +3,9 @@ raw lists of tokens from pyparsing. """ import abc import logging +from typing import List -from acme.magic_typing import List from certbot import errors logger = logging.getLogger(__name__) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index ccbe6a43b..62bceb74d 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * certbot-auto no longer checks for updates on any operating system. +* The module `acme.magic_typing` is deprecated and will be removed in a future release. + Please use the built-in module `typing` instead. * The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. ### Fixed diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index b2d50297e..07ae95e9d 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -18,8 +18,8 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import constants -from certbot.compat import os from certbot.compat import filesystem +from certbot.compat import os logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 17bf75225..2801e3c2f 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -2,15 +2,15 @@ import datetime import logging import time +from typing import Dict +from typing import List +from typing import Tuple import zope.component from acme import challenges from acme import errors as acme_errors from acme import messages -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Tuple from certbot import achallenges from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index ee2bd6254..2ac9fc538 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -3,11 +3,11 @@ import datetime import logging import re import traceback +from typing import List import pytz import zope.component -from acme.magic_typing import List from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 688efeb7a..30f444e5b 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -4,11 +4,11 @@ import logging import logging.handlers import argparse import sys +from typing import Optional + import certbot._internal.plugins.selection as plugin_selection from certbot._internal.plugins import disco as plugins_disco -from acme.magic_typing import Optional - # pylint: disable=ungrouped-imports import certbot from certbot._internal import constants diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index 5bdbbe02c..f1f9d0f3c 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -5,11 +5,11 @@ import copy import zope.interface.interface # pylint: disable=unused-import from acme import challenges +from certbot import errors from certbot import interfaces from certbot import util -from certbot import errors -from certbot.compat import os from certbot._internal import constants +from certbot.compat import os class _Default: diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index 2505c24c6..cbac4e232 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -5,6 +5,8 @@ import copy import functools import glob import sys +from typing import Any +from typing import Dict import configargparse import zope.component @@ -12,8 +14,6 @@ import zope.interface from zope.interface import interfaces as zope_interfaces -from acme.magic_typing import Any, Dict - from certbot import crypto_util from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index f2fc06937..b9137e249 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,6 +2,8 @@ import datetime import logging import platform +from typing import List +from typing import Optional from cryptography.hazmat.backends import default_backend # See https://github.com/pyca/cryptography/issues/4275 @@ -14,8 +16,6 @@ from acme import client as acme_client from acme import crypto_util as acme_crypto_util from acme import errors as acme_errors from acme import messages -from acme.magic_typing import List -from acme.magic_typing import Optional import certbot from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 5fbbd302a..6212c24ee 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,16 +1,15 @@ """Subscribes users to the EFF newsletter.""" import logging +from typing import Optional # pylint: disable=unused-import import requests import zope.component -from acme.magic_typing import Optional # pylint: disable=unused-import - from certbot import interfaces -from certbot.display import util as display_util from certbot._internal import constants from certbot._internal.account import Account # pylint: disable=unused-import from certbot._internal.account import AccountFileStorage +from certbot.display import util as display_util from certbot.interfaces import IConfig # pylint: disable=unused-import logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index 05af9d837..cb8301980 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -3,12 +3,12 @@ import functools import logging import signal import traceback +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Union -from acme.magic_typing import Any -from acme.magic_typing import Callable -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Union from certbot import errors from certbot.compat import os diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index 256fce532..b695d9930 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -1,9 +1,9 @@ """Facilities for implementing hooks that call shell commands.""" import logging +from typing import List +from typing import Set -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot import util from certbot.compat import filesystem diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index 32142fe44..f878946cb 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -1,8 +1,8 @@ """Implements file locks compatible with Linux and Windows for locking files and directories.""" import errno import logging +from typing import Optional -from acme.magic_typing import Optional from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 9b2141b5a..ba8d94bca 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,13 +4,17 @@ import functools import logging.handlers import sys +from typing import Iterable # pylint: disable=unused-import +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union import configobj import josepy as jose import zope.component from acme import errors as acme_errors -from acme.magic_typing import Union, Iterable, Optional, List, Tuple # pylint: disable=unused-import import certbot from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 1e1e00982..27b2605e0 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -2,13 +2,13 @@ import itertools import logging import sys +from typing import Dict from collections.abc import Mapping import pkg_resources import zope.interface import zope.interface.verify -from acme.magic_typing import Dict from certbot import errors from certbot import interfaces from certbot._internal import constants diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index a2e4bb28e..c28e333b9 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -1,9 +1,10 @@ """Manual authenticator plugin""" +from typing import Dict + import zope.component import zope.interface from acme import challenges -from acme.magic_typing import Dict from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index d5d9fd2ec..f511c60e9 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -4,17 +4,17 @@ import logging import socket # https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi from socket import errno as socket_errors # type: ignore +from typing import DefaultDict +from typing import Dict +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING import OpenSSL # pylint: disable=unused-import import zope.interface from acme import challenges from acme import standalone as acme_standalone -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import Set -from acme.magic_typing import Tuple -from acme.magic_typing import TYPE_CHECKING from certbot import achallenges from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 8789db604..730241066 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -3,15 +3,15 @@ import argparse import collections import json import logging +from typing import DefaultDict +from typing import Dict +from typing import List +from typing import Set import zope.component import zope.interface from acme import challenges -from acme.magic_typing import DefaultDict -from acme.magic_typing import Dict -from acme.magic_typing import List -from acme.magic_typing import Set from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index 0b99ae3d1..a09edfa1e 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -7,17 +7,17 @@ import random import sys import time import traceback +from typing import List +from typing import Optional # pylint: disable=unused-import from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import load_pem_private_key import OpenSSL import zope.component -from acme.magic_typing import List -from acme.magic_typing import Optional # pylint: disable=unused-import from certbot import crypto_util -from certbot.display import util as display_util from certbot import errors from certbot import interfaces from certbot import util @@ -29,6 +29,7 @@ from certbot._internal import storage from certbot._internal import updater from certbot._internal.plugins import disco as plugins_disco from certbot.compat import os +from certbot.display import util as display_util logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index a90740787..f2fe035db 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -1,13 +1,13 @@ """Module configuring Certbot in a snap environment""" import logging import socket +from typing import List from requests import Session from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import List from certbot.compat import os from certbot.errors import Error diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 0152685e9..aa6fe9134 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -5,8 +5,7 @@ import errno import os # pylint: disable=os-module-forbidden import stat import sys - -from acme.magic_typing import List +from typing import List try: import ntsecuritycon diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index f4ea4a5cc..297df80fc 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -8,12 +8,12 @@ import logging import select import subprocess import sys +from typing import Optional +from typing import Tuple from certbot import errors from certbot.compat import os -from acme.magic_typing import Tuple, Optional - try: from win32com.shell import shell as shellwin32 POSIX_MODE = False diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index f67d95d97..5bc9251a4 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -6,26 +6,29 @@ """ import hashlib import logging +import re +from typing import IO # pylint: disable=unused-import import warnings -import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore -from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm +from cryptography.exceptions import InvalidSignature +from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec -from cryptography.hazmat.primitives.asymmetric.ec import ECDSA, EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.ec import ECDSA +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey -from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat +from cryptography.hazmat.primitives.serialization import Encoding +from cryptography.hazmat.primitives.serialization import NoEncryption +from cryptography.hazmat.primitives.serialization import PrivateFormat from OpenSSL import crypto from OpenSSL import SSL # type: ignore - import pyrfc3339 import zope.component from acme import crypto_util as acme_crypto_util -from acme.magic_typing import IO # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot import util diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index f26ec468f..1e85ae808 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -12,11 +12,11 @@ Other messages can use the `logging` module. See `log.py`. import logging import sys import textwrap +from typing import List import zope.interface import zope.component -from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot._internal import constants diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index dcbf0381a..9a18b32f2 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -5,6 +5,8 @@ import logging import re from subprocess import PIPE from subprocess import Popen +from typing import Optional +from typing import Tuple from cryptography import x509 from cryptography.exceptions import InvalidSignature @@ -16,8 +18,6 @@ from cryptography.hazmat.primitives import serialization import pytz import requests -from acme.magic_typing import Optional -from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 489c75a3d..5e1c29669 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -3,12 +3,12 @@ import logging import re import shutil import tempfile +from typing import List from josepy import util as jose_util import pkg_resources import zope.interface -from acme.magic_typing import List from certbot import achallenges # pylint: disable=unused-import from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index a29509b79..5fec5e0d6 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,12 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" import logging +from typing import Any +from typing import Dict +from typing import Union from requests.exceptions import HTTPError from requests.exceptions import RequestException -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import Union from certbot import errors from certbot.plugins import dns_common diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index e674c32a2..3c3db7e71 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -1,9 +1,9 @@ """New interface style Certbot enhancements""" import abc +from typing import Any +from typing import Dict +from typing import List -from acme.magic_typing import Any -from acme.magic_typing import Dict -from acme.magic_typing import List from certbot._internal import constants ENHANCEMENTS = ["redirect", "ensure-http-header", "ocsp-stapling"] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index abef534f9..b5dd6a502 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -1,9 +1,9 @@ """Plugin storage class.""" import json import logging +from typing import Any +from typing import Dict -from acme.magic_typing import Any -from acme.magic_typing import Dict from certbot import errors from certbot.compat import filesystem from certbot.compat import os diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index ffd3a65f2..0643d5ba7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -12,13 +12,13 @@ import re import socket import subprocess import sys +from typing import Dict +from typing import Text +from typing import Tuple +from typing import Union import configargparse -from acme.magic_typing import Dict -from acme.magic_typing import Text -from acme.magic_typing import Tuple -from acme.magic_typing import Union from certbot import errors from certbot._internal import constants from certbot._internal import lock diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 1907995c2..f079f893f 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -45,7 +45,6 @@ import yaml from fabric import Config from fabric import Connection - # Command line parser #------------------------------------------------------------------------------- parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.') diff --git a/tools/pipstrap.py b/tools/pipstrap.py index df3e46003..2b2e3dcbb 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -4,7 +4,6 @@ import os import pip_install - _REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e6a44240f..e690d2916 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -2,12 +2,18 @@ import argparse import datetime import glob +from multiprocessing import Manager +from multiprocessing import Pool +from multiprocessing import Process +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from os.path import realpath import re import subprocess import sys import time -from multiprocessing import Pool, Process, Manager -from os.path import join, realpath, dirname, basename, exists CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] From f2d8c81e9b8165a37c27641a49b447e55921147e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 9 Mar 2021 16:53:44 -0800 Subject: [PATCH 24/99] remove reference to acme.magic_typing from docs (#8709) --- certbot/docs/contributing.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index def2c7fcd..13ac5ed68 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -478,13 +478,6 @@ to start contributing to Certbot. To run mypy on Certbot, use ``tox -e mypy`` on a machine that has Python 3 installed. -Note that instead of just importing ``typing``, due to packaging issues, in Certbot we import from -``acme.magic_typing`` and have to add some comments for pylint like this: - -.. code-block:: python - - from acme.magic_typing import Dict - Also note that OpenSSL, which we rely on, has type definitions for crypto but not SSL. We use both. Those imports should look like this: From dd6f2f565e8183b69ff3abf5d965d0d1575b2e5e Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 10 Mar 2021 20:51:27 +0100 Subject: [PATCH 25/99] Convert Python 2 type hints to Python 3 types annotations (#8640) Fixes #8427 This PR converts the Python 2 types hints into Python 3 types annotations. I have used the project https://github.com/ilevkivskyi/com2ann which has been designed for that specific purpose and did that very well. The only remaining things to do were to fix broken type hints that became wrong code after migration, and to fix lines too long with the new syntax. * Raw execution of com2ann * Fixing broken type annotations * Cleanup imports --- acme/acme/challenges.py | 4 +- acme/acme/client.py | 11 +- acme/acme/crypto_util.py | 4 +- acme/acme/messages.py | 6 +- acme/acme/standalone.py | 4 +- acme/tests/client_test.py | 2 +- acme/tests/crypto_util_test.py | 2 +- acme/tests/messages_test.py | 3 +- acme/tests/standalone_test.py | 4 +- .../certbot_apache/_internal/augeasparser.py | 2 +- .../certbot_apache/_internal/configurator.py | 33 +++--- .../certbot_apache/_internal/http_01.py | 4 +- .../certbot_apache/_internal/obj.py | 4 +- .../_internal/override_centos.py | 6 +- .../certbot_apache/_internal/parser.py | 11 +- certbot-apache/tests/augeasnode_test.py | 2 +- certbot-apache/tests/http_01_test.py | 2 +- .../nginx_tests/test_main.py | 4 +- .../utils/acme_server.py | 2 +- .../utils/dns_server.py | 5 +- .../configurators/nginx/common.py | 2 +- .../certbot_compatibility_test/test_driver.py | 2 +- .../_internal/dns_cloudflare.py | 2 +- .../_internal/dns_route53.py | 2 +- .../certbot_nginx/_internal/configurator.py | 8 +- .../certbot_nginx/_internal/constants.py | 4 +- .../certbot_nginx/_internal/http_01.py | 2 +- .../certbot_nginx/_internal/nginxparser.py | 102 +++++++++--------- .../certbot_nginx/_internal/parser.py | 15 ++- .../certbot_nginx/_internal/parser_obj.py | 9 +- certbot-nginx/tests/parser_test.py | 2 +- certbot/certbot/_internal/account.py | 18 ++-- certbot/certbot/_internal/auth_handler.py | 7 +- certbot/certbot/_internal/cert_manager.py | 4 +- certbot/certbot/_internal/cli/__init__.py | 2 +- certbot/certbot/_internal/cli/cli_utils.py | 2 +- certbot/certbot/_internal/cli/helpful.py | 4 +- certbot/certbot/_internal/client.py | 6 +- certbot/certbot/_internal/eff.py | 24 ++--- certbot/certbot/_internal/error_handler.py | 9 +- certbot/certbot/_internal/hooks.py | 4 +- certbot/certbot/_internal/lock.py | 71 +++++------- certbot/certbot/_internal/main.py | 37 +++---- certbot/certbot/_internal/plugins/disco.py | 4 +- certbot/certbot/_internal/plugins/manual.py | 5 +- .../certbot/_internal/plugins/standalone.py | 11 +- certbot/certbot/_internal/plugins/webroot.py | 11 +- certbot/certbot/_internal/renewal.py | 25 +++-- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/_internal/snap_config.py | 3 +- certbot/certbot/compat/filesystem.py | 58 ++++------ certbot/certbot/compat/misc.py | 15 +-- certbot/certbot/crypto_util.py | 15 ++- certbot/certbot/display/util.py | 8 +- certbot/certbot/ocsp.py | 16 ++- certbot/certbot/plugins/common.py | 6 +- certbot/certbot/plugins/dns_common_lexicon.py | 7 +- certbot/certbot/plugins/enhancements.py | 4 +- certbot/certbot/plugins/storage.py | 2 +- certbot/certbot/util.py | 8 +- certbot/tests/display/completer_test.py | 2 +- certbot/tests/error_handler_test.py | 2 +- certbot/tests/hook_test.py | 2 +- certbot/tests/log_test.py | 2 +- certbot/tests/main_test.py | 8 +- certbot/tests/plugins/disco_test.py | 2 +- certbot/tests/plugins/selection_test.py | 2 +- certbot/tests/plugins/standalone_test.py | 5 +- 68 files changed, 307 insertions(+), 371 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 58b457e0d..82f7ae1b8 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge.""" - TYPES = {} # type: dict + TYPES: dict = {} @classmethod def from_json(cls, jobj): @@ -38,7 +38,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" - TYPES = {} # type: dict + TYPES: dict = {} resource_type = 'challenge' resource = fields.Resource(resource_type) diff --git a/acme/acme/client.py b/acme/acme/client.py index b1b2089cf..f5aa1ff9e 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -112,8 +112,9 @@ class ClientBase: """ return self.update_registration(regr, update={'status': 'deactivated'}) - def deactivate_authorization(self, authzr): - # type: (messages.AuthorizationResource) -> messages.AuthorizationResource + def deactivate_authorization(self, + authzr: messages.AuthorizationResource + ) -> messages.AuthorizationResource: """Deactivate authorization. :param messages.AuthorizationResource authzr: The Authorization resource @@ -423,7 +424,7 @@ class Client(ClientBase): """ assert max_attempts > 0 - attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int] + attempts: Dict[messages.AuthorizationResource, int] = collections.defaultdict(int) exhausted = set() # priority queue with datetime.datetime (based on Retry-After) as key, @@ -536,7 +537,7 @@ class Client(ClientBase): :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509` """ - chain = [] # type: List[jose.ComparableX509] + chain: List[jose.ComparableX509] = [] uri = certr.cert_chain_uri while uri is not None and len(chain) < max_length: response, cert = self._get_cert(uri) @@ -968,7 +969,7 @@ class ClientNetwork: self.account = account self.alg = alg self.verify_ssl = verify_ssl - self._nonces = set() # type: Set[Text] + self._nonces: Set[Text] = set() self.user_agent = user_agent self.session = requests.Session() self._default_timeout = timeout diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 04faf6912..749478bf5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -168,7 +168,7 @@ def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-argu source_address[1] ) if any(source_address) else "" ) - socket_tuple = (host, port) # type: Tuple[str, int] + socket_tuple: Tuple[str, int] = (host, port) sock = socket.create_connection(socket_tuple, **socket_kwargs) # type: ignore except socket.error as error: raise errors.Error(error) @@ -256,7 +256,7 @@ def _pyopenssl_cert_or_req_san(cert_or_req): if isinstance(cert_or_req, crypto.X509): # pylint: disable=line-too-long - func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] + func: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]] = crypto.dump_certificate else: func = crypto.dump_certificate_request text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8") diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 0d73037ae..5e7e22c34 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -153,7 +153,7 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -166,7 +166,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} # type: dict + POSSIBLE_NAMES: dict = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -184,7 +184,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} # type: dict + _REGISTERED_TYPES: dict = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 123899470..0b5a8b5c7 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -63,8 +63,8 @@ class BaseDualNetworkedServers: def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] - self.threads = [] # type: List[threading.Thread] - self.servers = [] # type: List[ACMEServerMixin] + self.threads: List[threading.Thread] = [] + self.servers: List[ACMEServerMixin] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 6f9aecda2..14247335c 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -61,7 +61,7 @@ class ClientTestBase(unittest.TestCase): self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212') reg = messages.Registration( contact=self.contact, key=KEY.public_key()) - the_arg = dict(reg) # type: Dict + the_arg: Dict = dict(reg) self.new_reg = messages.NewRegistration(**the_arg) self.regr = messages.RegistrationResource( body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1') diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index f271ad37d..8075b68ed 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -180,7 +180,7 @@ class RandomSnTest(unittest.TestCase): def setUp(self): self.cert_count = 5 - self.serial_num = [] # type: List[int] + self.serial_num: List[int] = [] self.key = OpenSSL.crypto.PKey() self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 74d1737ec..99a3a9ce4 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -1,4 +1,5 @@ """Tests for acme.messages.""" +from typing import Dict import unittest from unittest import mock @@ -81,7 +82,7 @@ class ConstantTest(unittest.TestCase): from acme.messages import _Constant class MockConstant(_Constant): # pylint: disable=missing-docstring - POSSIBLE_NAMES = {} # type: Dict + POSSIBLE_NAMES: Dict = {} self.MockConstant = MockConstant # pylint: disable=invalid-name self.const_a = MockConstant('a') diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index e6aa8f2d6..a02d008a0 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -41,7 +41,7 @@ class HTTP01ServerTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01Server self.server = HTTP01Server(('', 0), resources=self.resources) @@ -218,7 +218,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase): def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) - self.resources = set() # type: Set + self.resources: Set = set() from acme.standalone import HTTP01DualNetworkedServers self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources) diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 461b5cb7b..4549e935a 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -355,7 +355,7 @@ class AugeasBlockNode(AugeasDirectiveNode): ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) - already_parsed = set() # type: Set[str] + already_parsed: Set[str] = set() for directive in directives: # Remove the /arg part from the Augeas path directive = directive.partition("/arg")[0] diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index ecd5c46de..29648d4c1 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -16,11 +16,6 @@ from typing import Union import zope.component import zope.interface -try: - import apacheconfig - HAS_APACHECONFIG = True -except ImportError: # pragma: no cover - HAS_APACHECONFIG = False from acme import challenges from certbot import errors @@ -41,6 +36,12 @@ from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + logger = logging.getLogger(__name__) @@ -211,23 +212,23 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = {} # type: Dict[str, obj.VirtualHost] + self.assoc: Dict[str, obj.VirtualHost] = {} # Outstanding challenges - self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] + self._chall_out: Set[KeyAuthorizationAnnotatedChallenge] = set() # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Maps enhancements to vhosts we've enabled the enhancement for - self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] + self._enhanced_vhosts: DefaultDict[str, Set[obj.VirtualHost]] = defaultdict(set) # Temporary state for AutoHSTS enhancement - self._autohsts = {} # type: Dict[str, Dict[str, Union[int, float]]] + self._autohsts: Dict[str, Dict[str, Union[int, float]]] = {} # Reverter save notes self.save_notes = "" # Should we use ParserNode implementation instead of the old behavior self.USE_PARSERNODE = use_parsernode # Saves the list of file paths that were parsed initially, and # not added to parser tree by self.conf("vhost-root") for example. - self.parsed_paths = [] # type: List[str] + self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False self.parser = None @@ -833,7 +834,7 @@ class ApacheConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() vhost_macro = [] @@ -997,8 +998,8 @@ class ApacheConfigurator(common.Installer): """ # Search base config, and all included paths for VirtualHosts - file_paths = {} # type: Dict[str, str] - internal_paths = defaultdict(set) # type: DefaultDict[str, Set[str]] + file_paths: Dict[str, str] = {} + internal_paths: DefaultDict[str, Set[str]] = defaultdict(set) vhs = [] # Make a list of parser paths because the parser_paths # dictionary may be modified during the loop. @@ -2157,7 +2158,7 @@ class ApacheConfigurator(common.Installer): # There can be other RewriteRule directive lines in vhost config. # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. - rewrite_args_dict = defaultdict(list) # type: DefaultDict[str, List[str]] + rewrite_args_dict: DefaultDict[str, List[str]] = defaultdict(list) pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) @@ -2251,7 +2252,7 @@ class ApacheConfigurator(common.Installer): if ssl_vhost.aliases: serveralias = "ServerAlias " + " ".join(ssl_vhost.aliases) - rewrite_rule_args = [] # type: List[str] + rewrite_rule_args: List[str] = [] if self.get_version() >= (2, 3, 9): rewrite_rule_args = constants.REWRITE_HTTPS_ARGS_WITH_END else: diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 4bd935d2d..66d888e97 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -57,7 +57,7 @@ class ApacheHttp01(common.ChallengePerformer): self.challenge_dir = os.path.join( self.configurator.config.work_dir, "http_challenges") - self.moded_vhosts = set() # type: Set[VirtualHost] + self.moded_vhosts: Set[VirtualHost] = set() def perform(self): """Perform all HTTP-01 challenges.""" @@ -93,7 +93,7 @@ class ApacheHttp01(common.ChallengePerformer): self.configurator.enable_mod(mod, temp=True) def _mod_config(self): - selected_vhosts = [] # type: List[VirtualHost] + selected_vhosts: List[VirtualHost] = [] http_port = str(self.configurator.config.http01_port) for chall in self.achalls: # Search for matching VirtualHosts diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 5e4680e1b..21fd042f6 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -137,7 +137,7 @@ class VirtualHost: def get_names(self): """Return a set of all names.""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() all_names.update(self.aliases) # Strip out any scheme:// and field from servername if self.name is not None: @@ -245,7 +245,7 @@ class VirtualHost: # already_found acts to keep everything very conservative. # Don't allow multiple ip:ports in same set. - already_found = set() # type: Set[str] + already_found: Set[str] = set() for addr in vhost.addrs: for local_addr in self.addrs: diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 4cde65d57..54edfc911 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -102,9 +102,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): loadmods = self.parser.find_dir("LoadModule", "ssl_module", exclude=False) - correct_ifmods = [] # type: List[str] - loadmod_args = [] # type: List[str] - loadmod_paths = [] # type: List[str] + correct_ifmods: List[str] = [] + loadmod_args: List[str] = [] + loadmod_paths: List[str] = [] for m in loadmods: noarg_path = m.rpartition("/")[0] path_args = self.parser.get_all_args(noarg_path) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index eb8f267f9..1a6af4c4b 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -6,7 +6,6 @@ import re from typing import Dict from typing import List - from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -49,9 +48,9 @@ class ApacheParser: "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = {} # type: Dict[str, str] - self.parser_paths = {} # type: Dict[str, List[str]] - self.variables = {} # type: Dict[str, str] + self.modules: Dict[str, str] = {} + self.parser_paths: Dict[str, List[str]] = {} + self.variables: Dict[str, str] = {} # Find configuration root and make sure augeas can parse it. self.root = os.path.abspath(root) @@ -264,7 +263,7 @@ class ApacheParser: the iteration issue. Else... parse and enable mods at same time. """ - mods = {} # type: Dict[str, str] + mods: Dict[str, str] = {} matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -551,7 +550,7 @@ class ApacheParser: else: arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg) - ordered_matches = [] # type: List[str] + ordered_matches: List[str] = [] # TODO: Wildcards should be included in alphabetical order # https://httpd.apache.org/docs/2.4/mod/core.html#include diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index abe72a5d0..83224fc98 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -107,7 +107,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def test_set_parameters(self): servernames = self.config.parser_root.find_directives("servername") - names = [] # type: List[str] + names: List[str] = [] for servername in servernames: names += servername.parameters self.assertFalse("going_to_set_this" in names) diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 696cd4a54..06e19e445 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -26,7 +26,7 @@ class ApacheHttp01Test(util.ApacheTest): super(ApacheHttp01Test, self).setUp(*args, **kwargs) self.account_key = self.rsa512jwk - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") # Takes the vhosts for encryption-example.demo, certbot.demo diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py index 4c746e1ee..2c52c8523 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -32,8 +32,8 @@ def test_context(request): '--preferred-challenges', 'http' ], {'default_server': False}), ], indirect=['context']) -def test_certificate_deployment(certname_pattern, params, context): - # type: (str, List[str], nginx_context.IntegrationTestsContext) -> None +def test_certificate_deployment(certname_pattern: str, params: List[str], + context: nginx_context.IntegrationTestsContext) -> None: """ Test various scenarios to deploy a certificate to nginx using certbot. """ diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 706c3ebae..ceebbe7ed 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -51,7 +51,7 @@ class ACMEServer: self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() - self._processes = [] # type: List[subprocess.Popen] + self._processes: List[subprocess.Popen] = [] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') self._dns_server = dns_server self._http_01_port = http_01_port diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index eaa601f1b..31e7aee18 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -38,7 +38,7 @@ class DNSServer: self.bind_root = tempfile.mkdtemp() - self.process = None # type: subprocess.Popen + self.process: subprocess.Popen = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -111,8 +111,7 @@ class DNSServer: self.stop() raise - def _wait_until_ready(self, attempts=30): - # type: (int) -> None + def _wait_until_ready(self, attempts: int = 30) -> None: """ Polls the DNS server over TCP until it gets a response, or until it runs out of attempts and raises a ValueError. diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index 58b72c205..d7b7a120e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -68,7 +68,7 @@ def _get_server_root(config): def _get_names(config): """Returns all and testable domain names in config""" - all_names = set() # type: Set[str] + all_names: Set[str] = set() for root, _dirs, files in os.walk(config): for this_file in files: update_names = _get_server_names(root, this_file) diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index fe634ba9f..4bb16e63f 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -178,7 +178,7 @@ def test_enhancements(plugin, domains): "enhancements") return False - domains_and_info = [(domain, []) for domain in domains] # type: List[Tuple[str, List[bool]]] + domains_and_info: List[Tuple[str, List[bool]]] = [(domain, []) for domain in domains] for domain, info in domains_and_info: try: diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 583b41410..034d7bbb2 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -172,7 +172,7 @@ class _CloudflareClient: """ zone_name_guesses = dns_common.base_domain_name_guesses(domain) - zones = [] # type: List[Dict[str, Any]] + zones: List[Dict[str, Any]] = [] code = msg = None for zone_name in zone_name_guesses: diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 4837e2036..4dda13f1d 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -39,7 +39,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) self.r53 = boto3.client("route53") - self._resource_records = collections.defaultdict(list) # type: DefaultDict[str, List[Dict[str, str]]] + self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) def more_info(self): # pylint: disable=missing-function-docstring return "Solve a DNS01 challenge using AWS Route53" diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 33baf076a..aab489315 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -112,8 +112,8 @@ class NginxConfigurator(common.Installer): # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] - self._wildcard_redirect_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts: Dict[str, List[obj.VirtualHost]] = {} + self._wildcard_redirect_vhosts: Dict[str, List[obj.VirtualHost]] = {} # Add number of outstanding challenges self._chall_out = 0 @@ -641,7 +641,7 @@ class NginxConfigurator(common.Installer): :rtype: set """ - all_names = set() # type: Set[str] + all_names: Set[str] = set() for vhost in self.parser.get_vhosts(): try: @@ -1222,7 +1222,7 @@ def nginx_restart(nginx_ctl, nginx_conf, sleep_duration): """ try: - reload_output = u"" # type: Text + reload_output: Text = u"" with tempfile.TemporaryFile() as out: proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], env=util.env_no_snap_for_external_calls(), diff --git a/certbot-nginx/certbot_nginx/_internal/constants.py b/certbot-nginx/certbot_nginx/_internal/constants.py index 7b4111577..52f4d8238 100644 --- a/certbot-nginx/certbot_nginx/_internal/constants.py +++ b/certbot-nginx/certbot_nginx/_internal/constants.py @@ -14,11 +14,11 @@ elif platform.system() in ('NetBSD',): else: server_root_tmp = LINUX_SERVER_ROOT -CLI_DEFAULTS = dict( +CLI_DEFAULTS: Dict[str, Any] = dict( server_root=server_root_tmp, ctl="nginx", sleep_seconds=1 -) # type: Dict[str, Any] +) """CLI defaults.""" diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index ec610f15b..abcdfbd53 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -113,7 +113,7 @@ class NginxHttp01(common.ChallengePerformer): :returns: list of :class:`certbot_nginx._internal.obj.Addr` to apply :rtype: list """ - addresses = [] # type: List[obj.Addr] + addresses: List[obj.Addr] = [] default_addr = "%s" % self.configurator.config.http01_port ipv6_addr = "[::]:{0}".format( self.configurator.config.http01_port) diff --git a/certbot-nginx/certbot_nginx/_internal/nginxparser.py b/certbot-nginx/certbot_nginx/_internal/nginxparser.py index 1aee1f329..787f7c363 100644 --- a/certbot-nginx/certbot_nginx/_internal/nginxparser.py +++ b/certbot-nginx/certbot_nginx/_internal/nginxparser.py @@ -2,8 +2,8 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging -from typing import Any # pylint: disable=unused-import -from typing import IO # pylint: disable=unused-import +from typing import Any +from typing import IO from pyparsing import Combine from pyparsing import Forward @@ -20,6 +20,7 @@ from pyparsing import ZeroOrMore logger = logging.getLogger(__name__) + class RawNginxParser: # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" @@ -105,57 +106,9 @@ class RawNginxDumper: return ''.join(self) -# Shortcut functions to respect Python's serialization interface -# (like pyyaml, picker or json) - -def loads(source): - """Parses from a string. - - :param str source: The string to parse - :returns: The parsed tree - :rtype: list - - """ - return UnspacedList(RawNginxParser(source).as_list()) - - -def load(_file): - """Parses from a file. - - :param file _file: The file to parse - :returns: The parsed tree - :rtype: list - - """ - return loads(_file.read()) - - -def dumps(blocks): - # type: (UnspacedList) -> str - """Dump to a Unicode string. - - :param UnspacedList block: The parsed tree - :rtype: str - - """ - return str(RawNginxDumper(blocks.spaced)) - - -def dump(blocks, _file): - # type: (UnspacedList, IO[Any]) -> None - """Dump to a file. - - :param UnspacedList block: The parsed tree - :param IO[Any] _file: The file stream to dump to. It must be opened with - Unicode encoding. - :rtype: None - - """ - _file.write(dumps(blocks)) - - spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' + class UnspacedList(list): """Wrap a list [of lists], making any whitespace entries magically invisible""" @@ -274,3 +227,50 @@ class UnspacedList(list): idx -= 1 pos += 1 return idx0 + spaces + + +# Shortcut functions to respect Python's serialization interface +# (like pyyaml, picker or json) + +def loads(source): + """Parses from a string. + + :param str source: The string to parse + :returns: The parsed tree + :rtype: list + + """ + return UnspacedList(RawNginxParser(source).as_list()) + + +def load(_file): + """Parses from a file. + + :param file _file: The file to parse + :returns: The parsed tree + :rtype: list + + """ + return loads(_file.read()) + + +def dumps(blocks: UnspacedList) -> str: + """Dump to a Unicode string. + + :param UnspacedList block: The parsed tree + :rtype: six.text_type + + """ + return str(RawNginxDumper(blocks.spaced)) + + +def dump(blocks: UnspacedList, _file: IO[Any]) -> None: + """Dump to a file. + + :param UnspacedList block: The parsed tree + :param IO[Any] _file: The file stream to dump to. It must be opened with + Unicode encoding. + :rtype: None + + """ + _file.write(dumps(blocks)) diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index ed7e211d5..8b353c095 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -32,7 +32,7 @@ class NginxParser: """ def __init__(self, root): - self.parsed = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + self.parsed: Dict[str, Union[List, nginxparser.UnspacedList]] = {} self.root = os.path.abspath(root) self.config_root = self._find_config_root() @@ -94,7 +94,7 @@ class NginxParser: """ servers = self._get_raw_servers() - addr_to_ssl = {} # type: Dict[Tuple[str, str], bool] + addr_to_ssl: Dict[Tuple[str, str], bool] = {} for filename in servers: for server, _ in servers[filename]: # Parse the server block to save addr info @@ -106,12 +106,11 @@ class NginxParser: addr_to_ssl[addr_tuple] = addr.ssl or addr_to_ssl[addr_tuple] return addr_to_ssl - def _get_raw_servers(self): + def _get_raw_servers(self) -> Dict: # pylint: disable=cell-var-from-loop - # type: () -> Dict """Get a map of unparsed all server blocks """ - servers = {} # type: Dict[str, Union[List, nginxparser.UnspacedList]] + servers: Dict[str, Union[List, nginxparser.UnspacedList]] = {} for filename in self.parsed: tree = self.parsed[filename] servers[filename] = [] @@ -741,9 +740,9 @@ def _parse_server_raw(server): :rtype: dict """ - addrs = set() # type: Set[obj.Addr] - ssl = False # type: bool - names = set() # type: Set[str] + addrs: Set[obj.Addr] = set() + ssl: bool = False + names: Set[str] = set() apply_ssl_to_all_addrs = False diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 9efd6651a..37392ea80 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -5,7 +5,6 @@ import abc import logging from typing import List - from certbot import errors logger = logging.getLogger(__name__) @@ -23,7 +22,7 @@ class Parsable: __metaclass__ = abc.ABCMeta def __init__(self, parent=None): - self._data = [] # type: List[object] + self._data: List[object] = [] self._tabs = None self.parent = parent @@ -182,7 +181,7 @@ class Statements(Parsable): def _space_list(list_): """ Inserts whitespace between adjacent non-whitespace tokens. """ - spaced_statement = [] # type: List[str] + spaced_statement: List[str] = [] for i in reversed(range(len(list_))): spaced_statement.insert(0, list_[i]) if i > 0 and not list_[i].isspace() and not list_[i-1].isspace(): @@ -271,8 +270,8 @@ class Block(Parsable): """ def __init__(self, parent=None): super(Block, self).__init__(parent) - self.names = None # type: Sentence - self.contents = None # type: Block + self.names: Sentence = None + self.contents: Block = None @staticmethod def should_parse(lists): diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 23fe390ad..35173d094 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -112,7 +112,7 @@ class NginxParserTest(util.NginxTest): ([[[0], [3], [4]], [[5], [3], [0]]], [])] for mylist, result in mylists: - paths = [] # type: List[List[int]] + paths: List[List[int]] = [] parser._do_for_subarray(mylist, lambda x: isinstance(x, list) and len(x) >= 1 and diff --git a/certbot/certbot/_internal/account.py b/certbot/certbot/_internal/account.py index 07ae95e9d..c5667a865 100644 --- a/certbot/certbot/_internal/account.py +++ b/certbot/certbot/_internal/account.py @@ -229,8 +229,7 @@ class AccountFileStorage(interfaces.AccountStorage): def load(self, account_id): return self._load_for_server_path(account_id, self.config.server_path) - def save(self, account, client): - # type: (Account, ClientBase) -> None + def save(self, account: Account, client: ClientBase) -> None: """Create a new account. :param Account account: account to create @@ -245,8 +244,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_regr(self, account, client): - # type: (Account, ClientBase) -> None + def update_regr(self, account: Account, client: ClientBase) -> None: """Update the registration resource. :param Account account: account to update @@ -259,8 +257,7 @@ class AccountFileStorage(interfaces.AccountStorage): except IOError as error: raise errors.AccountStorageError(error) - def update_meta(self, account): - # type: (Account) -> None + def update_meta(self, account: Account) -> None: """Update the meta resource. :param Account account: account to update @@ -338,19 +335,16 @@ class AccountFileStorage(interfaces.AccountStorage): return dir_path - def _prepare(self, account): - # type: (Account) -> str + def _prepare(self, account: Account) -> str: account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, self.config.strict_permissions) return account_dir_path - def _create(self, account, dir_path): - # type: (Account, str) -> None + def _create(self, account: Account, dir_path: str) -> None: with util.safe_open(self._key_path(dir_path), "w", chmod=0o400) as key_file: key_file.write(account.key.json_dumps()) - def _update_regr(self, account, acme, dir_path): - # type: (Account, ClientBase, str) -> None + def _update_regr(self, account: Account, acme: ClientBase, dir_path: str) -> None: with open(self._regr_path(dir_path), "w") as regr_file: regr = account.regr # If we have a value for new-authz, save it for forwards diff --git a/certbot/certbot/_internal/auth_handler.py b/certbot/certbot/_internal/auth_handler.py index 2801e3c2f..c2f323a36 100644 --- a/certbot/certbot/_internal/auth_handler.py +++ b/certbot/certbot/_internal/auth_handler.py @@ -98,8 +98,7 @@ class AuthHandler: return authzrs_validated - def deactivate_valid_authorizations(self, orderr): - # type: (messages.OrderResource) -> Tuple[List, List] + def deactivate_valid_authorizations(self, orderr: messages.OrderResource) -> Tuple[List, List]: """ Deactivate all `valid` authorizations in the order, so that they cannot be re-used in subsequent orders. @@ -191,7 +190,7 @@ class AuthHandler: """ pending_authzrs = [authzr for authzr in authzrs if authzr.body.status != messages.STATUS_VALID] - achalls = [] # type: List[achallenges.AnnotatedChallenge] + achalls: List[achallenges.AnnotatedChallenge] = [] if pending_authzrs: logger.info("Performing the following challenges:") for authzr in pending_authzrs: @@ -428,7 +427,7 @@ _ERROR_HELP = { def _report_failed_authzrs(failed_authzrs, account_key): """Notifies the user about failed authorizations.""" - problems = {} # type: Dict[str, List[achallenges.AnnotatedChallenge]] + problems: Dict[str, List[achallenges.AnnotatedChallenge]] = {} failed_achalls = [challb_to_achall(challb, account_key, authzr.body.identifier.value) for authzr in failed_authzrs for challb in authzr.body.challenges if challb.error] diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 2ac9fc538..8fab5735a 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -241,7 +241,7 @@ def match_and_check_overlaps(cli_config, acceptable_matches, match_func, rv_func def find_matches(candidate_lineage, return_value, acceptable_matches): """Returns a list of matches using _search_lineages.""" acceptable_matches = [func(candidate_lineage) for func in acceptable_matches] - acceptable_matches_rv = [] # type: List[str] + acceptable_matches_rv: List[str] = [] for item in acceptable_matches: if isinstance(item, list): acceptable_matches_rv += item @@ -364,7 +364,7 @@ def _report_human_readable(config, parsed_certs): def _describe_certs(config, parsed_certs, parse_failures): """Print information about the certs we know about""" - out = [] # type: List[str] + out: List[str] = [] notify = out.append diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 30f444e5b..b64ab0e9c 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -66,7 +66,7 @@ logger = logging.getLogger(__name__) # Global, to save us from a lot of argument passing within the scope of this module -helpful_parser = None # type: Optional[HelpfulArgumentParser] +helpful_parser: Optional[HelpfulArgumentParser] = None def prepare_and_parse_args(plugins, args, detect_defaults=False): diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index f1f9d0f3c..1f30261c1 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -62,7 +62,7 @@ def config_help(name, hidden=False): """Extract the help message for an `.IConfig` attribute.""" if hidden: return argparse.SUPPRESS - field = interfaces.IConfig.__getitem__(name) # type: zope.interface.interface.Attribute + field: zope.interface.interface.Attribute = interfaces.IConfig.__getitem__(name) return field.__doc__ diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index cbac4e232..d36147089 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -105,9 +105,9 @@ class HelpfulArgumentParser: self.visible_topics = self.determine_help_topics(self.help_arg) # elements are added by .add_group() - self.groups = {} # type: Dict[str, argparse._ArgumentGroup] + self.groups: Dict[str, argparse._ArgumentGroup] = {} # elements are added by .parse_args() - self.defaults = {} # type: Dict[str, Any] + self.defaults: Dict[str, Any] = {} self.parser = configargparse.ArgParser( prog="certbot", diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index b9137e249..ba696bf85 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -328,7 +328,7 @@ class Client: with open(old_keypath, "rb") as f: keypath = old_keypath keypem = f.read() - key = util.Key(file=keypath, pem=keypem) # type: Optional[util.Key] + key: Optional[util.Key] = util.Key(file=keypath, pem=keypem) logger.info("Reusing existing private key from %s.", old_keypath) else: # The key is set to None here but will be created below. @@ -390,8 +390,8 @@ class Client: cert, chain = self.obtain_certificate_from_csr(csr, orderr) return cert, chain, key, csr - def _get_order_and_authorizations(self, csr_pem, best_effort): - # type: (str, bool) -> List[messages.OrderResource] + def _get_order_and_authorizations(self, csr_pem: str, + best_effort: bool) -> List[messages.OrderResource]: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. diff --git a/certbot/certbot/_internal/eff.py b/certbot/certbot/_internal/eff.py index 6212c24ee..b01e2dd61 100644 --- a/certbot/certbot/_internal/eff.py +++ b/certbot/certbot/_internal/eff.py @@ -1,22 +1,21 @@ """Subscribes users to the EFF newsletter.""" import logging -from typing import Optional # pylint: disable=unused-import +from typing import Optional import requests import zope.component from certbot import interfaces from certbot._internal import constants -from certbot._internal.account import Account # pylint: disable=unused-import +from certbot._internal.account import Account from certbot._internal.account import AccountFileStorage from certbot.display import util as display_util -from certbot.interfaces import IConfig # pylint: disable=unused-import +from certbot.interfaces import IConfig logger = logging.getLogger(__name__) -def prepare_subscription(config, acc): - # type: (IConfig, Account) -> None +def prepare_subscription(config: IConfig, acc: Account) -> None: """High level function to store potential EFF newsletter subscriptions. The user may be asked if they want to sign up for the newsletter if @@ -44,8 +43,7 @@ def prepare_subscription(config, acc): storage.update_meta(acc) -def handle_subscription(config, acc): - # type: (IConfig, Account) -> None +def handle_subscription(config: IConfig, acc: Account) -> None: """High level function to take care of EFF newsletter subscriptions. Once subscription is handled, it will not be handled again. @@ -64,8 +62,7 @@ def handle_subscription(config, acc): storage.update_meta(acc) -def _want_subscription(): - # type: () -> bool +def _want_subscription() -> bool: """Does the user want to be subscribed to the EFF newsletter? :returns: True if we should subscribe the user, otherwise, False @@ -82,8 +79,7 @@ def _want_subscription(): return display.yesno(prompt, default=False) -def subscribe(email): - # type: (str) -> None +def subscribe(email: str) -> None: """Subscribe the user to the EFF mailing list. :param str email: the e-mail address to subscribe @@ -98,8 +94,7 @@ def subscribe(email): _check_response(requests.post(url, data=data)) -def _check_response(response): - # type: (requests.Response) -> None +def _check_response(response: requests.Response) -> None: """Check for errors in the server's response. If an error occurred, it will be reported to the user. @@ -119,8 +114,7 @@ def _check_response(response): _report_failure('there was a problem with the server response') -def _report_failure(reason=None): - # type: (Optional[str]) -> None +def _report_failure(reason: Optional[str] = None) -> None: """Notify the user of failing to sign them up for the newsletter. :param reason: a phrase describing what the problem was diff --git a/certbot/certbot/_internal/error_handler.py b/certbot/certbot/_internal/error_handler.py index cb8301980..01cc92b42 100644 --- a/certbot/certbot/_internal/error_handler.py +++ b/certbot/certbot/_internal/error_handler.py @@ -77,9 +77,9 @@ class ErrorHandler: def __init__(self, func, *args, **kwargs): self.call_on_regular_exit = False self.body_executed = False - self.funcs = [] # type: List[Callable[[], Any]] - self.prev_handlers = {} # type: Dict[int, Union[int, None, Callable]] - self.received_signals = [] # type: List[int] + self.funcs: List[Callable[[], Any]] = [] + self.prev_handlers: Dict[int, Union[int, None, Callable]] = {} + self.received_signals: List[int] = [] if func is not None: self.register(func, *args, **kwargs) @@ -108,8 +108,7 @@ class ErrorHandler: self._call_signals() return retval - def register(self, func, *args, **kwargs): - # type: (Callable, *Any, **Any) -> None + def register(self, func: Callable, *args: Any, **kwargs: Any) -> None: """Sets func to be run with the given arguments during cleanup. :param function func: function to be called in case of an error diff --git a/certbot/certbot/_internal/hooks.py b/certbot/certbot/_internal/hooks.py index b695d9930..b9f1f1531 100644 --- a/certbot/certbot/_internal/hooks.py +++ b/certbot/certbot/_internal/hooks.py @@ -77,7 +77,7 @@ def pre_hook(config): _run_pre_hook_if_necessary(cmd) -executed_pre_hooks = set() # type: Set[str] +executed_pre_hooks: Set[str] = set() def _run_pre_hook_if_necessary(command): @@ -127,7 +127,7 @@ def post_hook(config): _run_hook("post-hook", cmd) -post_hooks = [] # type: List[str] +post_hooks: List[str] = [] def _run_eventually(command): diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index f878946cb..d00302598 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -16,28 +16,9 @@ else: POSIX_MODE = True - logger = logging.getLogger(__name__) -def lock_dir(dir_path): - # type: (str) -> LockFile - """Place a lock file on the directory at dir_path. - - The lock file is placed in the root of dir_path with the name - .certbot.lock. - - :param str dir_path: path to directory - - :returns: the locked LockFile object - :rtype: LockFile - - :raises errors.LockError: if unable to acquire the lock - - """ - return LockFile(os.path.join(dir_path, '.certbot.lock')) - - class LockFile: """ Platform independent file lock system. @@ -52,8 +33,7 @@ class LockFile: LockFile is platform independent: it will proceed to the appropriate OS lock mechanism depending on Linux or Windows. """ - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a LockFile instance on the given file path, and acquire lock. :param str path: the path to the file that will hold a lock @@ -64,8 +44,7 @@ class LockFile: self.acquire() - def __repr__(self): - # type: () -> str + def __repr__(self) -> str: repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) if self.is_locked(): repr_str += 'acquired>' @@ -73,23 +52,20 @@ class LockFile: repr_str += 'released>' return repr_str - def acquire(self): - # type: () -> None + def acquire(self) -> None: """ Acquire the lock on the file, forbidding any other Certbot instance to acquire it. :raises errors.LockError: if unable to acquire the lock """ self._lock_mechanism.acquire() - def release(self): - # type: () -> None + def release(self) -> None: """ Release the lock on the file, allowing any other Certbot instance to acquire it. """ self._lock_mechanism.release() - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """ Check if the file is currently locked. :return: True if the file is locked, False otherwise @@ -98,17 +74,15 @@ class LockFile: class _BaseLockMechanism: - def __init__(self, path): - # type: (str) -> None + def __init__(self, path: str) -> None: """ Create a lock file mechanism for Unix. :param str path: the path to the lock file """ self._path = path - self._fd = None # type: Optional[int] + self._fd: Optional[int] = None - def is_locked(self): - # type: () -> bool + def is_locked(self) -> bool: """Check if lock file is currently locked. :return: True if the lock file is locked :rtype: bool @@ -129,8 +103,7 @@ class _UnixLockMechanism(_BaseLockMechanism): process exits. It cannot be used to provide synchronization between threads. It is based on the lock_file package by Martin Horcicka. """ - def acquire(self): - # type: () -> None + def acquire(self) -> None: """Acquire the lock.""" while self._fd is None: # Open the file @@ -144,8 +117,7 @@ class _UnixLockMechanism(_BaseLockMechanism): if self._fd is None: os.close(fd) - def _try_lock(self, fd): - # type: (int) -> None + def _try_lock(self, fd: int) -> None: """ Try to acquire the lock file without blocking. :param int fd: file descriptor of the opened file to lock @@ -158,8 +130,7 @@ class _UnixLockMechanism(_BaseLockMechanism): raise errors.LockError('Another instance of Certbot is already running.') raise - def _lock_success(self, fd): - # type: (int) -> bool + def _lock_success(self, fd: int) -> bool: """ Did we successfully grab the lock? Because this class deletes the locked file when the lock is @@ -185,8 +156,7 @@ class _UnixLockMechanism(_BaseLockMechanism): # the same device and inode, they're the same file. return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino - def release(self): - # type: () -> None + def release(self) -> None: """Remove, close, and release the lock file.""" # It is important the lock file is removed before it's released, # otherwise: @@ -269,3 +239,20 @@ class _WindowsLockMechanism(_BaseLockMechanism): logger.debug(str(e)) finally: self._fd = None + + +def lock_dir(dir_path: str) -> LockFile: + """Place a lock file on the directory at dir_path. + + The lock file is placed in the root of dir_path with the name + .certbot.lock. + + :param str dir_path: path to directory + + :returns: the locked LockFile object + :rtype: LockFile + + :raises errors.LockError: if unable to acquire the lock + + """ + return LockFile(os.path.join(dir_path, '.certbot.lock')) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index ba8d94bca..ac4d18449 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,7 +4,7 @@ import functools import logging.handlers import sys -from typing import Iterable # pylint: disable=unused-import +from typing import Iterable from typing import List from typing import Optional from typing import Tuple @@ -145,8 +145,8 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N return lineage -def _handle_unexpected_key_type_migration(config, cert): - # type: (configuration.NamespaceConfig, storage.RenewableCert) -> None +def _handle_unexpected_key_type_migration(config: configuration.NamespaceConfig, + cert: storage.RenewableCert) -> None: """ This function ensures that the user will not implicitly migrate an existing key from one type to another in the situation where a certificate for that lineage @@ -167,11 +167,10 @@ def _handle_unexpected_key_type_migration(config, cert): raise errors.Error(msg) -def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - cert # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_subset_cert_request(config: configuration.NamespaceConfig, + domains: List[str], + cert: storage.RenewableCert + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a previous cert had a subset of the names now requested :param config: Configuration object @@ -218,10 +217,9 @@ def _handle_subset_cert_request(config, # type: configuration.NamespaceConfig raise errors.Error(USER_CANCELLED) -def _handle_identical_cert_request(config, # type: configuration.NamespaceConfig - lineage, # type: storage.RenewableCert - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _handle_identical_cert_request(config: configuration.NamespaceConfig, + lineage: storage.RenewableCert, + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Figure out what to do if a lineage has the same names as a previously obtained one :param config: Configuration object @@ -337,11 +335,10 @@ def _find_cert(config, domains, certname): return (action != "reinstall"), lineage -def _find_lineage_for_domains_and_certname(config, # type: configuration.NamespaceConfig - domains, # type: List[str] - certname # type: str - ): - # type: (...) -> Tuple[str, Optional[storage.RenewableCert]] +def _find_lineage_for_domains_and_certname(config: configuration.NamespaceConfig, + domains: List[str], + certname: str + ) -> Tuple[str, Optional[storage.RenewableCert]]: """Find appropriate lineage based on given domains and/or certname. :param config: Configuration object @@ -758,7 +755,7 @@ def update_account(config, unused_plugins): cb_client = client.Client(config, acc, None, None, acme=acme) # Empty list of contacts in case the user is removing all emails - acc_contacts = () # type: Iterable[str] + acc_contacts: Iterable[str] = () if config.email: acc_contacts = ['mailto:' + email for email in config.email.split(',')] # We rely on an exception to interrupt this process if it didn't work. @@ -1362,8 +1359,8 @@ def set_displayer(config): """ if config.quiet: config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w")) \ - # type: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] + displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\ + display_util.NoninteractiveDisplay(open(os.devnull, "w")) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 27b2605e0..4097544e8 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,9 +1,9 @@ """Utilities for plugins discovery and selection.""" +from collections.abc import Mapping import itertools import logging import sys from typing import Dict -from collections.abc import Mapping import pkg_resources import zope.interface @@ -213,7 +213,7 @@ class PluginsRegistry(Mapping): @classmethod def find_all(cls): """Find plugins using setuptools entry points.""" - plugins = {} # type: Dict[str, PluginEntryPoint] + plugins: Dict[str, PluginEntryPoint] = {} plugin_paths_string = os.getenv('CERTBOT_PLUGIN_PATH') plugin_paths = plugin_paths_string.split(':') if plugin_paths_string else [] # XXX should ensure this only happens once diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index c28e333b9..ed2e0559e 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -5,7 +5,7 @@ import zope.component import zope.interface from acme import challenges -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import errors from certbot import interfaces from certbot import reverter @@ -74,8 +74,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = {} \ - # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] + self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} self.subsequent_dns_challenge = False self.subsequent_any_challenge = False diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index f511c60e9..833651346 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -10,7 +10,7 @@ from typing import Set from typing import Tuple from typing import TYPE_CHECKING -import OpenSSL # pylint: disable=unused-import +import OpenSSL import zope.interface from acme import challenges @@ -42,7 +42,7 @@ class ServerManager: """ def __init__(self, certs, http_01_resources): - self._instances = {} # type: Dict[int, acme_standalone.BaseDualNetworkedServers] + self._instances: Dict[int, acme_standalone.BaseDualNetworkedServers] = {} self.certs = certs self.http_01_resources = http_01_resources @@ -122,15 +122,14 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.served = collections.defaultdict(set) # type: ServedType + self.served: ServedType = collections.defaultdict(set) # Stuff below is shared across threads (i.e. servers read # values, main thread writes). Due to the nature of CPython's # GIL, the operations are safe, c.f. # https://docs.python.org/2/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = set() \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = set() self.servers = ServerManager(self.certs, self.http_01_resources) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index 730241066..c5b436b65 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -12,10 +12,10 @@ import zope.component import zope.interface from acme import challenges -from certbot import achallenges # pylint: disable=unused-import from certbot import errors from certbot import interfaces from certbot._internal import cli +from certbot.achallenges import KeyAuthorizationAnnotatedChallenge as AnnotatedChallenge from certbot.compat import filesystem from certbot.compat import os from certbot.display import ops @@ -67,11 +67,10 @@ to serve all files under specified web root ({0}).""" def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.full_roots = {} # type: Dict[str, str] - self.performed = collections.defaultdict(set) \ - # type: DefaultDict[str, Set[achallenges.KeyAuthorizationAnnotatedChallenge]] + self.full_roots: Dict[str, str] = {} + self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator - self._created_dirs = [] # type: List[str] + self._created_dirs: List[str] = [] def prepare(self): # pylint: disable=missing-function-docstring pass @@ -224,7 +223,7 @@ to serve all files under specified web root ({0}).""" os.remove(validation_path) self.performed[root_path].remove(achall) - not_removed = [] # type: List[str] + not_removed: List[str] = [] while self._created_dirs: path = self._created_dirs.pop() try: diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py index a09edfa1e..f29709ef4 100644 --- a/certbot/certbot/_internal/renewal.py +++ b/certbot/certbot/_internal/renewal.py @@ -8,7 +8,7 @@ import sys import time import traceback from typing import List -from typing import Optional # pylint: disable=unused-import +from typing import Optional from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import ec @@ -22,7 +22,7 @@ from certbot import errors from certbot import interfaces from certbot import util from certbot._internal import cli -from certbot._internal import client # pylint: disable=unused-import +from certbot._internal import client from certbot._internal import constants from certbot._internal import hooks from certbot._internal import storage @@ -143,7 +143,7 @@ def _restore_plugin_configs(config, renewalparams): # longer defined, stored copies of that parameter will be # deserialized as strings by this logic even if they were # originally meant to be some other type. - plugin_prefixes = [] # type: List[str] + plugin_prefixes: List[str] = [] if renewalparams["authenticator"] == "webroot": _restore_webroot_config(config, renewalparams) else: @@ -290,7 +290,7 @@ def _restore_str(name, value): def should_renew(config, lineage): - "Return true if any of the circumstances for automatic renewal apply." + """Return true if any of the circumstances for automatic renewal apply.""" if config.renew_by_default: logger.debug("Auto-renewal forced with --force-renewal...") return True @@ -305,7 +305,7 @@ def should_renew(config, lineage): def _avoid_invalidating_lineage(config, lineage, original_server): - "Do not renew a valid cert with one from a staging server!" + """Do not renew a valid cert with one from a staging server!""" # Some lineages may have begun with --staging, but then had production # certificates added to them with open(lineage.cert) as the_file: @@ -323,8 +323,8 @@ def _avoid_invalidating_lineage(config, lineage, original_server): "unless you use the --break-my-certs flag!".format(names)) -def renew_cert(config, domains, le_client, lineage): - # type: (interfaces.IConfig, Optional[List[str]], client.Client, storage.RenewableCert) -> None +def renew_cert(config: interfaces.IConfig, domains: Optional[List[str]], le_client: client.Client, + lineage: storage.RenewableCert) -> None: """Renew a certificate lineage.""" renewal_params = lineage.configuration["renewalparams"] original_server = renewal_params.get("server", cli.flag_default("server")) @@ -351,14 +351,14 @@ def renew_cert(config, domains, le_client, lineage): def report(msgs, category): - "Format a results report for a category of renewal outcomes" + """Format a results report for a category of renewal outcomes""" lines = ("%s (%s)" % (m, category) for m in msgs) return " " + "\n ".join(lines) -def _renew_describe_results(config, renew_successes, renew_failures, - renew_skipped, parse_failures): - # type: (interfaces.IConfig, List[str], List[str], List[str], List[str]) -> None +def _renew_describe_results(config: interfaces.IConfig, renew_successes: List[str], + renew_failures: List[str], renew_skipped: List[str], + parse_failures: List[str]) -> None: """ Print a report to the terminal about the results of the renewal process. @@ -511,8 +511,7 @@ def handle_renewal_request(config): logger.debug("no renewal failures") -def _update_renewal_params_from_key(key_path, config): - # type: (str, interfaces.IConfig) -> None +def _update_renewal_params_from_key(key_path: str, config: interfaces.IConfig) -> None: with open(key_path, 'rb') as file_h: key = load_pem_private_key(file_h.read(), password=None, backend=default_backend()) if isinstance(key, rsa.RSAPrivateKey): diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 3093ef8ca..0268fbf34 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -32,7 +32,7 @@ class Reporter: _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') def __init__(self, config): - self.messages = queue.PriorityQueue() # type: queue.PriorityQueue[Reporter._msg_type] + self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() self.config = config def add_message(self, msg, priority, on_crash=True): diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index f2fe035db..1ba53cad4 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -33,8 +33,7 @@ _ARCH_TRIPLET_MAP = { LOGGER = logging.getLogger(__name__) -def prepare_env(cli_args): - # type: (List[str]) -> List[str] +def prepare_env(cli_args: List[str]) -> List[str]: """ Prepare runtime environment for a certbot execution in snap. :param list cli_args: List of command line arguments diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index aa6fe9134..25e1687da 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -35,8 +35,7 @@ class _WindowsUmask: _WINDOWS_UMASK = _WindowsUmask() -def chmod(file_path, mode): - # type: (str, int) -> None +def chmod(file_path: str, mode: int) -> None: """ Apply a POSIX mode on given file_path: @@ -57,8 +56,7 @@ def chmod(file_path, mode): _apply_win_mode(file_path, mode) -def umask(mask): - # type: (int) -> int +def umask(mask: int) -> int: """ Set the current numeric umask and return the previous umask. On Linux, the built-in umask method is used. On Windows, our Certbot-side implementation is used. @@ -84,8 +82,8 @@ def umask(mask): # Since copying and editing arbitrary DACL is very difficult, and since we actually know # the mode to apply at the time the owner of a file should change, it is easier to just # change the owner, then reapply the known mode, as copy_ownership_and_apply_mode() does. -def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): - # type: (str, str, int, bool, bool) -> None +def copy_ownership_and_apply_mode(src: str, dst: str, mode: int, + copy_user: bool, copy_group: bool) -> None: """ Copy ownership (user and optionally group on Linux) from the source to the destination, then apply given mode in compatible way for Linux and Windows. @@ -117,8 +115,8 @@ def copy_ownership_and_apply_mode(src, dst, mode, copy_user, copy_group): # equivalent POSIX mode, because ownership and mode are copied altogether on the destination # file, so no recomputing of the DACL against the new owner is needed, as it would be # for a copy_ownership alone method. -def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): - # type: (str, str, bool, bool) -> None +def copy_ownership_and_mode(src: str, dst: str, + copy_user: bool = True, copy_group: bool = True) -> None: """ Copy ownership (user and optionally group on Linux) and mode/DACL from the source to the destination. @@ -142,8 +140,7 @@ def copy_ownership_and_mode(src, dst, copy_user=True, copy_group=True): _copy_win_mode(src, dst) -def check_mode(file_path, mode): - # type: (str, int) -> bool +def check_mode(file_path: str, mode: int) -> bool: """ Check if the given mode matches the permissions of the given file. On Linux, will make a direct comparison, on Windows, mode will be compared against @@ -160,8 +157,7 @@ def check_mode(file_path, mode): return _check_win_mode(file_path, mode) -def check_owner(file_path): - # type: (str) -> bool +def check_owner(file_path: str) -> bool: """ Check if given file is owned by current user. @@ -183,8 +179,7 @@ def check_owner(file_path): return _get_current_user() == user -def check_permissions(file_path, mode): - # type: (str, int) -> bool +def check_permissions(file_path: str, mode: int) -> bool: """ Check if given file has the given mode and is owned by current user. @@ -196,8 +191,7 @@ def check_permissions(file_path, mode): return check_owner(file_path) and check_mode(file_path, mode) -def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin - # type: (str, int, int) -> int +def open(file_path: str, flags: int, mode: int = 0o777) -> int: # pylint: disable=redefined-builtin """ Wrapper of original os.open function, that will ensure on Windows that given mode is correctly applied. @@ -266,8 +260,7 @@ def open(file_path, flags, mode=0o777): # pylint: disable=redefined-builtin return handle -def makedirs(file_path, mode=0o777): - # type: (str, int) -> None +def makedirs(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.makedirs function, that will ensure on Windows that given mode is correctly applied. @@ -299,8 +292,7 @@ def makedirs(file_path, mode=0o777): umask(current_umask) -def mkdir(file_path, mode=0o777): - # type: (str, int) -> None +def mkdir(file_path: str, mode: int = 0o777) -> None: """ Rewrite of original os.mkdir function, that will ensure on Windows that given mode is correctly applied. @@ -331,8 +323,7 @@ def mkdir(file_path, mode=0o777): return None -def replace(src, dst): - # type: (str, str) -> None +def replace(src: str, dst: str) -> None: """ Rename a file to a destination path and handles situations where the destination exists. @@ -349,8 +340,7 @@ def replace(src, dst): os.rename(src, dst) -def realpath(file_path): - # type: (str) -> str +def realpath(file_path: str) -> str: """ Find the real path for the given path. This method resolves symlinks, including recursive symlinks, and is protected against symlinks that creates an infinite loop. @@ -371,7 +361,7 @@ def realpath(file_path): raise RuntimeError('Error, link {0} is a loop!'.format(original_path)) return path - inspected_paths = [] # type: List[str] + inspected_paths: List[str] = [] while os.path.islink(file_path): link_path = file_path file_path = os.readlink(file_path) @@ -384,8 +374,7 @@ def realpath(file_path): return os.path.abspath(file_path) -def readlink(link_path): - # type: (str) -> str +def readlink(link_path: str) -> str: """ Return a string representing the path to which the symbolic link points. @@ -419,8 +408,7 @@ def readlink(link_path): # elevated privileges or not. However this is not a problem since certbot always # requires to be run under a privileged shell, so the user will always benefit # from the highest (privileged one) set of permissions on a given file. -def is_executable(path): - # type: (str) -> bool +def is_executable(path: str) -> bool: """ Is path an executable file? @@ -434,8 +422,7 @@ def is_executable(path): return _win_is_executable(path) -def has_world_permissions(path): - # type: (str) -> bool +def has_world_permissions(path: str) -> bool: """ Check if everybody/world has any right (read/write/execute) on a file given its path. @@ -456,8 +443,7 @@ def has_world_permissions(path): })) -def compute_private_key_mode(old_key, base_mode): - # type: (str, int) -> int +def compute_private_key_mode(old_key: str, base_mode: int) -> int: """ Calculate the POSIX mode to apply to a private key given the previous private key. @@ -478,8 +464,7 @@ def compute_private_key_mode(old_key, base_mode): return base_mode -def has_same_ownership(path1, path2): - # type: (str, str) -> bool +def has_same_ownership(path1: str, path2: str) -> bool: """ Return True if the ownership of two files given their respective path is the same. On Windows, ownership is checked against owner only, since files do not have a group owner. @@ -504,8 +489,7 @@ def has_same_ownership(path1, path2): return user1 == user2 -def has_min_permissions(path, min_mode): - # type: (str, int) -> bool +def has_min_permissions(path: str, min_mode: int) -> bool: """ Check if a file given its path has at least the permissions defined by the given minimal mode. On Windows, group permissions are ignored since files do not have a group owner. diff --git a/certbot/certbot/compat/misc.py b/certbot/certbot/compat/misc.py index 297df80fc..3d2624906 100644 --- a/certbot/certbot/compat/misc.py +++ b/certbot/certbot/compat/misc.py @@ -27,8 +27,7 @@ logger = logging.getLogger(__name__) STANDARD_BINARY_DIRS = ["/usr/sbin", "/usr/local/bin", "/usr/local/sbin"] if POSIX_MODE else [] -def raise_for_non_administrative_windows_rights(): - # type: () -> None +def raise_for_non_administrative_windows_rights() -> None: """ On Windows, raise if current shell does not have the administrative rights. Do nothing on Linux. @@ -39,8 +38,7 @@ def raise_for_non_administrative_windows_rights(): raise errors.Error('Error, certbot must be run on a shell with administrative rights.') -def readline_with_timeout(timeout, prompt): - # type: (float, str) -> str +def readline_with_timeout(timeout: float, prompt: str) -> str: """ Read user input to return the first line entered, or raise after specified timeout. @@ -81,8 +79,7 @@ LINUX_DEFAULT_FOLDERS = { } -def get_default_folder(folder_type): - # type: (str) -> str +def get_default_folder(folder_type: str) -> str: """ Return the relevant default folder for the current OS @@ -99,8 +96,7 @@ def get_default_folder(folder_type): return WINDOWS_DEFAULT_FOLDERS[folder_type] -def underscores_for_unsupported_characters_in_path(path): - # type: (str) -> str +def underscores_for_unsupported_characters_in_path(path: str) -> str: """ Replace unsupported characters in path for current OS by underscores. :param str path: the path to normalize @@ -116,8 +112,7 @@ def underscores_for_unsupported_characters_in_path(path): return drive + tail.replace(':', '_') -def execute_command(cmd_name, shell_cmd, env=None): - # type: (str, str, Optional[dict]) -> Tuple[str, str] +def execute_command(cmd_name: str, shell_cmd: str, env: Optional[dict] = None) -> Tuple[str, str]: """ Run a command: - on Linux command will be run by the standard shell selected with Popen(shell=True) diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 5bc9251a4..445618ea0 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -7,7 +7,6 @@ import hashlib import logging import re -from typing import IO # pylint: disable=unused-import import warnings # See https://github.com/pyca/cryptography/issues/4275 @@ -272,9 +271,9 @@ def verify_renewable_cert_sig(renewable_cert): :raises errors.Error: If signature verification fails. """ try: - with open(renewable_cert.chain_path, 'rb') as chain_file: # type: IO[bytes] + with open(renewable_cert.chain_path, 'rb') as chain_file: chain = x509.load_pem_x509_certificate(chain_file.read(), default_backend()) - with open(renewable_cert.cert_path, 'rb') as cert_file: # type: IO[bytes] + with open(renewable_cert.cert_path, 'rb') as cert_file: cert = x509.load_pem_x509_certificate(cert_file.read(), default_backend()) pk = chain.public_key() with warnings.catch_warnings(): @@ -349,11 +348,11 @@ def verify_fullchain(renewable_cert): :raises errors.Error: If cert and chain do not combine to fullchain. """ try: - with open(renewable_cert.chain_path) as chain_file: # type: IO[str] + with open(renewable_cert.chain_path) as chain_file: chain = chain_file.read() - with open(renewable_cert.cert_path) as cert_file: # type: IO[str] + with open(renewable_cert.cert_path) as cert_file: cert = cert_file.read() - with open(renewable_cert.fullchain_path) as fullchain_file: # type: IO[str] + with open(renewable_cert.fullchain_path) as fullchain_file: fullchain = fullchain_file.read() if (cert + chain) != fullchain: error_str = "fullchain does not match cert + chain for {0}!" @@ -487,7 +486,7 @@ def _notAfterBefore(cert_path, method): """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) # pyopenssl always returns bytes timestamp = method(x509) @@ -564,7 +563,7 @@ def get_serial_from_cert(cert_path): :rtype: int """ # pylint: disable=redefined-outer-name - with open(cert_path, "rb") as f: # type: IO[bytes] + with open(cert_path, "rb") as f: x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read()) return x509.get_serial_number() diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 1e85ae808..9443ae266 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -14,8 +14,8 @@ import sys import textwrap from typing import List -import zope.interface import zope.component +import zope.interface from certbot import errors from certbot import interfaces @@ -98,8 +98,7 @@ def input_with_timeout(prompt=None, timeout=36000.0): return line.rstrip('\n') -def notify(msg): - # type: (str) -> None +def notify(msg: str) -> None: """Display a basic status message. :param str msg: message to display @@ -636,8 +635,7 @@ def _parens_around_char(label): return "({first}){rest}".format(first=label[0], rest=label[1:]) -def summarize_domain_list(domains): - # type: (List[str]) -> str +def summarize_domain_list(domains: List[str]) -> str: """Summarizes a list of domains in the format of: example.com.com and N more domains or if there is are only two domains: diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 9a18b32f2..0a842e108 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -59,8 +59,7 @@ class RevocationChecker: else: self.host_args = lambda host: ["Host", host] - def ocsp_revoked(self, cert): - # type: (RenewableCert) -> bool + def ocsp_revoked(self, cert: RenewableCert) -> bool: """Get revoked status for a particular cert version. .. todo:: Make this a non-blocking call @@ -72,8 +71,7 @@ class RevocationChecker: """ return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) - def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): - # type: (str, str, int) -> bool + def ocsp_revoked_by_paths(self, cert_path: str, chain_path: str, timeout: int = 10) -> bool: """Performs the OCSP revocation check :param str cert_path: Certificate filepath @@ -102,8 +100,8 @@ class RevocationChecker: return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): - # type: (str, str, str, str, int) -> bool + def _check_ocsp_openssl_bin(self, cert_path: str, chain_path: str, + host: str, url: str, timeout: int) -> bool: # Minimal implementation of proxy selection logic as seen in, e.g., cURL # Some things that won't work, but may well be in use somewhere: # - username and password for proxy authentication @@ -140,8 +138,7 @@ class RevocationChecker: return _translate_ocsp_query(cert_path, output, err) -def _determine_ocsp_server(cert_path): - # type: (str) -> Tuple[Optional[str], Optional[str]] +def _determine_ocsp_server(cert_path: str) -> Tuple[Optional[str], Optional[str]]: """Extract the OCSP server host from a certificate. :param str cert_path: Path to the cert we're checking OCSP for @@ -171,8 +168,7 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): - # type: (str, str, str, int) -> bool +def _check_ocsp_cryptography(cert_path: str, chain_path: str, url: str, timeout: int) -> bool: # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 5e1c29669..06acefc69 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -9,7 +9,7 @@ from josepy import util as jose_util import pkg_resources import zope.interface -from certbot import achallenges # pylint: disable=unused-import +from certbot import achallenges from certbot import crypto_util from certbot import errors from certbot import interfaces @@ -313,8 +313,8 @@ class ChallengePerformer: def __init__(self, configurator): self.configurator = configurator - self.achalls = [] # type: List[achallenges.KeyAuthorizationAnnotatedChallenge] - self.indices = [] # type: List[int] + self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] + self.indices: List[int] = [] def add_chall(self, achall, idx=None): """Store challenge to be performed when perform() is called. diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index 5fec5e0d6..10efa5f85 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -116,8 +116,9 @@ class LexiconClient: return None -def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): - # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] +def build_lexicon_config(lexicon_provider_name: str, + lexicon_options: Dict, provider_options: Dict + ) -> Union[ConfigResolver, Dict]: """ Convenient function to build a Lexicon 2.x/3.x config object. :param str lexicon_provider_name: the name of the lexicon provider to use @@ -126,7 +127,7 @@ def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_option :return: configuration to apply to the provider :rtype: ConfigurationResolver or dict """ - config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config: Dict[str, Any] = {'provider_name': lexicon_provider_name} config.update(lexicon_options) if not ConfigResolver: # Lexicon 2.x diff --git a/certbot/certbot/plugins/enhancements.py b/certbot/certbot/plugins/enhancements.py index 3c3db7e71..c1de4d44f 100644 --- a/certbot/certbot/plugins/enhancements.py +++ b/certbot/certbot/plugins/enhancements.py @@ -156,7 +156,7 @@ class AutoHSTSEnhancement(object, metaclass=abc.ABCMeta): # This is used to configure internal new style enhancements in Certbot. These # enhancement interfaces need to be defined in this file. Please do not modify # this list from plugin code. -_INDEX = [ +_INDEX: List[Dict[str, Any]] = [ { "name": "AutoHSTS", "cli_help": "Gradually increasing max-age value for HTTP Strict Transport "+ @@ -171,4 +171,4 @@ _INDEX = [ "deployer_function": "deploy_autohsts", "enable_function": "enable_autohsts" } -] # type: List[Dict[str, Any]] +] diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index b5dd6a502..0d32d6850 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -42,7 +42,7 @@ class PluginStorage: :raises .errors.PluginStorageError: when unable to open or read the file """ - data = {} # type: Dict[str, Any] + data: Dict[str, Any] = {} filedata = "" try: with open(self._storagepath, 'r') as fh: diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index 0643d5ba7..5f4a08dc7 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -58,7 +58,7 @@ _INITIAL_PID = os.getpid() # the dict are attempted to be cleaned up at program exit. If the # program exits before the lock is cleaned up, it is automatically # released, but the file isn't deleted. -_LOCKS = {} # type: Dict[str, lock.LockFile] +_LOCKS: Dict[str, lock.LockFile] = {} def env_no_snap_for_external_calls(): @@ -216,10 +216,10 @@ def safe_open(path, mode="w", chmod=None): if ``None``. """ - open_args = () # type: Union[Tuple[()], Tuple[int]] + open_args: Union[Tuple[()], Tuple[int]] = () if chmod is not None: open_args = (chmod,) - fdopen_args = () # type: Union[Tuple[()], Tuple[int]] + fdopen_args: Union[Tuple[()], Tuple[int]] = () fd = filesystem.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, *open_args) return os.fdopen(fd, mode, *fdopen_args) @@ -577,7 +577,7 @@ def is_wildcard_domain(domain): :rtype: bool """ - wildcard_marker = b"*." # type: Union[Text, bytes] + wildcard_marker: Union[Text, bytes] = b"*." if isinstance(domain, str): wildcard_marker = u"*." return domain.startswith(wildcard_marker) diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 8ae6290bf..4b08fe3a1 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -30,7 +30,7 @@ class CompleterTest(test_util.TempDirTestCase): if self.tempdir[-1] != os.sep: self.tempdir += os.sep - self.paths = [] # type: List[str] + self.paths: List[str] = [] # create some files and directories in temp_dir for c in string.ascii_lowercase: path = os.path.join(self.tempdir, c) diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 9ccd63a3a..265a42023 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -27,7 +27,7 @@ def set_signals(sig_handler_dict): def signal_receiver(signums): """Context manager to catch signals""" signals = [] - prev_handlers = get_signals(signums) # type: Dict[int, Union[int, None, Callable]] + prev_handlers: Dict[int, Union[int, None, Callable]] = get_signals(signums) set_signals({s: lambda s, _: signals.append(s) for s in signums}) yield signals set_signals(prev_handlers) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 06a641216..b0522bd52 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -267,7 +267,7 @@ class RunSavedPostHooksTest(HookTest): def setUp(self): super(RunSavedPostHooksTest, self).setUp() - self.eventually = [] # type: List[str] + self.eventually: List[str] = [] def test_empty(self): self.assertFalse(self._call_with_mock_execute_and_eventually().called) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 088faa0fb..33d89c3ea 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -43,7 +43,7 @@ class PreArgParseSetupTest(unittest.TestCase): mock_root_logger.setLevel.assert_called_once_with(logging.DEBUG) self.assertEqual(mock_root_logger.addHandler.call_count, 2) - memory_handler = None # type: Optional[logging.handlers.MemoryHandler] + memory_handler: Optional[logging.handlers.MemoryHandler] = None for call in mock_root_logger.addHandler.call_args_list: handler = call[0][0] if memory_handler is None and isinstance(handler, logging.handlers.MemoryHandler): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7668a1b6a..d4afed858 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -855,7 +855,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -870,7 +870,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_no_args_unprivileged(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() def throw_error(directory, mode, strict): @@ -892,7 +892,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_init(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() @@ -910,7 +910,7 @@ class MainTest(test_util.ConfigTestCase): @mock.patch('certbot._internal.main.plugins_disco') @mock.patch('certbot._internal.main.cli.HelpfulArgumentParser.determine_help_topics') def test_plugins_prepare(self, _det, mock_disco): - ifaces = [] # type: List[interfaces.IPlugin] + ifaces: List[interfaces.IPlugin] = [] plugins = mock_disco.PluginsRegistry.find_all() stdout = io.StringIO() diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index a97de24b9..64829c8cc 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -277,7 +277,7 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.prepare.assert_called_once_with() def test_prepare_order(self): - order = [] # type: List[str] + order: List[str] = [] plugins = dict( (c, mock.MagicMock(prepare=functools.partial(order.append, c))) for c in string.ascii_letters) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index a5de99e60..027ebdc35 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -52,7 +52,7 @@ class PickPluginTest(unittest.TestCase): self.default = None self.reg = mock.MagicMock() self.question = "Question?" - self.ifaces = [] # type: List[interfaces.IPlugin] + self.ifaces: List[interfaces.IPlugin] = [] def _call(self): from certbot._internal.plugins.selection import pick_plugin diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 596cee622..80347d35a 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -24,9 +24,8 @@ class ServerManagerTest(unittest.TestCase): def setUp(self): from certbot._internal.plugins.standalone import ServerManager - self.certs = {} # type: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] - self.http_01_resources = {} \ - # type: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] + self.certs: Dict[bytes, Tuple[OpenSSL.crypto.PKey, OpenSSL.crypto.X509]] = {} + self.http_01_resources: Set[acme_standalone.HTTP01RequestHandler.HTTP01Resource] = {} self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): From 0962b0fc8346fd948b5a6847fb4f6f0dc90da8a6 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 11 Mar 2021 22:08:20 +0100 Subject: [PATCH 26/99] Kill current snapcraft build when a "Chroot problem" is encountered (#8442) * Kill snapcraft build when a "Chroot problem" is encountered * Display specific helper for "Chroot problem" status and cancel retry mechanism in this case. * Isolate build tmp directories * Configure XDG_CACHE_HOME * Kill snapcraftctl with chroot problem is encountered --- tools/snap/build_remote.py | 66 +++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index e690d2916..69c704e45 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -5,6 +5,8 @@ import glob from multiprocessing import Manager from multiprocessing import Pool from multiprocessing import Process +from multiprocessing.managers import SyncManager +import os from os.path import basename from os.path import dirname from os.path import exists @@ -13,27 +15,46 @@ from os.path import realpath import re import subprocess import sys +import tempfile +from threading import Lock import time +from typing import Dict +from typing import List +from typing import Set +from typing import Tuple CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] -def _execute_build(target, archs, status, workspace): - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs) - ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) +def _execute_build( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> Tuple[int, List[str]]: - process_output = [] + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=workspace) + + process_output: List[str] = [] for line in process.stdout: process_output.append(line) _extract_state(target, line, status) + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + return process.wait(), process_output -def _build_snap(target, archs, status, running, lock): +def _build_snap( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool], lock: Lock) -> Dict[str, str]: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -51,7 +72,14 @@ def _build_snap(target, archs, status, running, lock): with lock: dump_output = exit_code != 0 - failed_archs = [arch for arch in archs if status[target][arch] == 'Failed to build'] + failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] + if any(arch for arch in archs if status[target][arch] == 'Chroot problem'): + print('Some builds failed with the status "Chroot problem".') + print('This status is known to make any future build fail until either ' + 'the source code changes or the build on Launchpad is deleted.') + print('Please fix the build appropriately before trying a new one.') + # It is useless to retry in this situation. + retry = 0 if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -80,7 +108,7 @@ def _build_snap(target, archs, status, running, lock): return {target: workspace} -def _extract_state(project, output, status): +def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) @@ -95,7 +123,7 @@ def _extract_state(project, output, status): status[project] = state -def _dump_status_helper(archs, status): +def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: headers = ['project', *archs] print(''.join(f'| {item:<25}' for item in headers)) print(f'|{"-" * 26}' * len(headers)) @@ -107,14 +135,18 @@ def _dump_status_helper(archs, status): sys.stdout.flush() -def _dump_status(archs, status, running): +def _dump_status( + archs: Set[str], status: Dict[str, Dict[str, str]], + running: Dict[str, bool]) -> None: while any(running.values()): print(f'Remote build status at {datetime.datetime.now()}') _dump_status_helper(archs, status) time.sleep(10) -def _dump_results(targets, archs, status, workspaces): +def _dump_results( + targets: Set[str], archs: Set[str], status: Dict[str, Dict[str, str]], + workspaces: Dict[str, str]) -> bool: failures = False for target in targets: for arch in archs: @@ -152,8 +184,8 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('targets', nargs='+', choices=['ALL', 'DNS_PLUGINS', 'certbot', *PLUGINS], help='the list of snaps to build') - parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], - help='the architectures for which snaps are built') + parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], + default=['amd64'], help='the architectures for which snaps are built') parser.add_argument('--timeout', type=int, default=None, help='build process will fail after the provided timeout (in seconds)') args = parser.parse_args() @@ -180,8 +212,10 @@ def main(): print(f' - projects: {", ".join(sorted(targets))}') print() - with Manager() as manager, Pool(processes=len(targets)) as pool: - status = manager.dict() + manager: SyncManager = Manager() + pool = Pool(processes=len(targets)) + with manager, pool: + status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) lock = manager.Lock() From bc892e04c4e05b1b57f59179b7b465f7f981a176 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 11 Mar 2021 22:17:41 +0100 Subject: [PATCH 27/99] Fixing imports in cli module (#8708) While working on #8640, I realized that there were some hidden circular dependencies in certbot._internal.cli package. Then cerbot could break if the order of these imports changes. This PR fixes that and apply isort on top of the result. --- certbot/certbot/_internal/cli/__init__.py | 86 ++++++++----------- certbot/certbot/_internal/cli/group_adder.py | 2 +- certbot/certbot/_internal/cli/helpful.py | 33 +++---- certbot/certbot/_internal/cli/paths_parser.py | 6 +- .../certbot/_internal/cli/plugins_parsing.py | 2 +- certbot/certbot/_internal/cli/subparsers.py | 13 ++- certbot/certbot/_internal/cli/verb_help.py | 6 +- 7 files changed, 61 insertions(+), 87 deletions(-) diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index b64ab0e9c..d48fd419a 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -1,65 +1,51 @@ """Certbot command line argument & config processing.""" # pylint: disable=too-many-lines +import argparse import logging import logging.handlers -import argparse import sys from typing import Optional -import certbot._internal.plugins.selection as plugin_selection -from certbot._internal.plugins import disco as plugins_disco - -# pylint: disable=ungrouped-imports import certbot from certbot._internal import constants - -import certbot.plugins.enhancements as enhancements - - -from certbot._internal.cli.cli_constants import ( - LEAUTO, - old_path_fragment, - new_path_prefix, - cli_command, - SHORT_USAGE, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - ARGPARSE_PARAMS_TO_REMOVE, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - VAR_MODIFIERS, - DEPRECATED_OPTIONS -) - -from certbot._internal.cli.cli_utils import ( - _Default, - read_file, - flag_default, - config_help, - HelpfulArgumentGroup, - CustomHelpFormatter, - _DomainsAction, - add_domains, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction, - parse_preferred_challenges, - _PrefChallAction, - _DeployHookAction, - _RenewHookAction, - nonnegative_int -) - -# These imports depend on cli_constants and cli_utils. -from certbot._internal.cli.verb_help import VERB_HELP, VERB_HELP_MAP +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import cli_command +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import LEAUTO +from certbot._internal.cli.cli_constants import new_path_prefix +from certbot._internal.cli.cli_constants import old_path_fragment +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import VAR_MODIFIERS +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import _DeployHookAction +from certbot._internal.cli.cli_utils import _DomainsAction +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _PrefChallAction +from certbot._internal.cli.cli_utils import _RenewHookAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.cli_utils import nonnegative_int +from certbot._internal.cli.cli_utils import parse_preferred_challenges +from certbot._internal.cli.cli_utils import read_file from certbot._internal.cli.group_adder import _add_all_groups -from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.helpful import HelpfulArgumentParser from certbot._internal.cli.paths_parser import _paths_parser from certbot._internal.cli.plugins_parsing import _plugins_parsing - -# These imports depend on some or all of the submodules for cli. -from certbot._internal.cli.helpful import HelpfulArgumentParser -# pylint: enable=ungrouped-imports +from certbot._internal.cli.subparsers import _create_subparsers +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot._internal.plugins import disco as plugins_disco +import certbot._internal.plugins.selection as plugin_selection +import certbot.plugins.enhancements as enhancements logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/cli/group_adder.py b/certbot/certbot/_internal/cli/group_adder.py index f22fbc496..0c54c9fe1 100644 --- a/certbot/certbot/_internal/cli/group_adder.py +++ b/certbot/certbot/_internal/cli/group_adder.py @@ -1,6 +1,6 @@ """This module contains a function to add the groups of arguments for the help display""" -from certbot._internal.cli import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP def _add_all_groups(helpful): diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index d36147089..f185bdc26 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -11,35 +11,30 @@ from typing import Dict import configargparse import zope.component import zope.interface - from zope.interface import interfaces as zope_interfaces from certbot import crypto_util from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot._internal import constants from certbot._internal import hooks - +from certbot._internal.cli.cli_constants import ARGPARSE_PARAMS_TO_REMOVE +from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW +from certbot._internal.cli.cli_constants import EXIT_ACTIONS +from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS +from certbot._internal.cli.cli_utils import _Default +from certbot._internal.cli.cli_utils import add_domains +from certbot._internal.cli.cli_utils import CustomHelpFormatter +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import HelpfulArgumentGroup +from certbot._internal.cli.verb_help import VERB_HELP +from certbot._internal.cli.verb_help import VERB_HELP_MAP +from certbot.compat import os from certbot.display import util as display_util -from certbot._internal.cli import ( - SHORT_USAGE, - CustomHelpFormatter, - flag_default, - VERB_HELP, - VERB_HELP_MAP, - COMMAND_OVERVIEW, - HELP_AND_VERSION_USAGE, - _Default, - add_domains, - EXIT_ACTIONS, - ZERO_ARG_ACTIONS, - ARGPARSE_PARAMS_TO_REMOVE, - HelpfulArgumentGroup -) - class HelpfulArgumentParser: """Argparse Wrapper. diff --git a/certbot/certbot/_internal/cli/paths_parser.py b/certbot/certbot/_internal/cli/paths_parser.py index 43ee41c88..6197a4bf9 100644 --- a/certbot/certbot/_internal/cli/paths_parser.py +++ b/certbot/certbot/_internal/cli/paths_parser.py @@ -1,10 +1,8 @@ """This is a module that adds configuration to the argument parser regarding paths for certificates""" +from certbot._internal.cli.cli_utils import config_help +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - flag_default, - config_help -) def _paths_parser(helpful): diff --git a/certbot/certbot/_internal/cli/plugins_parsing.py b/certbot/certbot/_internal/cli/plugins_parsing.py index 9e11ad3ab..bbfdf22da 100644 --- a/certbot/certbot/_internal/cli/plugins_parsing.py +++ b/certbot/certbot/_internal/cli/plugins_parsing.py @@ -1,5 +1,5 @@ """This is a module that handles parsing of plugins for the argument parser""" -from certbot._internal.cli import flag_default +from certbot._internal.cli.cli_utils import flag_default def _plugins_parsing(helpful, plugins): diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 13f8705ce..822381d21 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -1,14 +1,11 @@ """This module creates subparsers for the argument parser""" from certbot import interfaces from certbot._internal import constants - -from certbot._internal.cli import ( - flag_default, - read_file, - CaseInsensitiveList, - _user_agent_comment_type, - _EncodeReasonAction -) +from certbot._internal.cli.cli_utils import _EncodeReasonAction +from certbot._internal.cli.cli_utils import _user_agent_comment_type +from certbot._internal.cli.cli_utils import CaseInsensitiveList +from certbot._internal.cli.cli_utils import flag_default +from certbot._internal.cli.cli_utils import read_file def _create_subparsers(helpful): diff --git a/certbot/certbot/_internal/cli/verb_help.py b/certbot/certbot/_internal/cli/verb_help.py index 131cfec96..dfb4e9c67 100644 --- a/certbot/certbot/_internal/cli/verb_help.py +++ b/certbot/certbot/_internal/cli/verb_help.py @@ -1,9 +1,7 @@ """This module contain help information for verbs supported by certbot""" +from certbot._internal.cli.cli_constants import SHORT_USAGE +from certbot._internal.cli.cli_utils import flag_default from certbot.compat import os -from certbot._internal.cli import ( - SHORT_USAGE, - flag_default -) # The attributes here are: # short: a string that will be displayed by "certbot -h commands" From 2324c1bb7ad977bc3525ebef1a3f61410e0e245f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 Mar 2021 14:10:44 -0700 Subject: [PATCH 28/99] Update installing from source instructions (#8713) --- certbot/docs/install.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 8ae1c82f2..ac6c798eb 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -270,15 +270,10 @@ Disable and remove the swapfile once the virtual environment is constructed:: user@webserver:~$ sudo swapoff /tmp/swapfile user@webserver:~$ sudo rm /tmp/swapfile -Installing from source ----------------------- +Pip +--- -Installation from source is only supported for developers and the -whole process is described in the :doc:`contributing`. - -.. warning:: Please do **not** use ``python certbot/setup.py install``, ``python pip - install certbot``, or ``easy_install certbot``. Please do **not** attempt the - installation commands as superuser/root and/or without virtual environment, - e.g. ``sudo python certbot/setup.py install``, ``sudo pip install``, ``sudo - ./venv/bin/...``. These modes of operation might corrupt your operating - system and are **not supported** by the Certbot team! +Installing Certbot through pip is only supported on a best effort basis and +when using a virtual environment. Instructions for installing Certbot through +pip can be found at https://certbot.eff.org/instructions by selecting your +server software and then choosing "pip" in the "System" dropdown menu. From 1b39d3dc478ee8df1e39ec6c8ed1e44edcca7409 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 Mar 2021 17:53:43 -0700 Subject: [PATCH 29/99] switch to wait_until_running (#8715) --- tests/letstest/multitester.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index f079f893f..2e4399000 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -190,18 +190,16 @@ def block_until_ssh_open(ipstring, wait_time=10, timeout=120): t_elapsed += wait_time sock.close() -def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20): +def block_until_instance_ready(booting_instance, extra_wait_time=20): "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections" - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - while state != 'running' or ip is None: - time.sleep(wait_time) - # The instance needs to be reloaded to update its local attributes. See - # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. - booting_instance.reload() - state = booting_instance.state['Name'] - ip = booting_instance.public_ip_address - block_until_ssh_open(ip) + booting_instance.wait_until_running() + # The instance needs to be reloaded to update its local attributes. See + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Instance.reload. + booting_instance.reload() + # After waiting for the instance to be running and reloading the instance + # state, we should have an IP address. + assert booting_instance.public_ip_address is not None + block_until_ssh_open(booting_instance.public_ip_address) time.sleep(extra_wait_time) return booting_instance From 40ae5d939ee0449ce54b8bf12e75e9311e361d91 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Fri, 19 Mar 2021 14:27:29 -0700 Subject: [PATCH 30/99] Fix linux-py39-cover test (#8720) * update setuptools * upgrade Markupsafe --- tools/dev_constraints.txt | 2 +- tools/pipstrap_constraints.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index f4059a3f9..4f5eda34e 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -56,7 +56,7 @@ jsonschema==2.6.0 lazy-object-proxy==1.4.3 logger==1.4 logilab-common==1.4.1 -MarkupSafe==1.0 +MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_constraints.txt index 5de9e147d..54ab8b429 100644 --- a/tools/pipstrap_constraints.txt +++ b/tools/pipstrap_constraints.txt @@ -10,9 +10,9 @@ pip==20.2.4 \ --hash=sha256:51f1c7514530bd5c145d8f13ed936ad6b8bfcb8cf74e10403d0890bc986f0033 \ --hash=sha256:85c99a857ea0fb0aedf23833d9be5c40cf253fe24443f0829c7b472e23c364a1 -setuptools==44.1.1 \ - --hash=sha256:27a714c09253134e60a6fa68130f78c7037e5562c4f21f8f318f2ae900d152d5 \ - --hash=sha256:c67aa55db532a0dadc4d2e20ba9961cbd3ccc84d544e9029699822542b5a476b +setuptools==54.1.2 \ + --hash=sha256:dd20743f36b93cbb8724f4d2ccd970dce8b6e6e823a13aa7e5751bb4e674c20b \ + --hash=sha256:ebd0148faf627b569c8d2a1b20f5d3b09c873f12739d71c7ee88f037d5be82ff wheel==0.35.1 \ --hash=sha256:497add53525d16c173c2c1c733b8f655510e909ea78cc0e29d374243544b77a2 \ --hash=sha256:99a22d87add3f634ff917310a3d87e499f19e663413a52eb9232c447aa646c9f From 6bc8b3d2ba9a0f22a6682a46ae4905c462e19388 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sun, 21 Mar 2021 22:39:54 +0100 Subject: [PATCH 31/99] Precise the certificate naming convention mechanism in the compatibility document (#8652) * Precise the certificate naming convention mechanism in a note. * Add certificate name convention in user guide, refer to it in compatibility page. * Update certbot/docs/compatibility.rst Co-authored-by: alexzorin * Update certbot/docs/using.rst Co-authored-by: alexzorin * Update certbot/docs/using.rst Co-authored-by: alexzorin * Improve the note about naming conventions Co-authored-by: alexzorin --- certbot/docs/compatibility.rst | 6 +++--- certbot/docs/using.rst | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/certbot/docs/compatibility.rst b/certbot/docs/compatibility.rst index a4f33c281..d94642ec6 100644 --- a/certbot/docs/compatibility.rst +++ b/certbot/docs/compatibility.rst @@ -21,9 +21,9 @@ may change at any time. The second is that Certbot's behavior should only be considered stable with certain files but not all. Files with which users should expect Certbot to maintain its current behavior with are: -* ``/etc/letsencrypt/live//{cert,chain,fullchain,privkey}.pem`` where - ```` is the name given to ``--cert-name``. If ``--cert-name`` is not - set by the user, it is the first domain given to ``--domains``. +* ``/etc/letsencrypt/live/$domain/{cert,chain,fullchain,privkey}.pem``, where + ``$domain`` is the certificate name (see :ref:`where-certs` + for more details) * :ref:`CLI configuration files ` * Hook directories in ``/etc/letsencrypt/renewal-hooks`` diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 1d97caecc..87c4b9569 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -717,12 +717,24 @@ Where are my certificates? ========================== All generated keys and issued certificates can be found in -``/etc/letsencrypt/live/$domain``. In the case of creating a SAN certificate -with multiple alternative names, ``$domain`` is the first domain passed in -via -d parameter. Rather than copying, please point -your (web) server configuration directly to those files (or create -symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated -with the latest necessary files. +``/etc/letsencrypt/live/$domain``, where ``$domain`` is the certificate +name (see the note below). Rather than copying, please point your (web) +server configuration directly to those files (or create symlinks). +During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest +necessary files. + +.. note:: + The certificate name ``$domain`` used in the path ``/etc/letsencrypt/live/$domain`` + follows this convention: + + * it is the name given to ``--cert-name``, + * if ``--cert-name`` is not set by the user it is the first domain given to + ``--domains``, + * if the first domain is a wildcard domain (eg. ``*.example.com``) the + certificate name will be ``example.com``, + * if a name collision would occur with a certificate already named ``example.com``, + the new certificate name will be constructed using a numerical sequence + as ``example.com-001``. For historical reasons, the containing directories are created with permissions of ``0700`` meaning that certificates are accessible only From ae2247163e42934c0486c36079bc7ae50bc4d4e2 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Sun, 21 Mar 2021 22:42:23 +0100 Subject: [PATCH 32/99] Remove empty lines from `certbot certificates` when (#8723) .. envoked with `--cert-name` or `-d`. --- certbot/CHANGELOG.md | 3 ++- certbot/certbot/_internal/cert_manager.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 62bceb74d..9b37e9a1a 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -17,7 +17,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Fixed -* +* Don't output an empty line for a hidden certificate when `certbot certificates` is being used + in combination with `--cert-name` or `-d`. More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index 8fab5735a..b9b4ad2d7 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -266,9 +266,9 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): checker = ocsp.RevocationChecker() if config.certname and cert.lineagename != config.certname and not skip_filter_checks: - return "" + return None if config.domains and not set(config.domains).issubset(cert.names()): - return "" + return None now = pytz.UTC.fromutc(datetime.datetime.utcnow()) reasons = [] @@ -358,7 +358,9 @@ def _report_human_readable(config, parsed_certs): """Format a results report for a parsed cert""" certinfo = [] for cert in parsed_certs: - certinfo.append(human_readable_cert_info(config, cert)) + cert_info = human_readable_cert_info(config, cert) + if cert_info is not None: + certinfo.append(cert_info) return "\n".join(certinfo) From 84178e27737205fe547983b8214894dabba48026 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 22 Mar 2021 18:39:09 +0100 Subject: [PATCH 33/99] Do not reuse existing builds on Launchpad when executing snapcraft remote-build (#8719) We observed recently several unexpected behavior during the execution of snap jobs in Azure. In particular it seems that `snapcraft remote-build` is tending to reattach to the latest builds on Launchpad triggered by the nightly builds on master, independently from the actual branch, status of the code, or targeted architectures. Primarily if the builds on Launchpad are stalled for some reason, it blocks effectively any other Azure snap jobs until someone manually cancel the builds on Launchpad. Secondarily it means that the outcome of the builds may be inconsistent, because they can be the result of a build for the master source even if you are on a PR that modifieds these sources (including `snapcraft.yaml`). After digging in `snapcraft` source code, I realized that the signature computed to understand if a build should be resumed, is not based one some hashes against the snapcraft working directory content, but is simply a hash of the working directory absolute path *itself*. It means that every builds triggered from the working directory `/my/path/certbot` for instance, are recognized as the same unique build on Launchpad side, and may be resumed if they already exist, and so independently from the source code, `snapcraft.yaml` or targeted archs. For the record, relevant parts in `snapcraft` source code: https://github.com/snapcore/snapcraft/blob/82024d3748f96fc63077de8188c5752b58d151be/snapcraft/project/_project.py#L44 https://github.com/snapcore/snapcraft/blob/82024d3748f96fc63077de8188c5752b58d151be/snapcraft/project/_project.py#L86-L89 https://github.com/snapcore/snapcraft/blob/82024d3748f96fc63077de8188c5752b58d151be/snapcraft/cli/remote.py#L128-L132 This PR makes effectively the resume build mechanism effectively a noop by moving the source code first in a temporary directory with random name before running `snapcraft remote-build`. This way the signature is never the same and builds are always recognized as brand new builds. * Invalidate snapcraft remote-build cache by using a temporary workspace. * Capture one more state in the build --- tools/snap/build_remote.py | 81 +++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index 69c704e45..b12799dc3 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -13,6 +13,7 @@ from os.path import exists from os.path import join from os.path import realpath import re +import shutil import subprocess import sys import tempfile @@ -31,25 +32,54 @@ def _execute_build( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], workspace: str) -> Tuple[int, List[str]]: - with tempfile.TemporaryDirectory() as tempdir: - environ = os.environ.copy() - environ['XDG_CACHE_HOME'] = tempdir - process = subprocess.Popen([ - 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', - '--build-on', ','.join(archs)], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, env=environ, cwd=workspace) + temp_workspace = None + try: + # Snapcraft remote-build has a recover feature, that will make it reconnect to an existing + # build on Launchpad if possible. However, the signature used to retrieve a potential + # build is not based on the content of the sources used to build a snap, but on a hash + # of the snapcraft current working directory (the path itself, not the content). + # It means that every build started from /my/path/to/certbot will always be considered + # as the same build, whatever the actual sources are. + # To circumvent this, we create a temporary folder and use it as a workspace to build + # the snap: this path is random, making the recover feature effectively noop. + temp_workspace = tempfile.mkdtemp() + ignore = None + if target == 'certbot': + ignore = shutil.ignore_patterns(".git", "venv*", ".tox") + shutil.copytree(workspace, temp_workspace, + dirs_exist_ok=True, symlinks=True, ignore=ignore) # type:ignore - process_output: List[str] = [] - for line in process.stdout: - process_output.append(line) - _extract_state(target, line, status) + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', + '--build-on', ','.join(archs)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=temp_workspace) - if any(state for state in status[target].values() if state == 'Chroot problem'): - # On this error the snapcraft process stales. Let's finish it. - process.kill() + process_output: List[str] = [] + for line in process.stdout: + process_output.append(line) + _extract_state(target, line, status) - return process.wait(), process_output + if any(state for state in status[target].values() if state == 'Chroot problem'): + # On this error the snapcraft process stales. Let's finish it. + process.kill() + + process_state = process.wait() + + for path in glob.glob(join(temp_workspace, '*.snap')): + shutil.copy(path, workspace) + + return process_state, process_output + except BaseException as e: + print(e) + sys.stdout.flush() + raise e + finally: + if temp_workspace: + shutil.rmtree(temp_workspace, ignore_errors=True) def _build_snap( @@ -109,18 +139,23 @@ def _build_snap( def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: + state = status[project] + + if "Sending build data to Launchpad..." in output: + for arch in state.keys(): + state[arch] = "Sending build data" + match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) if match: arch = match.group(1) - state = status[project] state[arch] = match.group(2) - # You need to reassign the value of status[project] here (rather than doing - # something like status[project][arch] = match.group(2)) for the state change - # to propagate to other processes. See - # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for - # more info. - status[project] = state + # You need to reassign the value of status[project] here (rather than doing + # something like status[project][arch] = match.group(2)) for the state change + # to propagate to other processes. See + # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for + # more info. + status[project] = state def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: From 9fdb24331c463d08666aecfbb694418945c86bc9 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 23 Mar 2021 09:05:37 +1100 Subject: [PATCH 34/99] docs: rewrite "Automated Renewals" in User Guide (#8717) --- certbot/docs/using.rst | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 87c4b9569..cd52c7fe5 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -515,11 +515,8 @@ Renewing certificates days). Make sure you renew the certificates at least once in 3 months. -.. seealso:: Many of the certbot clients obtained through a - distribution come with automatic renewal out of the box, - such as Debian and Ubuntu versions installed through `apt`, - CentOS/RHEL 7 through EPEL, etc. See `Automated Renewals`_ - for more details. +.. seealso:: Most Certbot installations come with automatic + renewal out of the box. See `Automated Renewals`_ for more details. As of version 0.10.0, Certbot supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew @@ -689,27 +686,15 @@ The following commands could be used to specify where these files are located:: Automated Renewals ------------------ -Many Linux distributions provide automated renewal when you use the -packages installed through their system package manager. The -following table is an *incomplete* list of distributions which do so, -as well as their methods for doing so. +Most Certbot installations come with automatic renewals preconfigured. This +is done by means of a scheduled task which runs ``certbot renew`` periodically. -If you are not sure whether or not your system has this already -automated, refer to your distribution's documentation, or check your -system's crontab (typically in `/etc/crontab/` and `/etc/cron.*/*` and -systemd timers (`systemctl list-timers`). +If you are unsure whether you need to configure automated renewal: -.. csv-table:: Distributions with Automated Renewal - :header: "Distribution Name", "Distribution Version", "Automation Method" - - "CentOS", "EPEL 7", "systemd" - "Debian", "stretch", "cron, systemd" - "Debian", "testing/sid", "cron, systemd" - "Fedora", "26", "systemd" - "Fedora", "27", "systemd" - "RHEL", "EPEL 7", "systemd" - "Ubuntu", "17.10", "cron, systemd" - "Ubuntu", "certbot PPA", "cron, systemd" +1. Review the instructions for your system at https://certbot.eff.org/instructions. + They will describe how to set up a scheduled task, if necessary. +2. (Linux/BSD): Check your system's crontab (typically `/etc/crontab` and + `/etc/cron.*/*`) and systemd timers (``systemctl list-timers``). .. _where-certs: From 54b0b98988ec3dbc28cefe69bae627eecb347235 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Mar 2021 11:29:01 -0700 Subject: [PATCH 35/99] use worker terminology (#8728) This will be needed for me to update `pytest-xdist` as part of https://github.com/certbot/certbot/issues/8705 since `pytest-xdist` removed the "slave" terminology. See https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst#deprecations-and-removals. --- .../certbot_integration_tests/certbot_tests/context.py | 6 +++--- certbot-ci/certbot_integration_tests/conftest.py | 6 +++--- .../certbot_integration_tests/rfc2136_tests/context.py | 4 ++-- certbot-ci/setup.py | 4 +++- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index b052e375d..9af1bc15c 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -12,9 +12,9 @@ class IntegrationTestsContext: def __init__(self, request): self.request = request - if hasattr(request.config, 'slaveinput'): # Worker node - self.worker_id = request.config.slaveinput['slaveid'] - acme_xdist = request.config.slaveinput['acme_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self.worker_id = request.config.workerinput['workerid'] + acme_xdist = request.config.workerinput['acme_xdist'] else: # Primary node self.worker_id = 'primary' acme_xdist = request.config.acme_xdist diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 6fc77480d..5e6ed5562 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -34,7 +34,7 @@ def pytest_configure(config): Standard pytest hook used to add a configuration logic for each node of a pytest run. :param config: the current pytest configuration """ - if not hasattr(config, 'slaveinput'): # If true, this is the primary node + if not hasattr(config, 'workerinput'): # If true, this is the primary node with _print_on_err(): _setup_primary_node(config) @@ -44,8 +44,8 @@ def pytest_configure_node(node): Standard pytest-xdist hook used to configure a worker node. :param node: current worker node """ - node.slaveinput['acme_xdist'] = node.config.acme_xdist - node.slaveinput['dns_xdist'] = node.config.dns_xdist + node.workerinput['acme_xdist'] = node.config.acme_xdist + node.workerinput['dns_xdist'] = node.config.dns_xdist @contextlib.contextmanager diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index c8024b21d..8df89f473 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -18,8 +18,8 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): self.request = request self._dns_xdist = None - if hasattr(request.config, 'slaveinput'): # Worker node - self._dns_xdist = request.config.slaveinput['dns_xdist'] + if hasattr(request.config, 'workerinput'): # Worker node + self._dns_xdist = request.config.workerinput['dns_xdist'] else: # Primary node self._dns_xdist = request.config.dns_xdist diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 9f9c1f462..7f89cc934 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -14,7 +14,9 @@ install_requires = [ 'pyopenssl', 'pytest', 'pytest-cov', - 'pytest-xdist', + # This version is needed for "worker" attributes we currently use like + # "workerinput". See https://github.com/pytest-dev/pytest-xdist/pull/268. + 'pytest-xdist>=1.22.1', 'python-dateutil', 'pyyaml', 'requests', From 1d7ddb0c0c1b587d56c6cce1d45ae1edceed7254 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 23 Mar 2021 13:01:01 -0700 Subject: [PATCH 36/99] fix pylint (#8729) --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index a2468b0cf..e19077d8d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,7 +8,10 @@ jobs=0 # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +# CERTBOT COMMENT +# This is needed for pylint to import linter_plugin.py since +# https://github.com/PyCQA/pylint/pull/3396. +init-hook="import pylint.config, os, sys; sys.path.append(os.path.dirname(pylint.config.PYLINTRC))" # Profiled execution. profile=no From d3b74f41e0cbfd603d69f722c3cd99f973e05a2a Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Tue, 23 Mar 2021 21:33:47 +0100 Subject: [PATCH 37/99] Added missing from typing imports. (#8724) --- acme/tests/client_test.py | 1 + acme/tests/crypto_util_test.py | 1 + acme/tests/standalone_test.py | 1 + certbot-apache/tests/augeasnode_test.py | 2 ++ certbot-apache/tests/http_01_test.py | 1 + certbot-nginx/tests/parser_test.py | 1 + certbot/tests/display/completer_test.py | 2 ++ certbot/tests/error_handler_test.py | 1 + certbot/tests/log_test.py | 2 +- certbot/tests/main_test.py | 1 + certbot/tests/plugins/disco_test.py | 1 + certbot/tests/plugins/selection_test.py | 1 + certbot/tests/plugins/standalone_test.py | 1 + 13 files changed, 15 insertions(+), 1 deletion(-) diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 14247335c..6addb09f3 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -5,6 +5,7 @@ import datetime import http.client as http_client import json import unittest +from typing import Dict from unittest import mock import josepy as jose diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 8075b68ed..8c1f905d2 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -5,6 +5,7 @@ import socketserver import threading import time import unittest +from typing import List import josepy as jose import OpenSSL diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index a02d008a0..17d73dba8 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,6 +4,7 @@ import socket import socketserver import threading import unittest +from typing import Set from unittest import mock import josepy as jose diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 83224fc98..42126a5a9 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,4 +1,6 @@ """Tests for AugeasParserNode classes""" +from typing import List + try: import mock except ImportError: # pragma: no cover diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 06e19e445..9d66c3aa0 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,6 +1,7 @@ """Test for certbot_apache._internal.http_01.""" import unittest import errno +from typing import List try: import mock diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index 35173d094..b062c4196 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -3,6 +3,7 @@ import glob import re import shutil import unittest +from typing import List from certbot import errors from certbot.compat import os diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 4b08fe3a1..86f939491 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -1,4 +1,6 @@ """Test certbot._internal.display.completer.""" +from typing import List + try: import readline # pylint: disable=import-error except ImportError: diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 265a42023..521b6976b 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -3,6 +3,7 @@ import contextlib import signal import sys import unittest +from typing import Callable, Dict, Union try: import mock diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 33d89c3ea..7b1aacd5c 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,7 +5,7 @@ import logging.handlers import sys import time import unittest - +from typing import Optional from acme import messages from certbot import errors diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index d4afed858..e093edd20 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -11,6 +11,7 @@ import sys import tempfile import traceback import unittest +from typing import List import josepy as jose import pytz diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 64829c8cc..1d3fec722 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -2,6 +2,7 @@ import functools import string import unittest +from typing import List try: import mock diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index 027ebdc35..2dfd60a63 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -1,6 +1,7 @@ """Tests for letsencrypt.plugins.selection""" import sys import unittest +from typing import List try: import mock diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 80347d35a..636b251de 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -3,6 +3,7 @@ import socket from socket import errno as socket_errors # type: ignore import unittest +from typing import Dict, Set, Tuple import josepy as jose try: From f90e93134c94aeef0ffad19bd66752a865201d96 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Mar 2021 10:29:12 -0700 Subject: [PATCH 38/99] Upgrade cryptography to 3.4.6 (#8730) * Upgrade cryptography to 3.4.6 * Fix comment with instructions for how to use hashin * run tools/rebuild_certbot_constraints.py * add deps for building cryptography in snaps * Update cryptography build dependencies for docker * Update sources for test farm tests * Remove rust if it's installed for test farm tests * source bootstrap script and call sudo as needed --- snap/snapcraft.yaml | 10 +- .../letstest/scripts/bootstrap_os_packages.sh | 22 +++- tests/letstest/scripts/test_sdists.sh | 2 +- tools/certbot_constraints.txt | 122 +++++++++--------- tools/docker/core/Dockerfile | 2 + tools/rebuild_certbot_constraints.py | 5 +- tools/snap/generate_dnsplugins_snapcraft.sh | 9 +- 7 files changed, 98 insertions(+), 74 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d53fba88b..ccaafd93b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -70,7 +70,15 @@ parts: - python3-pkg-resources - python3.8-minimal # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, git, libaugeas-dev, python3-dev] + build-packages: + - gcc + - git + - libaugeas-dev + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo build-environment: - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/tests/letstest/scripts/bootstrap_os_packages.sh index 7ad93f63e..3f4c6e30e 100755 --- a/tests/letstest/scripts/bootstrap_os_packages.sh +++ b/tests/letstest/scripts/bootstrap_os_packages.sh @@ -34,9 +34,9 @@ DeterminePythonVersion() { } BootstrapDebCommon() { - apt-get update || error apt-get update hit problems but continuing anyway... + sudo apt-get update || error apt-get update hit problems but continuing anyway... - apt-get install -y --no-install-recommends \ + sudo apt-get install -y --no-install-recommends \ python3 \ python3-dev \ python3-venv \ @@ -46,8 +46,19 @@ BootstrapDebCommon() { openssl \ libffi-dev \ ca-certificates \ + build-essential \ + curl \ make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel + # make sure rust isn't installed by the package manager + if ! sudo apt-get remove -y rustc; then + error "Could not remove existing rust. Aborting bootstrap!" + exit 1 + fi + + # Install rust for cryptography (needed on Debian) + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . $HOME/.cargo/env } # Sets TOOL to the name of the package manager @@ -79,6 +90,7 @@ BootstrapRpmCommonBase() { libffi-devel redhat-rpm-config ca-certificates + cargo " # Add the python packages @@ -92,7 +104,7 @@ BootstrapRpmCommonBase() { " fi - if ! $TOOL install -y $pkgs; then + if ! sudo $TOOL install -y $pkgs; then error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi @@ -105,8 +117,8 @@ BootstrapRpmPython3() { python3-devel " - if ! $TOOL list 'python3*-devel' >/dev/null 2>&1; then - yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional + if ! sudo $TOOL list 'python3*-devel' >/dev/null 2>&1; then + sudo yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional fi BootstrapRpmCommonBase "$python_pkgs" diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index becdd6d9a..a8d0362fa 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -6,7 +6,7 @@ BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" VENV_PATH=venv # install OS packages -sudo $BOOTSTRAP_SCRIPT +. $BOOTSTRAP_SCRIPT # setup venv # We strip the hashes because the venv creation script includes unhashed diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt index 77bfef9db..2f48c8b3e 100644 --- a/tools/certbot_constraints.txt +++ b/tools/certbot_constraints.txt @@ -7,80 +7,78 @@ # If you want to update a single dependency, run commands similar to these: # ``` # pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 +# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 # ``` -ConfigArgParse==1.2.3 \ - --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc +ConfigArgParse==1.4 \ + --hash=sha256:abef9ff44fb0091f0e3bb2ee7e5b26a02b5b62d45408a5272a9bd461f5b59b4b certifi==2020.12.5 \ --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 -cffi==1.14.4 \ - --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \ - --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \ - --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \ - --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \ - --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \ - --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \ - --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \ - --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \ - --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \ - --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \ - --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \ - --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \ - --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \ - --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \ - --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \ - --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \ - --hash=sha256:7ef7d4ced6b325e92eb4d3502946c78c5367bc416398d387b39591532536734e \ - --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \ - --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \ - --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \ - --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \ - --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \ - --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \ - --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \ - --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \ - --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \ - --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \ - --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \ - --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \ - --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \ - --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \ - --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \ - --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \ - --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \ - --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \ - --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \ - --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f +cffi==1.14.5 \ + --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ + --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ + --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ + --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ + --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ + --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ + --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ + --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ + --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ + --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ + --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ + --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ + --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ + --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ + --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ + --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ + --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ + --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ + --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ + --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ + --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ + --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ + --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ + --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ + --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ + --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ + --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ + --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ + --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ + --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ + --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ + --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ + --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ + --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ + --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ + --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ + --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c chardet==4.0.0 \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==3.3.2 \ - --hash=sha256:0d7b69674b738068fa6ffade5c962ecd14969690585aaca0a1b1fc9058938a72 \ - --hash=sha256:1bd0ccb0a1ed775cd7e2144fe46df9dc03eefd722bbcf587b3e0616ea4a81eff \ - --hash=sha256:3c284fc1e504e88e51c428db9c9274f2da9f73fdf5d7e13a36b8ecb039af6e6c \ - --hash=sha256:49570438e60f19243e7e0d504527dd5fe9b4b967b5a1ff21cc12b57602dd85d3 \ - --hash=sha256:541dd758ad49b45920dda3b5b48c968f8b2533d8981bcdb43002798d8f7a89ed \ - --hash=sha256:5a60d3780149e13b7a6ff7ad6526b38846354d11a15e21068e57073e29e19bed \ - --hash=sha256:7951a966613c4211b6612b0352f5bf29989955ee592c4a885d8c7d0f830d0433 \ - --hash=sha256:922f9602d67c15ade470c11d616f2b2364950602e370c76f0c94c94ae672742e \ - --hash=sha256:a0f0b96c572fc9f25c3f4ddbf4688b9b38c69836713fb255f4a2715d93cbaf44 \ - --hash=sha256:a777c096a49d80f9d2979695b835b0f9c9edab73b59e4ceb51f19724dda887ed \ - --hash=sha256:a9a4ac9648d39ce71c2f63fe7dc6db144b9fa567ddfc48b9fde1b54483d26042 \ - --hash=sha256:aa4969f24d536ae2268c902b2c3d62ab464b5a66bcb247630d208a79a8098e9b \ - --hash=sha256:c7390f9b2119b2b43160abb34f63277a638504ef8df99f11cb52c1fda66a2e6f \ - --hash=sha256:e18e6ab84dfb0ab997faf8cca25a86ff15dfea4027b986322026cc99e0a892da +cryptography==3.4.6 \ + --hash=sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b \ + --hash=sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336 \ + --hash=sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87 \ + --hash=sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7 \ + --hash=sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799 \ + --hash=sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b \ + --hash=sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df \ + --hash=sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0 \ + --hash=sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3 \ + --hash=sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724 \ + --hash=sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2 \ + --hash=sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 distro==1.5.0 \ --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 -josepy==1.6.0 \ - --hash=sha256:0aab1c3ceffe045e7fd5bcfe7685e27e9d2758518d9ba7116b5de34087e70bf5 \ - --hash=sha256:65f077fc5902aca1e140ddb000e7abb081d5fb8421db60b6071076ef81c5bd27 +josepy==1.8.0 \ + --hash=sha256:6d632fcdaf0bed09e33f81f13b10575d4f0b7c37319350b725454e04a41e6a49 \ + --hash=sha256:a5a182eb499665d99e7ec54bb3fe389f9cbc483d429c9651f20384ba29564269 parsedatetime==2.6 \ --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b @@ -110,9 +108,9 @@ requests-toolbelt==0.9.1 \ six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.26.3 \ - --hash=sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80 \ - --hash=sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73 +urllib3==1.26.4 \ + --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ + --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 zope.component==4.6.2 \ --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index d2ebe3537..256db0169 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -37,6 +37,8 @@ RUN apk add --no-cache --virtual .build-deps \ openssl-dev \ musl-dev \ libffi-dev \ + python3-dev \ + cargo \ && python tools/pipstrap.py \ && python tools/pip_install.py --no-cache-dir \ --editable src/acme \ diff --git a/tools/rebuild_certbot_constraints.py b/tools/rebuild_certbot_constraints.py index f5e5d3ca7..14479d8ec 100755 --- a/tools/rebuild_certbot_constraints.py +++ b/tools/rebuild_certbot_constraints.py @@ -37,9 +37,6 @@ AUTHORITATIVE_CONSTRAINTS = { # Too touchy to move to a new version. And will be removed soon # in favor of pure python parser for Apache. 'python-augeas': '0.5.0', - # We avoid cryptography 3.4+ since it requires Rust to compile the wheels, and - # this needs some work on the snap builds. - 'cryptography': '3.3.2', } # ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) @@ -214,7 +211,7 @@ def _write_requirements(dest_file, requirements, conflicts): # If you want to update a single dependency, run commands similar to these: # ``` # pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 +# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 # ``` ''') diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index d93d8ec73..139c8b0d3 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -35,7 +35,14 @@ parts: - PIP_CONSTRAINT: \$SNAPCRAFT_PART_SRC/snap-constraints.txt - SNAP_BUILD: "True" # To build cryptography and cffi if needed - build-packages: [gcc, libffi-dev, libssl-dev, python3-dev] + build-packages: + - gcc + - git + - build-essential + - libssl-dev + - libffi-dev + - python3-dev + - cargo certbot-metadata: plugin: dump source: . From 0480959893e173ae45678d76a510dde8f92d0f96 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Mar 2021 11:50:34 -0700 Subject: [PATCH 39/99] use pip in test_sdists.sh (#8737) --- tests/letstest/scripts/test_sdists.sh | 33 +++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index a8d0362fa..ad729e0eb 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -9,31 +9,31 @@ VENV_PATH=venv . $BOOTSTRAP_SCRIPT # setup venv -# We strip the hashes because the venv creation script includes unhashed -# constraints in the commands given to pip and the mix of hashed and unhashed -# packages makes pip error out. -python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt -python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt +python3 -m venv $VENV_PATH +$VENV_PATH/bin/python3 tools/pipstrap.py +. "$VENV_PATH/bin/activate" +# pytest is needed to run tests on our packages so we install a pinned version here. +tools/pip_install.py pytest + +# setup constraints +TEMP_DIR=$(mktemp -d) +CONSTRAINTS="$TEMP_DIR/constraints.txt" +# We strip the hashes because we don't have hashes of our local packages and +# the mix of hashed and unhashed packages makes pip error out. +python3 tools/strip_hashes.py tools/certbot_constraints.txt > "$CONSTRAINTS" +python3 tools/strip_hashes.py tools/pipstrap_constraints.txt >> "$CONSTRAINTS" # We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 # because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been # dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. # Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then - sed -i 's|cryptography==.*|cryptography==3.1.1|g' requirements.txt - sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' requirements.txt + sed -i 's|cryptography==.*|cryptography==3.1.1|g' "$CONSTRAINTS" + sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' "$CONSTRAINTS" fi -python3 -m venv $VENV_PATH -$VENV_PATH/bin/python3 tools/pipstrap.py -PIP_CONSTRAINT=constraints.txt PIP_NO_BINARY=:all: $VENV_PATH/bin/python3 -m pip install --requirement requirements.txt -. "$VENV_PATH/bin/activate" -# pytest is needed to run tests on some of our packages so we install a pinned version here. -tools/pip_install.py pytest PLUGINS="certbot-apache certbot-nginx" -TEMP_DIR=$(mktemp -d) - # build sdists for pkg_dir in acme certbot $PLUGINS; do cd $pkg_dir @@ -50,8 +50,7 @@ cd $TEMP_DIR for pkg in acme certbot $PLUGINS; do tar -xvf "$pkg-$VERSION.tar.gz" cd "$pkg-$VERSION" - python setup.py build + PIP_CONSTRAINT=../constraints.txt PIP_NO_BINARY=:all: pip install . python -m pytest - python setup.py install cd - done From 74f6f734c8100385e395057082acee2926f9a5a9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Mar 2021 14:00:47 -0700 Subject: [PATCH 40/99] remove outdated comment (#8736) --- tools/docker/core/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 256db0169..4c614d5e2 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -27,10 +27,6 @@ RUN apk add --no-cache --virtual .certbot-deps \ binutils # Install certbot from sources -# -# We don't use tools/pip_install.py below so the hashes in -# dependency-requirements.txt can be used when installing packages for extra -# security. RUN apk add --no-cache --virtual .build-deps \ gcc \ linux-headers \ From 8d8b35b7c0dfe43e997f4bad6c94783508e79251 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 24 Mar 2021 15:55:30 -0700 Subject: [PATCH 41/99] update requirements (#8739) --- tests/letstest/requirements.txt | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index d30c507cf..b49489283 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -1,23 +1,22 @@ -awscli==1.18.88 -bcrypt==3.1.7 -boto3==1.14.11 -botocore==1.17.11 -cffi==1.14.0 +awscli==1.19.36 +bcrypt==3.2.0 +boto3==1.17.36 +botocore==1.20.36 +cffi==1.14.5 colorama==0.4.3 -cryptography==2.8 +cryptography==3.4.6 docutils==0.15.2 -enum34==1.1.9 -fabric==2.5.0 -invoke==1.4.1 -ipaddress==1.0.23 -jmespath==0.9.5 -paramiko==2.7.1 +fabric==2.6.0 +invoke==1.5.0 +jmespath==0.10.0 +paramiko==2.7.2 +pathlib2==2.3.5 pyasn1==0.4.8 -pycparser==2.19 -PyNaCl==1.3.0 +pycparser==2.20 +PyNaCl==1.4.0 python-dateutil==2.8.1 -PyYAML==5.3 -rsa==3.4.2 -s3transfer==0.3.3 -six==1.14.0 -urllib3==1.25.8 +PyYAML==5.4.1 +rsa==4.5 +s3transfer==0.3.6 +six==1.15.0 +urllib3==1.26.4 From fd62a091973a9beeac721d53249dd4a9d9663269 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Mar 2021 16:19:54 -0700 Subject: [PATCH 42/99] dump test farm failure logs (#8740) --- tests/letstest/multitester.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 2e4399000..8c0528990 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -496,12 +496,17 @@ def main(): outputs = [outq for outq in iter(outqueue.get, SENTINEL)] outputs.sort(key=lambda x: x[0]) failed = False + results_msg = "" for outq in outputs: ii, target, status = outq if status == Status.FAIL: failed = True - print('%d %s %s'%(ii, target['name'], status)) + with open(log_dir+'/'+'%d_%s.log'%(ii,target['name']), 'r') as f: + print(target['name'] + " test failed. Test log:") + print(f.read()) + results_msg = results_msg + '%d %s %s\n'%(ii, target['name'], status) results_file.write('%d %s %s\n'%(ii, target['name'], status)) + print(results_msg) if len(outputs) != num_processes: failed = True failure_message = 'FAILURE: Some target machines failed to run and were not tested. ' +\ From fa25d8356d8b5ae3c454f13ccd3e1d371d873f75 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Wed, 24 Mar 2021 16:58:15 -0700 Subject: [PATCH 43/99] Remove references to certbot-auto and letsencrypt-auto that we don't need for the final release (#8738) Fixes #8661. As mentioned in https://github.com/certbot/certbot/issues/8661#issuecomment-806168214, there are quite a few remaining references, but until we modify the release script, we still need those. The changes here and the list there were created by grepping for the following terms: ``` certbot-auto cb-auto cbauto certbotauto letsencrypt-auto le-auto leauto letsencryptauto LEAUTO LE_AUTO LETSENCRYPT_AUTO LETSENCRYPTAUTO CB_AUTO CERTBOT_AUTO CBAUTO CERTBOTAUTO ``` * Remove references to certbot-auto from certbot code * Remove references to LEAUTO * Remove references to CERTBOT_AUTO * Remove references to letsencrypt-auto * Remove references to certbot-auto from docs and tools * remove cli constants header files * Remove Python virtual environment section --- certbot/certbot/_internal/cli/__init__.py | 3 -- .../certbot/_internal/cli/cli_constants.py | 26 +--------- .../certbot/_internal/plugins/selection.py | 11 +---- certbot/docs/contributing.rst | 49 ------------------- certbot/docs/install.rst | 19 ------- certbot/docs/using.rst | 4 +- certbot/tests/cli_test.py | 1 - certbot/tests/plugins/selection_test.py | 20 -------- tests/letstest/scripts/test_tests.sh | 31 ------------ tools/dev_constraints.txt | 2 +- tools/pip_install.py | 2 +- tools/pip_install_editable.py | 2 +- 12 files changed, 7 insertions(+), 163 deletions(-) delete mode 100755 tests/letstest/scripts/test_tests.sh diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index d48fd419a..d0982f989 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -14,9 +14,6 @@ from certbot._internal.cli.cli_constants import COMMAND_OVERVIEW from certbot._internal.cli.cli_constants import DEPRECATED_OPTIONS from certbot._internal.cli.cli_constants import EXIT_ACTIONS from certbot._internal.cli.cli_constants import HELP_AND_VERSION_USAGE -from certbot._internal.cli.cli_constants import LEAUTO -from certbot._internal.cli.cli_constants import new_path_prefix -from certbot._internal.cli.cli_constants import old_path_fragment from certbot._internal.cli.cli_constants import SHORT_USAGE from certbot._internal.cli.cli_constants import VAR_MODIFIERS from certbot._internal.cli.cli_constants import ZERO_ARG_ACTIONS diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index bd25f9bee..df815f2e6 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -1,29 +1,5 @@ """Certbot command line constants""" -import sys - -from certbot.compat import os - -# For help strings, figure out how the user ran us. -# When invoked from letsencrypt-auto, sys.argv[0] is something like: -# "/home/user/.local/share/certbot/bin/certbot" -# Note that this won't work if the user set VENV_PATH or XDG_DATA_HOME before -# running letsencrypt-auto (and sudo stops us from seeing if they did), so it -# should only be used for purposes where inability to detect letsencrypt-auto -# fails safely - -LEAUTO = "letsencrypt-auto" -if "CERTBOT_AUTO" in os.environ: - # if we're here, this is probably going to be certbot-auto, unless the - # user saved the script under a different name - LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) - -old_path_fragment = os.path.join(".local", "share", "letsencrypt") -new_path_prefix = os.path.abspath(os.path.join(os.sep, "opt", - "eff.org", "certbot", "venv")) -if old_path_fragment in sys.argv[0] or sys.argv[0].startswith(new_path_prefix): - cli_command = LEAUTO -else: - cli_command = "certbot" +cli_command = "certbot" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index bf7e505b9..f0cce002f 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -134,19 +134,10 @@ def choose_plugin(prepared, question): opts = [plugin_ep.description_with_name + (" [Misconfigured]" if plugin_ep.misconfigured else "") for plugin_ep in prepared] - names = set(plugin_ep.name for plugin_ep in prepared) while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: - # The possibility of being offered exactly apache and nginx here - # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, - # so set apache as a default for those kinds of non-interactive use - # (the user will get a warning to set --non-interactive or --force-interactive) - apache_idx = [n for n, p in enumerate(prepared) if p.name == "apache"][0] - code, index = disp.menu(question, opts, default=apache_idx) - else: - code, index = disp.menu(question, opts, force_interactive=True) + code, index = disp.menu(question, opts, force_interactive=True) if code == display_util.OK: plugin_ep = prepared[index] diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 13ac5ed68..94e35f1bf 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -222,8 +222,6 @@ certbot-apache and certbot-nginx client code to configure specific web servers certbot-dns-* client code to configure DNS providers -certbot-auto and letsencrypt-auto - shell scripts to install Certbot and its dependencies on UNIX systems windows installer Installs Certbot on Windows and is built using the files in windows-installer/ @@ -548,53 +546,6 @@ Instructions for how to manually build and run the Certbot snap and the external snapped DNS plugins that the Certbot project supplies are located in the README file at https://github.com/certbot/certbot/tree/master/tools/snap. -Updating certbot-auto and letsencrypt-auto -========================================== - -.. note:: We are currently only accepting changes to certbot-auto that fix - regressions on platforms where certbot-auto is the recommended installation - method at https://certbot.eff.org/instructions. If you are unsure if a change - you want to make qualifies, don't hesitate to `ask for help`_! - -Updating the scripts --------------------- -Developers should *not* modify the ``certbot-auto`` and ``letsencrypt-auto`` files -in the root directory of the repository. Rather, modify the -``letsencrypt-auto.template`` and associated platform-specific shell scripts in -the ``letsencrypt-auto-source`` and -``letsencrypt-auto-source/pieces/bootstrappers`` directory, respectively. - -Building letsencrypt-auto-source/letsencrypt-auto -------------------------------------------------- -Once changes to any of the aforementioned files have been made, the -``letsencrypt-auto-source/letsencrypt-auto`` script should be updated. In lieu of -manually updating this script, run the build script, which lives at -``letsencrypt-auto-source/build.py``: - -.. code-block:: shell - - python letsencrypt-auto-source/build.py - -Running ``build.py`` will update the ``letsencrypt-auto-source/letsencrypt-auto`` -script. Note that the ``certbot-auto`` and ``letsencrypt-auto`` scripts in the root -directory of the repository will remain **unchanged** after this script is run. -Your changes will be propagated to these files during the next release of -Certbot. - -Opening a PR ------------- -When opening a PR, ensure that the following files are committed: - -1. ``letsencrypt-auto-source/letsencrypt-auto.template`` and - ``letsencrypt-auto-source/pieces/bootstrappers/*`` -2. ``letsencrypt-auto-source/letsencrypt-auto`` (generated by ``build.py``) - -It might also be a good idea to double check that **no** changes were -inadvertently made to the ``certbot-auto`` or ``letsencrypt-auto`` scripts in the -root of the repository. These scripts will be updated by the core developers -during the next release. - - Updating the documentation ========================== diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index ac6c798eb..99cf11c16 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -251,25 +251,6 @@ Certbot on UNIX operating systems, however, this script is no longer supported. If you want to uninstall ``certbot-auto``, you can follow our instructions :doc:`here `. -Problems with Python virtual environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When using ``certbot-auto`` on a low memory system such as VPS with less than -512MB of RAM, the required dependencies of Certbot may fail to build. This can -be identified if the pip outputs contains something like ``internal compiler -error: Killed (program cc1)``. You can workaround this restriction by creating -a temporary swapfile:: - - user@webserver:~$ sudo fallocate -l 1G /tmp/swapfile - user@webserver:~$ sudo chmod 600 /tmp/swapfile - user@webserver:~$ sudo mkswap /tmp/swapfile - user@webserver:~$ sudo swapon /tmp/swapfile - -Disable and remove the swapfile once the virtual environment is constructed:: - - user@webserver:~$ sudo swapoff /tmp/swapfile - user@webserver:~$ sudo rm /tmp/swapfile - Pip --- diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index cd52c7fe5..e38725ca3 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -14,7 +14,7 @@ obtaining, renewing, or revoking certificates. The most important and commonly-used commands will be discussed throughout this document; an exhaustive list also appears near the end of the document. -The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used an alternate installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. +The ``certbot`` script on your web server might be named ``letsencrypt`` if your system uses an older package, or ``certbot-auto`` if you used a now-deprecated installation method. Throughout the docs, whenever you see ``certbot``, swap in the correct name as needed. .. _plugins: @@ -420,7 +420,7 @@ option to control the curve used in ECDSA certificates. .. warning:: If you obtain certificates using ECDSA keys, you should be careful not to downgrade your Certbot installation since ECDSA keys are not supported by older versions of Certbot. Downgrades like this are possible if - you switch from something like the snaps or certbot-auto to packages + you switch from something like the snaps or pip to packages provided by your operating system which often lag behind. Changing existing certificates from RSA to ECDSA diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index fca2b3e3e..0a0d2634a 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -166,7 +166,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("--checkpoints" not in out) out = self._help_output(['-h']) - self.assertTrue("letsencrypt-auto" not in out) # test cli.cli_command if "nginx" in PLUGINS: self.assertTrue("Use the Nginx plugin" in out) else: diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index 2dfd60a63..eed07ae76 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -155,26 +155,6 @@ class ChoosePluginTest(unittest.TestCase): mock_util().menu.return_value = (display_util.CANCEL, 0) self.assertTrue(self._call() is None) - @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") - def test_new_interaction_avoidance(self, mock_util): - mock_nginx = mock.Mock( - description_with_name="n", misconfigured=False) - mock_nginx.init().more_info.return_value = "nginx plugin" - mock_nginx.name = "nginx" - self.plugins[1] = mock_nginx - mock_util().menu.return_value = (display_util.CANCEL, 0) - - unset_cb_auto = os.environ.get("CERTBOT_AUTO") is None - if unset_cb_auto: - os.environ["CERTBOT_AUTO"] = "foo" - try: - self._call() - finally: - if unset_cb_auto: - del os.environ["CERTBOT_AUTO"] - - self.assertTrue("default" in mock_util().menu.call_args[1]) - class GetUnpreparedInstallerTest(test_util.ConfigTestCase): """Tests for certbot._internal.plugins.selection.get_unprepared_installer.""" diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh deleted file mode 100755 index 858fc1f18..000000000 --- a/tests/letstest/scripts/test_tests.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -xe -# -# This script is useful for testing that the packages we've built for a release -# work on a variety of systems. For an example of the kinds of problems that -# can occur, see https://github.com/certbot/certbot/issues/3455. - -REPO_ROOT="letsencrypt" -LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" -LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" -MODULES="acme certbot certbot-apache certbot-nginx" -PIP_INSTALL="tools/pip_install.py" -VENV_NAME=venv -BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh" -VENV_SCRIPT="tools/venv.py" - -sudo $BOOTSTRAP_SCRIPT - -cd $REPO_ROOT -$VENV_SCRIPT -. $VENV_NAME/bin/activate -"$PIP_INSTALL" pytest - -# To run tests that aren't packaged in modules, run pytest -# from the repo root. The directory structure should still -# cause the installed packages to be tested while using -# the tests available in the subdirectories. - -for module in $MODULES ; do - echo testing $module - pytest -v $module -done diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 4f5eda34e..ae175cdd4 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,5 @@ # Specifies Python package versions for development and building Docker images. -# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# It includes in particular packages not specified in tools/certbot_constraints.txt. # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. tools/certbot_constraints.txt). alabaster==0.7.10 diff --git a/tools/pip_install.py b/tools/pip_install.py index e06650ff2..4396af17a 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -3,7 +3,7 @@ # to 1, a combination of tools/oldest_constraints.txt, # tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the # top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# tools/certbot_constraints.txt and tools/dev_constraints.txt is used. The # other file always takes precedence over tools/dev_constraints.txt. If # CERTBOT_OLDEST is set, this script must be run with `-e ` and # no other arguments. diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index de2a0ff57..646344f3a 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pip installs packages in editable mode using certbot-auto's requirements file +# pip installs packages in editable mode using tools/certbot_constraints.txt # as constraints # # cryptography is currently using this script in their CI at From 018efc241c7fd219b8c3b4d7a52e4c365850060d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 25 Mar 2021 22:38:34 +0100 Subject: [PATCH 44/99] Split snap build over three Azure jobs (one per architecture) (#8731) Fixes #8700 Now that `snapcraft remote-build` truly uses new builds for each call, we can split the builds to have a dedicated Azure job for each target architecture. This PR does that. * Split snap_build job on each architecture * Also parallelize the publish_snap jobs over each architecture --- .../templates/jobs/packaging-jobs.yml | 26 +++++++++++-------- .../templates/stages/deploy-stage.yml | 10 ++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 28255919f..1b8bcad68 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -116,13 +116,17 @@ jobs: - job: snaps_build pool: vmImage: ubuntu-18.04 + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + # Do not run the heavy non-amd64 builds for test branches + ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: + armhf: + SNAP_ARCH: armhf + arm64: + SNAP_ARCH: arm64 timeoutInMinutes: 0 - variables: - # Do not run the heavy non-amd64 builds for test branches - ${{ if not(startsWith(variables['Build.SourceBranchName'], 'test-')) }}: - ARCHS: amd64 arm64 armhf - ${{ if startsWith(variables['Build.SourceBranchName'], 'test-') }}: - ARCHS: amd64 steps: - script: | set -e @@ -144,7 +148,7 @@ jobs: git config --global user.name "$(Build.RequestedFor)" mkdir -p ~/.local/share/snapcraft/provider/launchpad cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials - python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800 + python3 tools/snap/build_remote.py ALL --archs ${SNAP_ARCH} --timeout 19800 displayName: Build snaps - script: | set -e @@ -154,7 +158,7 @@ jobs: - task: PublishPipelineArtifact@1 inputs: path: $(Build.ArtifactStagingDirectory) - artifact: snaps + artifact: snaps_$(SNAP_ARCH) displayName: Store snaps artifacts - job: snap_run dependsOn: snaps_build @@ -175,12 +179,12 @@ jobs: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | set -e - sudo snap install --dangerous --classic snap/certbot_*_amd64.snap + sudo snap install --dangerous --classic snap/certbot_*.snap displayName: Install Certbot snap - script: | set -e @@ -202,7 +206,7 @@ jobs: addToPath: true - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_amd64 path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - script: | diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index ac2044f99..f42dd2a28 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -37,6 +37,14 @@ stages: vmImage: ubuntu-18.04 variables: - group: certbot-common + strategy: + matrix: + amd64: + SNAP_ARCH: amd64 + arm32v6: + SNAP_ARCH: armhf + arm64v8: + SNAP_ARCH: arm64 steps: - bash: | set -e @@ -46,7 +54,7 @@ stages: displayName: Install dependencies - task: DownloadPipelineArtifact@2 inputs: - artifact: snaps + artifact: snaps_$(SNAP_ARCH) path: $(Build.SourcesDirectory)/snap displayName: Retrieve Certbot snaps - task: DownloadSecureFile@1 From f4fc3e636d295776f0e6be53763f401e1cf4c41e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Mar 2021 23:51:59 -0700 Subject: [PATCH 45/99] Redo the majority of Certbot's pinning system (#8741) * add initial pyproject.toml * add extra dependencies * add simple bash script * polish * reuse pipstrap * add requirements.txt * temporarily remove hashin dep * Switch to requirements.txt * remove hashin check * update requirements.txt again * remove unnecessary merge * pin back augeas * unpin cryptography * simplify pywin32 pinning * update comment * pin back pytest and pylint * pin back pytest-forked * pin back coverage * update script comments * fix pyopenssl case * add minimum poetry version * run pin.sh --- .gitignore | 1 + certbot/setup.py | 5 +- snap/snapcraft.yaml | 4 +- tests/letstest/scripts/test_sdists.sh | 13 +- tools/certbot_constraints.txt | 260 ------------------------ tools/dev_constraints.txt | 2 +- tools/pinning/pin.sh | 51 +++++ tools/pinning/pyproject.toml | 56 ++++++ tools/pip_install.py | 62 ++---- tools/rebuild_certbot_constraints.py | 277 -------------------------- tools/requirements.txt | 176 ++++++++++++++++ tools/snap/generate_dnsplugins_all.sh | 5 +- windows-installer/construct.py | 35 +--- 13 files changed, 323 insertions(+), 624 deletions(-) delete mode 100644 tools/certbot_constraints.txt create mode 100755 tools/pinning/pin.sh create mode 100644 tools/pinning/pyproject.toml delete mode 100755 tools/rebuild_certbot_constraints.py create mode 100644 tools/requirements.txt diff --git a/.gitignore b/.gitignore index 5169defd6..34b3fc99e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ dist*/ letsencrypt.log certbot.log letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 +poetry.lock # coverage .coverage diff --git a/certbot/setup.py b/certbot/setup.py index 8b2874f20..0f54179a3 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -59,7 +59,7 @@ install_requires = [ # However environment markers are supported only with setuptools >= 36.2. # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=300' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py +pywin32_req = 'pywin32>=300' setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") @@ -79,6 +79,9 @@ dev_extras = [ 'ipdb', 'mypy', 'PyGithub', + # 1.1.0+ is required for poetry to use the poetry-core library for the + # build system declared in tools/pinning/pyproject.toml. + 'poetry>=1.1.0', 'pylint', 'pytest', 'pytest-cov', diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ccaafd93b..b3f1349e0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -93,9 +93,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + grep -v python-augeas "${SNAPCRAFT_PART_SRC}/tools/requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index ad729e0eb..aa385ad2f 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -18,18 +18,15 @@ tools/pip_install.py pytest # setup constraints TEMP_DIR=$(mktemp -d) CONSTRAINTS="$TEMP_DIR/constraints.txt" -# We strip the hashes because we don't have hashes of our local packages and -# the mix of hashed and unhashed packages makes pip error out. -python3 tools/strip_hashes.py tools/certbot_constraints.txt > "$CONSTRAINTS" -python3 tools/strip_hashes.py tools/pipstrap_constraints.txt >> "$CONSTRAINTS" +cp tools/requirements.txt "$CONSTRAINTS" -# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# We pin cryptography to 3.1.1 and pyopenssl to 19.1.0 specifically for CentOS 7 / RHEL 7 # because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been -# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. -# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +# dropped on cryptography>=3.2 and pyopenssl>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyopenssl wheels builds. if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then sed -i 's|cryptography==.*|cryptography==3.1.1|g' "$CONSTRAINTS" - sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' "$CONSTRAINTS" + sed -i 's|pyopenssl==.*|pyopenssl==19.1.0|g' "$CONSTRAINTS" fi diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt deleted file mode 100644 index 2f48c8b3e..000000000 --- a/tools/certbot_constraints.txt +++ /dev/null @@ -1,260 +0,0 @@ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.4 \ - --hash=sha256:abef9ff44fb0091f0e3bb2ee7e5b26a02b5b62d45408a5272a9bd461f5b59b4b -certifi==2020.12.5 \ - --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ - --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 -cffi==1.14.5 \ - --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ - --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ - --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ - --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ - --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ - --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ - --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ - --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ - --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ - --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ - --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ - --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ - --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ - --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ - --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ - --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ - --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ - --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ - --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ - --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ - --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ - --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ - --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ - --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ - --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ - --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ - --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ - --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ - --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ - --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ - --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ - --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ - --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ - --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ - --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ - --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ - --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c -chardet==4.0.0 \ - --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ - --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==3.4.6 \ - --hash=sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b \ - --hash=sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336 \ - --hash=sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87 \ - --hash=sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7 \ - --hash=sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799 \ - --hash=sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b \ - --hash=sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df \ - --hash=sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0 \ - --hash=sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3 \ - --hash=sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724 \ - --hash=sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2 \ - --hash=sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -idna==2.10 \ - --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ - --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 -josepy==1.8.0 \ - --hash=sha256:6d632fcdaf0bed09e33f81f13b10575d4f0b7c37319350b725454e04a41e6a49 \ - --hash=sha256:a5a182eb499665d99e7ec54bb3fe389f9cbc483d429c9651f20384ba29564269 -parsedatetime==2.6 \ - --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ - --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b -pyOpenSSL==20.0.1 \ - --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ - --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2021.1 \ - --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ - --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 -requests==2.25.1 \ - --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ - --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 -zope.component==4.6.2 \ - --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ - --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.5.0 \ - --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ - --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.2.0 \ - --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ - --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ - --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ - --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ - --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ - --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ - --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ - --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ - --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ - --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ - --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ - --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ - --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ - --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ - --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ - --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ - --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ - --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ - --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ - --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ - --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ - --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ - --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ - --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ - --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ - --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ - --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ - --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ - --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ - --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ - --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ - --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ - --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ - --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ - --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ - --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ - --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ - --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ - --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ - --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ - --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ - --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ - --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ - --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ - --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ - --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ - --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ - --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ - --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ - --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ - --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ - --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index ae175cdd4..43a27a3d0 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in tools/certbot_constraints.txt. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. tools/certbot_constraints.txt). +# files during tests (eg. tools/oldest_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 diff --git a/tools/pinning/pin.sh b/tools/pinning/pin.sh new file mode 100755 index 000000000..ce47f02a0 --- /dev/null +++ b/tools/pinning/pin.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies. Dependencies can be pinned to older versions by +# modifying pyproject.toml in the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" +PIPSTRAP_CONSTRAINTS="${REPO_ROOT}/tools/pipstrap_constraints.txt" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" +STRIP_HASHES="${REPO_ROOT}/tools/strip_hashes.py" + +if ! command -v poetry >/dev/null; then + echo "Please install poetry." + echo "You may need to recreate Certbot's virtual environment and activate it." + exit 1 +fi + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock + +TEMP_REQUIREMENTS=$(mktemp) +trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT + +poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes +# We need to remove local packages from the requirements file. +sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" +# Poetry currently will not include pip, setuptools, or wheel in lockfiles or +# requirements files. This was resolved by +# https://github.com/python-poetry/poetry/pull/2826, but as of writing this it +# hasn't been included in a release yet. For now, we continue to keep +# pipstrap's pinning separate which has the added benefit of having it continue +# to check hashes when pipstrap is run directly. +"${STRIP_HASHES}" "${PIPSTRAP_CONSTRAINTS}" >> "${TEMP_REQUIREMENTS}" + +cat << EOF > "$REQUIREMENTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +EOF +cat "${TEMP_REQUIREMENTS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml new file mode 100644 index 000000000..553ef9476 --- /dev/null +++ b/tools/pinning/pyproject.toml @@ -0,0 +1,56 @@ +[tool.poetry] +name = "certbot-pinner" +version = "0.1.0" +description = "A simple project for pinning Certbot's dependencies using Poetry." +authors = ["Certbot Project"] +license = "Apache License 2.0" + +[tool.poetry.dependencies] +python = "^3.6" + +# Local dependencies +# Any local packages that have dependencies on other local packages must be +# listed below before the package it depends on. For instance, certbot depends +# on acme so certbot must be listed before acme. +certbot-ci = {path = "../../certbot-ci", extras = ["docs"]} +certbot-compatibility-test = {path = "../../certbot-compatibility-test", extras = ["docs"]} +certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]} +certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]} +certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]} +certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]} +certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]} +certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]} +certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]} +certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]} +certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]} +certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]} +certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]} +certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]} +certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]} +certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]} +certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} +certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../certbot", extras = ["dev", "docs"]} +acme = {path = "../../acme", extras = ["dev", "docs"]} + +# Extra dependencies +# See https://github.com/certbot/certbot/issues/8425. +mypy = "0.710" +# Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many +# test failures so let's pin these packages back for now. +coverage = "4.5.4" +pylint = "2.4.3" +pytest = "3.2.5" +pytest-forked = "0.2" +# We were originally pinning back python-augeas for certbot-auto because we +# found the way older versions of the library linked to Augeas were more +# reliable. That's no longer a concern, however, we continue to pin back the +# library for now because it causes Certbot tests on Windows to fail. See +# https://github.com/certbot/certbot/issues/8732. +python-augeas = "0.5.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/pip_install.py b/tools/pip_install.py index 4396af17a..054ca1229 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -39,51 +39,34 @@ def find_tools_path(): return os.path.dirname(readlink.main(__file__)) -def certbot_oldest_processing(tools_path, args, test_constraints): +def certbot_oldest_processing(tools_path, args, constraints_path): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' 'with a single -e argument.') # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + # The order of the files in this list matters as files specified later can + # override the pinnings found in earlier files. + pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'), + os.path.join(tools_path, 'oldest_constraints.txt')] requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements - if not os.path.isfile(requirements): - return None - + if os.path.isfile(requirements): + # We add requirements to the end of the list so it can override + # anything that it needs to. + pinning_files.append(requirements) + else: + requirements = None + with open(constraints_path, 'w') as fd: + fd.write(merge_module.main(*pinning_files)) return requirements -def certbot_normal_processing(tools_path, test_constraints): +def certbot_normal_processing(tools_path, constraints_path): repo_path = os.path.dirname(tools_path) - certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'tools/certbot_constraints.txt')) - with open(certbot_requirements, 'r') as fd: - certbot_reqs = fd.readlines() - with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: - pipstrap_reqs = fd.readlines() - with open(test_constraints, 'w') as fd: - data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) - data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) - data = "\n".join([data_certbot, data_pipstrap]) - fd.write(data) - - -def merge_requirements(tools_path, requirements, test_constraints, all_constraints): - # Order of the files in the merge function matters. - # Indeed version retained for a given package will be the last version - # found when following all requirements in the given order. - # Here is the order by increasing priority: - # 1) The general development constraints (tools/dev_constraints.txt) - # 2) The general tests constraints (oldest_requirements.txt or - # certbot_constraints.txt + pipstrap's constraints for the normal processing) - # 3) The local requirement file, typically local-oldest-requirement in oldest tests - files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] - if requirements: - files.append(requirements) - merged_requirements = merge_module.main(*files) - with open(all_constraints, 'w') as fd: - fd.write(merged_requirements) + requirements = os.path.normpath(os.path.join( + repo_path, 'tools/requirements.txt')) + shutil.copy(requirements, constraints_path) def call_with_print(command, env=None): @@ -104,24 +87,21 @@ def main(args): tools_path = find_tools_path() with temporary_directory() as working_dir: - test_constraints = os.path.join(working_dir, 'test_constraints.txt') - all_constraints = os.path.join(working_dir, 'all_constraints.txt') - if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint pip_install_with_print(' '.join(args)) else: # Otherwise, we merge requirements to build the constraints and pin dependencies + constraints_path = os.path.join(working_dir, 'constraints.txt') requirements = None if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + requirements = certbot_oldest_processing(tools_path, args, constraints_path) else: - certbot_normal_processing(tools_path, test_constraints) + certbot_normal_processing(tools_path, constraints_path) env = os.environ.copy() - env["PIP_CONSTRAINT"] = all_constraints + env["PIP_CONSTRAINT"] = constraints_path - merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. diff --git a/tools/rebuild_certbot_constraints.py b/tools/rebuild_certbot_constraints.py deleted file mode 100755 index 14479d8ec..000000000 --- a/tools/rebuild_certbot_constraints.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python -""" -Gather and consolidate the up-to-date dependencies available and required to install certbot -on various Linux distributions. It generates a requirements file contained the pinned and hashed -versions, ready to be used by pip to install the certbot dependencies. - -This script is typically used to update the certbot_constraints.txt file. - -To achieve its purpose, this script will start a certbot installation with unpinned dependencies, -then gather them, on various distributions started as Docker containers. - -Usage: tools/rebuild_certbot_constraints.py new_requirements.txt - -NB1: Docker must be installed on the machine running this script. -NB2: Python library 'hashin' must be installed on the machine running this script. -""" -from __future__ import print_function -import re -import shutil -import subprocess -import tempfile -import os -from os.path import dirname, abspath, join -import sys -import argparse - -# The list of docker distributions to test dependencies against with. -DISTRIBUTION_LIST = [ - 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', - 'centos:8', 'centos:7', 'fedora:29', -] - -# These constraints will be added while gathering dependencies on each distribution. -# It can be used because a particular version for a package is required for any reason, -# or to solve a version conflict between two distributions requirements. -AUTHORITATIVE_CONSTRAINTS = { - # Too touchy to move to a new version. And will be removed soon - # in favor of pure python parser for Apache. - 'python-augeas': '0.5.0', -} - -# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) -CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) - -# The script will be used to gather dependencies for a given distribution. -# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv -# - then this venv is used to consistently construct an empty new venv -# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), -# without pinned dependencies, and respecting input authoritative requirements -# - `certbot plugins` is called to check we have a healthy environment -# - finally current set of dependencies is extracted out of the docker using pip freeze -SCRIPT = r"""#!/bin/sh -set -ex - -cd /tmp/certbot -tests/letstest/scripts/bootstrap_os_packages.sh - -python3 -m venv /tmp/venv - -/tmp/venv/bin/python tools/pipstrap.py -/tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt -/tmp/venv/bin/certbot plugins -/tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt -""" - - -def _read_from(file): - """Read all content of the file, and return it as a string.""" - with open(file, 'r') as file_h: - return file_h.read() - - -def _write_to(file, content): - """Write given string content to the file, overwriting its initial content.""" - with open(file, 'w') as file_h: - file_h.write(content) - - -def _requirements_from_one_distribution(distribution, verbose): - """ - Calculate the Certbot dependencies expressed for the given distribution, using the official - Docker for this distribution, and return the lines of the generated requirements file. - """ - print('===> Gathering dependencies for {0}.'.format(distribution)) - workspace = tempfile.mkdtemp() - script = join(workspace, 'script.sh') - authoritative_constraints = join(workspace, 'constraints.txt') - cid_file = join(workspace, 'cid') - - try: - _write_to(script, SCRIPT) - os.chmod(script, 0o755) - - _write_to(authoritative_constraints, '\n'.join( - '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) - - command = ['docker', 'run', '--rm', '--cidfile', cid_file, - '--network=host', - '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), - '-v', '{0}:/tmp/workspace'.format(workspace), - '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), - distribution, '/tmp/workspace/script.sh'] - sub_stdout = sys.stdout if verbose else subprocess.PIPE - sub_stderr = sys.stderr if verbose else subprocess.STDOUT - process = subprocess.Popen(command, stdout=sub_stdout, stderr=sub_stderr, universal_newlines=True) - stdoutdata, _ = process.communicate() - - if process.returncode: - if stdoutdata: - sys.stderr.write('Output was:\n{0}'.format(stdoutdata)) - raise RuntimeError('Error while gathering dependencies for {0}.'.format(distribution)) - - with open(join(workspace, 'requirements.txt'), 'r') as file_h: - return file_h.readlines() - finally: - if os.path.isfile(cid_file): - cid = _read_from(cid_file) - try: - subprocess.check_output(['docker', 'kill', cid], stderr=subprocess.PIPE) - except subprocess.CalledProcessError: - pass - shutil.rmtree(workspace) - - -def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution): - """ - Extract every requirement from the given requirements file, and merge it in the dependency map. - Merging here means that the map contain every encountered dependency, and the version used in - each distribution. - - Example: - # dependencies_map = { - # } - _parse_and_merge_requirements(['cryptography=='1.2','requests=='2.1.0'], dependencies_map, 'debian:stretch') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch')] - # } - _parse_and_merge_requirements(['requests=='2.4.0', 'mock==1.3'], dependencies_map, 'centos:7') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch'), ('2.4.0', 'centos:7')], - # 'mock': [('2.4.0', 'centos:7')] - # } - """ - for line in requirements_file_lines: - match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and not line.startswith('#') and match: - package, version = match.groups() - if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: - dependencies_map.setdefault(package, []).append((version, distribution)) - - -def _consolidate_and_validate_dependencies(dependency_map): - """ - Given the dependency map of all requirements found in all distributions for Certbot, - construct an array containing the unit requirements for Certbot to be used by pip, - and the version conflicts, if any, between several distributions for a package. - Return requirements and conflicts as a tuple. - """ - print('===> Consolidate and validate the dependency map.') - requirements = [] - conflicts = [] - for package, versions in dependency_map.items(): - reduced_versions = _reduce_versions(versions) - - if len(reduced_versions) > 1: - version_list = ['{0} ({1})'.format(version, ','.join(distributions)) - for version, distributions in reduced_versions.items()] - conflict = ('package {0} is declared with several versions: {1}' - .format(package, ', '.join(version_list))) - conflicts.append(conflict) - sys.stderr.write('ERROR: {0}\n'.format(conflict)) - else: - requirements.append((package, list(reduced_versions)[0])) - - requirements.sort(key=lambda x: x[0]) - return requirements, conflicts - - -def _reduce_versions(version_dist_tuples): - """ - Get an array of version/distribution tuples, - and reduce it to a map based on the version values. - - Example: [('1.2.0', 'debian:stretch'), ('1.4.0', 'ubuntu:18.04'), ('1.2.0', 'centos:6')] - => {'1.2.0': ['debiqn:stretch', 'centos:6'], '1.4.0': ['ubuntu:18.04']} - """ - version_dist_map = {} - for version, distribution in version_dist_tuples: - version_dist_map.setdefault(version, []).append(distribution) - - return version_dist_map - - -def _write_requirements(dest_file, requirements, conflicts): - """ - Given the list of requirements and conflicts, write a well-formatted requirements file, - whose requirements are hashed signed using hashin library. Conflicts are written at the end - of the generated file. - """ - print('===> Calculating hashes for the requirement file.') - - _write_to(dest_file, '''\ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \\ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 -# ``` -''') - - for req in requirements: - if req[0] in AUTHORITATIVE_CONSTRAINTS: - # If requirement is in AUTHORITATIVE_CONSTRAINTS, take its value instead of the - # computed one to get any environment descriptor that would have been added. - req = (req[0], AUTHORITATIVE_CONSTRAINTS[req[0]]) - subprocess.check_call(['hashin', '{0}=={1}'.format(req[0], req[1]), - '--requirements-file', dest_file]) - - if conflicts: - with open(dest_file, 'a') as file_h: - file_h.write('\n## ! SOME ERRORS OCCURRED ! ##\n') - file_h.write('\n'.join('# {0}'.format(conflict) for conflict in conflicts)) - file_h.write('\n') - - return _read_from(dest_file) - - -def _gather_dependencies(dest_file, verbose): - """ - Main method of this script. Given a destination file path, will write the file - containing the consolidated and hashed requirements for Certbot, validated - against several Linux distributions. - """ - dependencies_map = {} - - for distribution in DISTRIBUTION_LIST: - requirements_file_lines = _requirements_from_one_distribution(distribution, verbose) - _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution) - - requirements, conflicts = _consolidate_and_validate_dependencies(dependencies_map) - - return _write_requirements(dest_file, requirements, conflicts) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' - ' artifacts, validated against several OS distributions using Docker.')) - parser.add_argument('requirements_path', - help='path for the generated requirements file') - parser.add_argument('--verbose', '-v', action='store_true', - help='verbose will display all output during docker execution') - - namespace = parser.parse_args() - - try: - subprocess.check_output(['hashin', '--version']) - except subprocess.CalledProcessError: - raise RuntimeError('Python library hashin is not installed in the current environment.') - - try: - subprocess.check_output(['docker', '--version'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - raise RuntimeError('Docker is not installed or accessible to current user.') - - file_content = _gather_dependencies(namespace.requirements_path, namespace.verbose) - - print(file_content) - print('===> Rebuilt requirement file is available on path {0}' - .format(abspath(namespace.requirements_path))) diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 000000000..89c57c974 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,176 @@ +# This file was generated by tools/pinning/pin.sh and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +alabaster==0.7.12; python_version >= "3.6" +apacheconfig==0.3.2; python_version >= "3.6" +apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +appnope==0.1.2 +astroid==2.3.3; python_version >= "3.6" +attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +azure-devops==6.0.0b4; python_version >= "3.6" +babel==2.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +backcall==0.2.0 +bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" +bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +boto3==1.17.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cloudflare==2.8.15; python_version >= "3.6" +colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +configobj==5.0.6; python_version >= "3.6" +coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") +crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +decorator==4.4.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.2.0" +deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +dns-lexicon==3.5.5; python_version >= "3.6" and python_version < "4.0" +dnspython==2.1.0; python_version >= "3.6" +docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" +future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" +google-api-core==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-api-python-client==2.0.2; python_version >= "3.6" +google-auth-httplib2==0.1.0; python_version >= "3.6" +google-auth==1.28.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +httplib2==0.19.0; python_version >= "3.6" +idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" +imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +ipdb==0.13.7; python_version >= "3.6" +ipython-genutils==0.2.0; python_version == "3.6" +ipython==7.16.1; python_version == "3.6" +ipython==7.21.0; python_version >= "3.7" +isodate==0.6.0; python_version >= "3.6" +isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +jedi==0.18.0 +jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +jinja2==2.11.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +josepy==1.8.0; python_version >= "3.6" +jsonlines==2.0.0; python_version >= "3.6" +jsonpickle==2.0.0; python_version >= "3.6" +jsonschema==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +lockfile==0.12.2 +markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +mccabe==0.6.1; python_version >= "3.6" +msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +msrest==0.6.21; python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" +mypy==0.710 +oauth2client==4.1.3; python_version >= "3.6" +oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +parsedatetime==2.6; python_version >= "3.6" +parso==0.8.1; python_version == "3.6" +pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pexpect==4.8.0 +pickleshare==0.7.5 +pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +ply==3.11; python_version >= "3.6" +poetry-core==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +prompt-toolkit==3.0.3 +protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pygithub==1.54.1; python_version >= "3.6" +pygments==2.8.1 +pyjwt==1.7.1; python_version >= "3.6" +pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pylint==2.4.3; python_version >= "3.5" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyrfc3339==1.1; python_version >= "3.6" +pyrsistent==0.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-cov==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-forked==0.2 +pytest-xdist==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest==3.2.5 +python-augeas==0.5.0 +python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +python-digitalocean==1.16.0; python_version >= "3.6" +python-dotenv==0.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" +pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0" +readme-renderer==29.0; python_version >= "3.6" +repoze.sphinx.autointerface==0.8; python_version >= "3.6" +requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" +requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" +rfc3986==1.4.0; python_version >= "3.6" +rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +s3transfer==0.3.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +snowballstemmer==2.1.0; python_version >= "3.6" +soupsieve==2.2.1; python_version >= "3.6" +sphinx-rtd-theme==0.5.1; python_version >= "3.6" +sphinx==3.5.3; python_version >= "3.6" +sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" +sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" +sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" +sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" +sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" +sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.6" +texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" +toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" +tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tqdm==4.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +traitlets==4.3.3 +twine==3.3.0; python_version >= "3.6" +typed-ast==1.4.2; python_version >= "3.6" and implementation_name == "cpython" and python_version < "3.8" +uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +wcwidth==0.2.5; python_version == "3.6" +webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") +zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.interface==5.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pip==20.2.4 +setuptools==54.1.2 +wheel==0.35.1 diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 976b0dd7b..769b4ef4d 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -9,8 +9,5 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file - "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ - > "${PLUGIN_PATH}"/snap-constraints.txt + cp "${CERTBOT_DIR}"/tools/requirements.txt "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index eb199a7e1..21ee4c8be 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -12,7 +12,6 @@ import time PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 -PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' @@ -46,12 +45,12 @@ def _compile_wheels(repo_path, build_path, venv_python): # certbot_packages.extend([name for name in os.listdir(repo_path) if name.startswith('certbot-dns-')]) wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] - with _prepare_constraints(repo_path) as constraints_file_path: - env = os.environ.copy() - env['PIP_CONSTRAINT'] = constraints_file_path - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] - command.extend(wheels_project) - subprocess.check_call(command, env=env) + constraints_file_path = os.path.join(repo_path, 'tools', 'requirements.txt') + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] + command.extend(wheels_project) + subprocess.check_call(command, env=env) # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible @@ -77,28 +76,6 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION]) -@contextlib.contextmanager -def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') - reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') - constraints_certbot = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], - universal_newlines=True) - constraints_pipstrap = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], - universal_newlines=True) - workdir = tempfile.mkdtemp() - try: - constraints_file_path = os.path.join(workdir, 'constraints.txt') - with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints_pipstrap) - file_h.write(constraints_certbot) - file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) - yield constraints_file_path - finally: - shutil.rmtree(workdir) - - def _copy_assets(build_path, repo_path): print('Copy assets') if os.path.exists(build_path): From 8759ccaecb69ea93c5b3c95f6af71bb9130117b1 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 29 Mar 2021 13:56:08 -0700 Subject: [PATCH 46/99] Update issue template to list snap as an installation option (#8754) --- ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 03917f8ca..87ecddc6c 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -7,7 +7,7 @@ questions. ## My operating system is (include version): -## I installed Certbot with (certbot-auto, OS package manager, pip, etc): +## I installed Certbot with (snap, OS package manager, pip, certbot-auto, etc): ## I ran this command and it produced this output: From 24fd4121cfe97478041bd91b654b62b193c7868f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 31 Mar 2021 12:41:29 -0700 Subject: [PATCH 47/99] use snapcraft login (#8756) --- .azure-pipelines/templates/stages/deploy-stage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml index f42dd2a28..abbb1fd1a 100644 --- a/.azure-pipelines/templates/stages/deploy-stage.yml +++ b/.azure-pipelines/templates/stages/deploy-stage.yml @@ -63,8 +63,7 @@ stages: secureFile: snapcraft.cfg - bash: | set -e - mkdir -p .snapcraft - ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg + snapcraft login --with $(snapcraftCfg.secureFilePath) for SNAP_FILE in snap/*.snap; do tools/retry.sh eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}" done From 1ea588d504b7fe413a2b3cb5aed28724cef62a05 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 31 Mar 2021 12:42:42 -0700 Subject: [PATCH 48/99] increase ARM build timeout (#8757) --- .azure-pipelines/templates/jobs/packaging-jobs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 1b8bcad68..949e57bb5 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -12,6 +12,9 @@ jobs: DOCKER_ARCH: arm32v6 arm64v8: DOCKER_ARCH: arm64v8 + # The default timeout of 60 minutes is a little low for compiling + # cryptography on ARM architectures. + timeoutInMinutes: 180 steps: - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH displayName: Build the Docker images From fea0b4e2e5458b0135b3d02791e2152a3ccd3d6f Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 1 Apr 2021 22:57:03 +0200 Subject: [PATCH 49/99] Pin pynsist (#8749) * Pin pynsist * Update dependencies * Set windows installer a proper python project * Optimize usage of the venvs * Add windows-installer when venv is set up * Fix call * Remove env marker --- .../templates/jobs/packaging-jobs.yml | 8 ++- tools/pinning/pyproject.toml | 1 + tools/requirements.txt | 21 ++++---- tools/venv.py | 3 ++ windows-installer/{ => assets}/certbot.ico | Bin windows-installer/{ => assets}/renew-down.ps1 | 0 windows-installer/{ => assets}/renew-up.ps1 | 0 windows-installer/{ => assets}/run.bat | 0 windows-installer/{ => assets}/template.nsi | 0 windows-installer/setup.py | 39 ++++++++++++++ .../windows_installer/__init__.py | 0 .../{ => windows_installer}/construct.py | 50 +++++++++--------- 12 files changed, 86 insertions(+), 36 deletions(-) rename windows-installer/{ => assets}/certbot.ico (100%) rename windows-installer/{ => assets}/renew-down.ps1 (100%) rename windows-installer/{ => assets}/renew-up.ps1 (100%) rename windows-installer/{ => assets}/run.bat (100%) rename windows-installer/{ => assets}/template.nsi (100%) create mode 100644 windows-installer/setup.py create mode 100644 windows-installer/windows_installer/__init__.py rename windows-installer/{ => windows_installer}/construct.py (87%) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 949e57bb5..005b5ef48 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -62,7 +62,13 @@ jobs: versionSpec: 3.8 architecture: x86 addToPath: true - - script: python windows-installer/construct.py + - script: | + python -m venv venv + venv\Scripts\python tools\pipstrap.py + venv\Scripts\python tools\pip_install.py -e windows-installer + displayName: Prepare Windows installer build environment + - script: | + venv\Scripts\construct-windows-installer displayName: Build Certbot installer - task: CopyFiles@2 inputs: diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 553ef9476..9112fb054 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -32,6 +32,7 @@ certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} certbot = {path = "../../certbot", extras = ["dev", "docs"]} acme = {path = "../../acme", extras = ["dev", "docs"]} +windows-installer = {path = "../../windows-installer"} # Extra dependencies # See https://github.com/certbot/certbot/issues/8425. diff --git a/tools/requirements.txt b/tools/requirements.txt index 89c57c974..4c6c4825e 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -18,8 +18,8 @@ backcall==0.2.0 bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.42; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.42; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -40,7 +40,7 @@ decorator==4.4.2; python_version == "3.6" and python_full_version < "3.0.0" or p deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -dns-lexicon==3.5.5; python_version >= "3.6" and python_version < "4.0" +dns-lexicon==3.5.6; python_version >= "3.6" and python_version < "4.0" dnspython==2.1.0; python_version >= "3.6" docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" @@ -50,13 +50,13 @@ docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or ( execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" -google-api-core==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.0.2; python_version >= "3.6" +google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-api-python-client==2.1.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" google-auth==1.28.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -httplib2==0.19.0; python_version >= "3.6" +httplib2==0.19.1; python_version >= "3.6" idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") @@ -64,7 +64,7 @@ importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3 ipdb==0.13.7; python_version >= "3.6" ipython-genutils==0.2.0; python_version == "3.6" ipython==7.16.1; python_version == "3.6" -ipython==7.21.0; python_version >= "3.7" +ipython==7.22.0; python_version >= "3.7" isodate==0.6.0; python_version >= "3.6" isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" jedi==0.18.0 @@ -89,7 +89,7 @@ oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or py packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" parsedatetime==2.6; python_version >= "3.6" -parso==0.8.1; python_version == "3.6" +parso==0.8.2; python_version == "3.6" pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" pexpect==4.8.0 pickleshare==0.7.5 @@ -111,6 +111,7 @@ pyjwt==1.7.1; python_version >= "3.6" pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" pylint==2.4.3; python_version >= "3.5" pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pynsist==2.7; python_version >= "3.6" pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") @@ -123,13 +124,14 @@ pytest==3.2.5 python-augeas==0.5.0 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" python-digitalocean==1.16.0; python_version >= "3.6" -python-dotenv==0.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +python-dotenv==0.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0" readme-renderer==29.0; python_version >= "3.6" repoze.sphinx.autointerface==0.8; python_version >= "3.6" +requests-download==0.1.2; python_version >= "3.6" requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" @@ -166,6 +168,7 @@ wcwidth==0.2.5; python_version == "3.6" webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") +yarg==0.1.9; python_version >= "3.6" zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" diff --git a/tools/venv.py b/tools/venv.py index 9f7488008..2e0607e70 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -46,6 +46,9 @@ REQUIREMENTS = [ '-e certbot-ci', ] +if sys.platform == 'win32': + REQUIREMENTS.append('-e windows-installer') + VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') diff --git a/windows-installer/certbot.ico b/windows-installer/assets/certbot.ico similarity index 100% rename from windows-installer/certbot.ico rename to windows-installer/assets/certbot.ico diff --git a/windows-installer/renew-down.ps1 b/windows-installer/assets/renew-down.ps1 similarity index 100% rename from windows-installer/renew-down.ps1 rename to windows-installer/assets/renew-down.ps1 diff --git a/windows-installer/renew-up.ps1 b/windows-installer/assets/renew-up.ps1 similarity index 100% rename from windows-installer/renew-up.ps1 rename to windows-installer/assets/renew-up.ps1 diff --git a/windows-installer/run.bat b/windows-installer/assets/run.bat similarity index 100% rename from windows-installer/run.bat rename to windows-installer/assets/run.bat diff --git a/windows-installer/template.nsi b/windows-installer/assets/template.nsi similarity index 100% rename from windows-installer/template.nsi rename to windows-installer/assets/template.nsi diff --git a/windows-installer/setup.py b/windows-installer/setup.py new file mode 100644 index 000000000..01d75a99b --- /dev/null +++ b/windows-installer/setup.py @@ -0,0 +1,39 @@ +from setuptools import find_packages +from setuptools import setup + +version = '1.0' + +setup( + name='windows-installer', + version=version, + description='Environment to build the Certbot Windows installer', + url='https://github.com/letsencrypt/letsencrypt', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Operating System :: Microsoft :: Windows', + 'Topic :: Software Development :: Build Tools', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + 'pynsist==2.7' + ], + entry_points={ + 'console_scripts': [ + 'construct-windows-installer = windows_installer.construct:main', + ], + }, +) diff --git a/windows-installer/windows_installer/__init__.py b/windows-installer/windows_installer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/windows-installer/construct.py b/windows-installer/windows_installer/construct.py similarity index 87% rename from windows-installer/construct.py rename to windows-installer/windows_installer/construct.py index 21ee4c8be..0cec3811b 100644 --- a/windows-installer/construct.py +++ b/windows-installer/windows_installer/construct.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import contextlib import ctypes import os import re @@ -7,7 +6,6 @@ import shutil import struct import subprocess import sys -import tempfile import time PYTHON_VERSION = (3, 8, 8) @@ -16,6 +14,21 @@ NSIS_VERSION = '3.06.1' def main(): + if os.name != 'nt': + raise RuntimeError('This script must be run under Windows.') + + if ctypes.windll.shell32.IsUserAnAdmin() == 0: + # Administrator privileges are required to properly install NSIS through Chocolatey + raise RuntimeError('This script must be run with administrator privileges.') + + if sys.version_info[:2] != PYTHON_VERSION[:2]: + raise RuntimeError('This script must be run with Python {0}' + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) + + if struct.calcsize('P') * 8 != PYTHON_BITNESS: + raise RuntimeError('This script must be run with a {0} bit version of Python.' + .format(PYTHON_BITNESS)) + build_path, repo_path, venv_path, venv_python = _prepare_environment() _copy_assets(build_path, repo_path) @@ -24,14 +37,14 @@ def main(): _prepare_build_tools(venv_path, venv_python, repo_path) _compile_wheels(repo_path, build_path, venv_python) - _build_installer(installer_cfg_path, venv_path) + _build_installer(installer_cfg_path) print('Done') -def _build_installer(installer_cfg_path, venv_path): +def _build_installer(installer_cfg_path): print('Build the installer') - subprocess.check_call([os.path.join(venv_path, 'Scripts', 'pynsist.exe'), installer_cfg_path]) + subprocess.check_call([sys.executable, '-m', 'nsist', installer_cfg_path]) def _compile_wheels(repo_path, build_path, venv_python): @@ -72,7 +85,6 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') subprocess.check_call([sys.executable, '-m', 'venv', venv_path]) subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')]) - subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist']) subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION]) @@ -81,11 +93,11 @@ def _copy_assets(build_path, repo_path): if os.path.exists(build_path): os.rename(build_path, '{0}.{1}.bak'.format(build_path, int(time.time()))) os.makedirs(build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'certbot.ico'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'run.bat'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'template.nsi'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-up.ps1'), build_path) - shutil.copy(os.path.join(repo_path, 'windows-installer', 'renew-down.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'certbot.ico'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'run.bat'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'template.nsi'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-up.ps1'), build_path) + shutil.copy(os.path.join(repo_path, 'windows-installer', 'assets', 'renew-down.ps1'), build_path) def _generate_pynsist_config(repo_path, build_path): @@ -141,7 +153,7 @@ def _prepare_environment(): raise RuntimeError('Error: Chocolatey (https://chocolatey.org/) needs ' 'to be installed to run this script.') script_path = os.path.realpath(__file__) - repo_path = os.path.dirname(os.path.dirname(script_path)) + repo_path = os.path.dirname(os.path.dirname(os.path.dirname(script_path))) build_path = os.path.join(repo_path, 'windows-installer', 'build') venv_path = os.path.join(build_path, 'venv-config') venv_python = os.path.join(venv_path, 'Scripts', 'python.exe') @@ -150,18 +162,4 @@ def _prepare_environment(): if __name__ == '__main__': - if os.name != 'nt': - raise RuntimeError('This script must be run under Windows.') - - if ctypes.windll.shell32.IsUserAnAdmin() == 0: - # Administrator privileges are required to properly install NSIS through Chocolatey - raise RuntimeError('This script must be run with administrator privileges.') - - if sys.version_info[:2] != PYTHON_VERSION[:2]: - raise RuntimeError('This script must be run with Python {0}' - .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) - - if struct.calcsize('P') * 8 != PYTHON_BITNESS: - raise RuntimeError('This script must be run with a {0} bit version of Python.' - .format(PYTHON_BITNESS)) main() From 8a84c88fee4cecec14e63658e283a3e6834a2864 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 2 Apr 2021 19:37:19 +0200 Subject: [PATCH 50/99] Remove wheel hack in windows installer construction script (#8752) In #8649 we added some code to trick pynsist and make it understand that `abi3` wheels for Windows are forward compatible, meaning that the cryptography wheel tagged `cp36-abi3` is in fact compatible with Python 3.6+, and not only Python 3.6. Since pynsist 2.7 the tool now understand `abi3` wheels properly, and this trick is not needed anymore. Please note that despite modifying the pynsist pinning in `dev_constraints.txt`, it will have no effect since pynsist currently escape the pinning system. This is handled in https://github.com/certbot/certbot/pull/8749. --- tools/dev_constraints.txt | 4 ++-- windows-installer/windows_installer/construct.py | 16 ---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 43a27a3d0..2cc2039ea 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -83,10 +83,10 @@ PyGithub==1.52 Pygments==2.2.0 pyjwt==1.7.1 pylint==2.4.3 +pynacl==1.3.0 # If pynsist version is upgraded, our NSIS template windows-installer/template.nsi # must be upgraded if necessary using the new built-in one from pynsist. -pynacl==1.3.0 -pynsist==2.6 +pynsist==2.7 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 diff --git a/windows-installer/windows_installer/construct.py b/windows-installer/windows_installer/construct.py index 0cec3811b..983923049 100644 --- a/windows-installer/windows_installer/construct.py +++ b/windows-installer/windows_installer/construct.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import ctypes import os -import re import shutil import struct import subprocess @@ -65,21 +64,6 @@ def _compile_wheels(repo_path, build_path, venv_python): command.extend(wheels_project) subprocess.check_call(command, env=env) - # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where - # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible - # with Python 3.6+). While technically valid to describe a wheel compliant with the Stable - # Application Binary Interface, this naming convention makes pynsist falsely think that the - # wheel is compatible with Python 3.6 only. - # Let's trick pynsist by renaming the wheel until this is fixed upstream. - for file in os.listdir(wheels_path): - # Given that our Python version is 3.8, this rename files like - # cryptography-VERSION-cpXX-abi3-win32.whl into cryptography-VERSION-cp38-abi3-win32.whl - renamed = re.sub(r'^(.*)-cp\d+-abi3-(\w+)\.whl$', r'\1-cp{0}{1}-abi3-\2.whl' - .format(PYTHON_VERSION[0], PYTHON_VERSION[1]), file) - print(renamed) - if renamed != file: - os.replace(os.path.join(wheels_path, file), os.path.join(wheels_path, renamed)) - def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') From 28fac893f4929e9a49f23291c42f74b1aef5665c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 2 Apr 2021 10:37:40 -0700 Subject: [PATCH 51/99] add and update pynsist template comments (#8759) --- windows-installer/assets/template.nsi | 4 ++-- windows-installer/setup.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/windows-installer/assets/template.nsi b/windows-installer/assets/template.nsi index 64bceb065..566e1b004 100644 --- a/windows-installer/assets/template.nsi +++ b/windows-installer/assets/template.nsi @@ -1,7 +1,7 @@ -; This NSIS template is based on the built-in one in pynsist 2.6. +; This NSIS template is based on the built-in one in pynsist 2.7. ; Added lines are enclosed within "CERTBOT CUSTOM BEGIN/END" comments. ; If pynsist is upgraded, this template must be updated if necessary using the new built-in one. -; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.6/nsist/pyapp.nsi +; Original file can be found here: https://github.com/takluyver/pynsist/blob/2.7/nsist/pyapp.nsi !define PRODUCT_NAME "[[ib.appname]]" !define PRODUCT_VERSION "[[ib.version]]" diff --git a/windows-installer/setup.py b/windows-installer/setup.py index 01d75a99b..d5b4d2bde 100644 --- a/windows-installer/setup.py +++ b/windows-installer/setup.py @@ -29,6 +29,12 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ + # pynsist is pinned to an exact version so we can update + # assets/template.nsi as needed. The file is based on the default + # pynsist NSIS template and pynsist's documentation warns that custom + # templates may need to be updated for them to work with new versions + # of pynsist. See + # https://pynsist.readthedocs.io/en/latest/cfgfile.html#build-section. 'pynsist==2.7' ], entry_points={ From 584a1a3ecee8c48721167a5d4e25d8726167162b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 2 Apr 2021 10:37:48 -0700 Subject: [PATCH 52/99] simplify setup.py (#8760) I recently noticed that we only support versions of `setuptools` that support environment markers which allows us to simplify our `setup.py` files a bit. --- certbot-ci/setup.py | 21 ++++++++++----------- certbot/setup.py | 29 ++++++++++------------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index 7f89cc934..ad7672e17 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -7,6 +7,13 @@ from setuptools import setup version = '0.32.0.dev0' +# setuptools 36.2+ is needed for support for environment markers +min_setuptools_version='36.2' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + install_requires = [ 'coverage', 'cryptography', @@ -18,21 +25,13 @@ install_requires = [ # "workerinput". See https://github.com/pytest-dev/pytest-xdist/pull/268. 'pytest-xdist>=1.22.1', 'python-dateutil', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', 'pyyaml', 'requests', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -if LooseVersion(setuptools_version) >= LooseVersion('36.2'): - install_requires.append("pywin32>=224 ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') - setup( name='certbot-ci', version=version, diff --git a/certbot/setup.py b/certbot/setup.py index 0f54179a3..8843a35a2 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -8,6 +8,12 @@ from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup +min_setuptools_version='39.0.1' +# This conditional isn't necessary, but it provides better error messages to +# people who try to install this package with older versions of setuptools. +if LooseVersion(setuptools_version) < LooseVersion(min_setuptools_version): + raise RuntimeError(f'setuptools {min_setuptools_version}+ is required') + # Workaround for https://bugs.python.org/issue8876, see # https://bugs.python.org/issue8876#msg208792 # This can be removed when using Python 2.7.9 or later: @@ -49,29 +55,14 @@ install_requires = [ 'parsedatetime>=2.4', 'pyrfc3339', 'pytz', - 'setuptools>=39.0.1', + # This dependency needs to be added using environment markers to avoid its + # installation on Linux. + 'pywin32>=300 ; sys_platform == "win32"', + f'setuptools>={min_setuptools_version}', 'zope.component', 'zope.interface', ] -# Add pywin32 on Windows platforms to handle low-level system calls. -# This dependency needs to be added using environment markers to avoid its installation on Linux. -# However environment markers are supported only with setuptools >= 36.2. -# So this dependency is not added for old Linux distributions with old setuptools, -# in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=300' -setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) -if setuptools_known_environment_markers: - install_requires.append(pywin32_req + " ; sys_platform == 'win32'") -elif 'bdist_wheel' in sys.argv[1:]: - raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' - 'of setuptools. Version 36.2+ of setuptools is required.') -elif os.name == 'nt': - # This branch exists to improve this package's behavior on Windows. Without - # it, if the sdist is installed on Windows with an old version of - # setuptools, pywin32 will not be specified as a dependency. - install_requires.append(pywin32_req) - dev_extras = [ 'astroid', 'azure-devops', From 06a53cb7df4a9551bf1d087bb221089d596b357a Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 2 Apr 2021 20:54:40 +0200 Subject: [PATCH 53/99] Upgrade to mypy 0.812 (#8748) Fixes #8425 This PR upgrades mypy to the latest version available, 0.812. Given the advanced type inference capabilities provided by this newer version, this PRs also fixes various type inconsistencies that are now detected. Here are the non obvious changes done to fix types: * typing in mixins has been solved using `Protocol` classes, as recommended by mypy (https://mypy.readthedocs.io/en/latest/more_types.html#mixin-classes, https://mypy.readthedocs.io/en/stable/protocols.html) * `cast` when we are playing with `Union` types This PR also disables the strict optional checks that have been enable by default in recent versions of mypy. Once this PR is merged, I will create an issue to study how these checks can be enabled. `typing.Protocol` is available only since Python 3.8. To keep compatibility with Python 3.6, I try to import the class `Protocol` from `typing`, and fallback to assign `object` to `Protocol` if that fails. This way the code is working with all versions of Python, but the mypy check can be run only with Python 3.8+ because it needs the protocol feature. As a consequence, tox runs mypy under Python 3.8. Alternatives are: * importing `typing_extensions`, that proposes backport of newest typing features to Python 3.6, but this implies to add a dependency to Certbot just to run mypy * redesign the concerned classes to not use mixins, or use them differently, but this implies to modify the code itself even if there is nothing wrong with it and it is just a matter of instructing mypy to understand in which context the mixins can be used * ignoring type for these classes with `# type: ignore` but we loose the benefit of mypy for them * Upgrade mypy * First step for acme * Cast for the rescue * Fixing types for certbot * Fix typing for certbot-nginx * Finalize type fixes, configure no optional strict check for mypy in tox * Align requirements * Isort * Pylint * Protocol for python 3.6 * Use Python 3.9 for mypy, make code compatible with Python 3.8< * Pylint and mypy * Pragma no cover * Pythonic NotImplemented constant * More type definitions * Add comments * Simplify typing logic * Use vararg tuple * Relax constraints on mypy * Add more type * Do not silence error if target is not defined * Conditionally import Protocol for type checking only * Clean up imports * Add comments * Align python version linting with mypy and coverage * Just ignore types in an unused module * Add comments * Fix lint --- .../templates/jobs/standard-tests-jobs.yml | 8 +- acme/acme/challenges.py | 5 +- acme/acme/client.py | 31 ++++--- acme/acme/jws.py | 4 +- acme/acme/messages.py | 25 ++++-- acme/acme/standalone.py | 20 ++++- .../certbot_apache/_internal/apacheparser.py | 5 +- .../certbot_apache/_internal/configurator.py | 7 +- .../_internal/override_centos.py | 3 +- .../_internal/override_fedora.py | 9 +- .../_internal/override_gentoo.py | 5 +- .../certbot_tests/test_main.py | 7 +- .../utils/certbot_call.py | 2 +- .../utils/pebble_ocsp_server.py | 5 +- certbot-nginx/certbot_nginx/_internal/obj.py | 1 - .../certbot_nginx/_internal/parser.py | 3 + .../certbot_nginx/_internal/parser_obj.py | 8 +- certbot/certbot/_internal/cli/__init__.py | 1 - certbot/certbot/_internal/cli/helpful.py | 2 + certbot/certbot/_internal/client.py | 3 +- certbot/certbot/_internal/log.py | 7 +- certbot/certbot/_internal/reporter.py | 2 +- certbot/certbot/_internal/storage.py | 6 +- certbot/certbot/achallenges.py | 5 +- certbot/certbot/compat/_path.py | 6 +- certbot/certbot/compat/os.py | 31 ++++--- certbot/certbot/interfaces.py | 2 +- certbot/certbot/plugins/dns_common.py | 3 + certbot/certbot/plugins/dns_test_common.py | 44 ++++++++-- .../plugins/dns_test_common_lexicon.py | 85 ++++++++++++++----- certbot/certbot/reverter.py | 1 - tools/dev_constraints.txt | 3 +- tools/pinning/pyproject.toml | 2 - tools/requirements.txt | 5 +- tox.ini | 3 +- 35 files changed, 248 insertions(+), 111 deletions(-) diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index fec11c6c5..62f22b223 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -40,13 +40,13 @@ jobs: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.9 TOXENV: py39-cover - linux-py37-lint: + linux-py39-lint: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.7 + PYTHON_VERSION: 3.9 TOXENV: lint - linux-py36-mypy: + linux-py39-mypy: IMAGE_NAME: ubuntu-18.04 - PYTHON_VERSION: 3.6 + PYTHON_VERSION: 3.9 TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-18.04 diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 82f7ae1b8..4fad9a421 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -5,6 +5,7 @@ import functools import hashlib import logging import socket +from typing import Type from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose @@ -152,8 +153,8 @@ class KeyAuthorizationChallenge(_TokenChallenge, metaclass=abc.ABCMeta): that will be used to generate ``response``. :param str typ: type of the challenge """ - typ = NotImplemented - response_cls = NotImplemented + typ: str = NotImplemented + response_cls: Type[KeyAuthorizationChallengeResponse] = NotImplemented thumbprint_hash_function = ( KeyAuthorizationChallengeResponse.thumbprint_hash_function) diff --git a/acme/acme/client.py b/acme/acme/client.py index f5aa1ff9e..ae4a64b23 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -8,10 +8,12 @@ import http.client as http_client import logging import re import time +from typing import cast from typing import Dict from typing import List from typing import Set from typing import Text +from typing import Union import josepy as jose import OpenSSL @@ -818,6 +820,7 @@ class BackwardsCompatibleClientV2: def __init__(self, net, key, server): directory = messages.Directory.from_json(net.get(server).json()) self.acme_version = self._acme_version_from_directory(directory) + self.client: Union[Client, ClientV2] if self.acme_version == 1: self.client = Client(directory, key=key, net=net) else: @@ -837,16 +840,18 @@ class BackwardsCompatibleClientV2: if check_tos_cb is not None: check_tos_cb(tos) if self.acme_version == 1: - regr = self.client.register(regr) + client_v1 = cast(Client, self.client) + regr = client_v1.register(regr) if regr.terms_of_service is not None: _assess_tos(regr.terms_of_service) - return self.client.agree_to_tos(regr) + return client_v1.agree_to_tos(regr) return regr else: - if "terms_of_service" in self.client.directory.meta: - _assess_tos(self.client.directory.meta.terms_of_service) + client_v2 = cast(ClientV2, self.client) + if "terms_of_service" in client_v2.directory.meta: + _assess_tos(client_v2.directory.meta.terms_of_service) regr = regr.update(terms_of_service_agreed=True) - return self.client.new_account(regr) + return client_v2.new_account(regr) def new_order(self, csr_pem): """Request a new Order object from the server. @@ -864,14 +869,15 @@ class BackwardsCompatibleClientV2: """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem) # pylint: disable=protected-access dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr) authorizations = [] for domain in dnsNames: - authorizations.append(self.client.request_domain_challenges(domain)) + authorizations.append(client_v1.request_domain_challenges(domain)) return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem) - return self.client.new_order(csr_pem) + return cast(ClientV2, self.client).new_order(csr_pem) def finalize_order(self, orderr, deadline, fetch_alternative_chains=False): """Finalize an order and obtain a certificate. @@ -886,8 +892,9 @@ class BackwardsCompatibleClientV2: """ if self.acme_version == 1: + client_v1 = cast(Client, self.client) csr_pem = orderr.csr_pem - certr = self.client.request_issuance( + certr = client_v1.request_issuance( jose.ComparableX509( OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)), orderr.authorizations) @@ -895,7 +902,7 @@ class BackwardsCompatibleClientV2: chain = None while datetime.datetime.now() < deadline: try: - chain = self.client.fetch_chain(certr) + chain = client_v1.fetch_chain(certr) break except errors.Error: time.sleep(1) @@ -910,7 +917,8 @@ class BackwardsCompatibleClientV2: chain = crypto_util.dump_pyopenssl_chain(chain).decode() return orderr.update(fullchain_pem=(cert + chain)) - return self.client.finalize_order(orderr, deadline, fetch_alternative_chains) + return cast(ClientV2, self.client).finalize_order( + orderr, deadline, fetch_alternative_chains) def revoke(self, cert, rsn): """Revoke certificate. @@ -936,7 +944,7 @@ class BackwardsCompatibleClientV2: Always return False for ACMEv1 servers, as it doesn't use External Account Binding.""" if self.acme_version == 1: return False - return self.client.external_account_required() + return cast(ClientV2, self.client).external_account_required() class ClientNetwork: @@ -1129,6 +1137,7 @@ class ClientNetwork: # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. + debug_content: Union[bytes, str] if response.headers.get("Content-Type") == DER_CONTENT_TYPE: debug_content = base64.b64encode(response.content) else: diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 2188c3727..7eccf0fdf 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -14,7 +14,9 @@ class Header(jose.Header): kid = jose.Field('kid', omitempty=True) url = jose.Field('url', omitempty=True) - @nonce.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that nonce is redefined. Let's ignore the type check here. + @nonce.decoder # type: ignore def nonce(value): # pylint: disable=no-self-argument,missing-function-docstring try: return jose.decode_b64jose(value) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 5e7e22c34..c8b971b8f 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,6 +1,9 @@ """ACME protocol messages.""" -import json from collections.abc import Hashable +import json +from typing import Any +from typing import Dict +from typing import Type import josepy as jose @@ -87,7 +90,9 @@ class Error(jose.JSONObjectWithFields, errors.Error): raise ValueError("The supplied code: %s is not a known ACME error" " code" % code) typ = ERROR_PREFIX + code - return cls(typ=typ, **kwargs) + # Mypy will not understand that the Error constructor accepts a named argument + # "typ" because of josepy magic. Let's ignore the type check here. + return cls(typ=typ, **kwargs) # type: ignore @property def description(self): @@ -124,7 +129,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) - POSSIBLE_NAMES = NotImplemented + POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented def __init__(self, name): super(_Constant, self).__init__() @@ -166,7 +171,7 @@ STATUS_DEACTIVATED = Status('deactivated') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES: dict = {} + POSSIBLE_NAMES: Dict[str, 'IdentifierType'] = {} IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -184,7 +189,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES: dict = {} + _REGISTERED_TYPES: Dict[str, Type[Any]] = {} class Meta(jose.JSONObjectWithFields): """Directory Meta.""" @@ -218,7 +223,7 @@ class Directory(jose.JSONDeSerializable): return getattr(key, 'resource_type', key) @classmethod - def register(cls, resource_body_cls): + def register(cls, resource_body_cls: Type[Any]) -> Type[Any]: """Register resource.""" resource_type = resource_body_cls.resource_type assert resource_type not in cls._REGISTERED_TYPES @@ -528,7 +533,9 @@ class Authorization(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) wildcard = jose.Field('wildcard', omitempty=True) - @challenges.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that challenge is redefined. Let's ignore the type check here. + @challenges.decoder # type: ignore def challenges(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(ChallengeBody.from_json(chall) for chall in value) @@ -627,7 +634,9 @@ class Order(ResourceBody): expires = fields.RFC3339Field('expires', omitempty=True) error = jose.Field('error', omitempty=True, decoder=Error.from_json) - @identifiers.decoder + # Mypy does not understand the josepy magic happening here, and falsely claims + # that identifiers is redefined. Let's ignore the type check here. + @identifiers.decoder # type: ignore def identifiers(value): # pylint: disable=no-self-argument,missing-function-docstring return tuple(Identifier.from_json(identifier) for identifier in value) diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 0b5a8b5c7..eda45304c 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -64,7 +64,7 @@ class BaseDualNetworkedServers: def __init__(self, ServerClass, server_address, *remaining_args, **kwargs): port = server_address[1] self.threads: List[threading.Thread] = [] - self.servers: List[ACMEServerMixin] = [] + self.servers: List[socketserver.BaseServer] = [] # Must try True first. # Ubuntu, for example, will fail to bind to IPv4 if we've already bound @@ -203,8 +203,24 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) - self.timeout = kwargs.pop('timeout', 30) + self._timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + self.server: HTTP01Server + + # In parent class BaseHTTPRequestHandler, 'timeout' is a class-level property but we + # need to define its value during the initialization phase in HTTP01RequestHandler. + # However MyPy does not appreciate that we dynamically shadow a class-level property + # with an instance-level property (eg. self.timeout = ... in __init__()). So to make + # everyone happy, we statically redefine 'timeout' as a method property, and set the + # timeout value in a new internal instance-level property _timeout. + @property + def timeout(self): + """ + The default timeout this server should apply to requests. + :return: timeout to apply + :rtype: int + """ + return self._timeout def log_message(self, format, *args): # pylint: disable=redefined-builtin """Log arbitrary message.""" diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index c7b723ae6..adfbc4442 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -1,4 +1,5 @@ """ apacheconfig implementation of the ParserNode interfaces """ +from typing import Tuple from certbot_apache._internal import assertions from certbot_apache._internal import interfaces @@ -21,7 +22,7 @@ class ApacheParserNode(interfaces.ParserNode): self.metadata = metadata self._raw = self.metadata["ac_ast"] - def save(self, msg): # pragma: no cover + def save(self, msg): # pragma: no cover pass def find_ancestors(self, name): # pylint: disable=unused-variable @@ -83,7 +84,7 @@ class ApacheBlockNode(ApacheDirectiveNode): def __init__(self, **kwargs): super(ApacheBlockNode, self).__init__(**kwargs) - self.children = () + self.children: Tuple[ApacheParserNode, ...] = () def __eq__(self, other): # pragma: no cover if isinstance(other, self.__class__): diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 29648d4c1..c24a646db 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -8,6 +8,7 @@ import logging import re import socket import time +from typing import cast from typing import DefaultDict from typing import Dict from typing import List @@ -156,9 +157,9 @@ class ApacheConfigurator(common.Installer): self.options[o] = self.OS_DEFAULTS[o] # Special cases - self.options["version_cmd"][0] = self.option("ctl") - self.options["restart_cmd"][0] = self.option("ctl") - self.options["conftest_cmd"][0] = self.option("ctl") + cast(List[str], self.options["version_cmd"])[0] = self.option("ctl") + cast(List[str], self.options["restart_cmd"])[0] = self.option("ctl") + cast(List[str], self.options["conftest_cmd"])[0] = self.option("ctl") @classmethod def add_parser_arguments(cls, add): diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 54edfc911..fc77aaafc 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,5 +1,6 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging +from typing import cast from typing import List import zope.interface @@ -76,7 +77,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): alternative restart cmd used in CentOS. """ super(CentOSConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") def get_parser(self): """Initializes the ApacheParser""" diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 9b521846c..0dc3df66b 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -1,4 +1,7 @@ """ Distribution specific override class for Fedora 29+ """ +from typing import cast +from typing import List + import zope.interface from certbot import errors @@ -69,9 +72,9 @@ class FedoraConfigurator(configurator.ApacheConfigurator): of Fedora to restart httpd. """ super(FedoraConfigurator, self)._prepare_options() - self.options["restart_cmd"][0] = 'apachectl' - self.options["restart_cmd_alt"][0] = 'apachectl' - self.options["conftest_cmd"][0] = 'apachectl' + cast(List[str], self.options["restart_cmd"])[0] = 'apachectl' + cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl' + cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl' class FedoraParser(parser.ApacheParser): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index 59a3d981f..773fdd568 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -1,4 +1,7 @@ """ Distribution specific override class for Gentoo Linux """ +from typing import cast +from typing import List + import zope.interface from certbot import interfaces @@ -36,7 +39,7 @@ class GentooConfigurator(configurator.ApacheConfigurator): alternative restart cmd used in Gentoo. """ super(GentooConfigurator, self)._prepare_options() - self.options["restart_cmd_alt"][0] = self.option("ctl") + cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") def get_parser(self): """Initializes the ApacheParser""" diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 0b649acdc..c9fa5ff65 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -8,19 +8,20 @@ import shutil import subprocess import time -from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1, SECP384R1, SECP521R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP256R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP384R1 +from cryptography.hazmat.primitives.asymmetric.ec import SECP521R1 from cryptography.x509 import NameOID - import pytest from certbot_integration_tests.certbot_tests import context as certbot_context from certbot_integration_tests.certbot_tests.assertions import assert_cert_count_for_lineage from certbot_integration_tests.certbot_tests.assertions import assert_elliptic_key -from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_owner from certbot_integration_tests.certbot_tests.assertions import assert_equals_group_permissions from certbot_integration_tests.certbot_tests.assertions import assert_equals_world_read_permissions from certbot_integration_tests.certbot_tests.assertions import assert_hook_execution +from certbot_integration_tests.certbot_tests.assertions import assert_rsa_key from certbot_integration_tests.certbot_tests.assertions import assert_saved_renew_hook from certbot_integration_tests.certbot_tests.assertions import assert_world_no_permissions from certbot_integration_tests.certbot_tests.assertions import assert_world_read_permissions diff --git a/certbot-ci/certbot_integration_tests/utils/certbot_call.py b/certbot-ci/certbot_integration_tests/utils/certbot_call.py index b319eca4c..965ab6881 100755 --- a/certbot-ci/certbot_integration_tests/utils/certbot_call.py +++ b/certbot-ci/certbot_integration_tests/utils/certbot_call.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """Module to call certbot in test mode""" +from distutils.version import LooseVersion import os import subprocess import sys -from distutils.version import LooseVersion import certbot_integration_tests # pylint: disable=wildcard-import,unused-wildcard-import diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py index 4db72998f..3458929ad 100755 --- a/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py @@ -29,10 +29,7 @@ class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False) issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend()) - try: - content_len = int(self.headers.getheader('content-length', 0)) - except AttributeError: - content_len = int(self.headers.get('Content-Length')) + content_len = int(self.headers.get('Content-Length')) ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len)) response = requests.get('{0}/cert-status-by-serial/{1}'.format( diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index 1511cba6d..f77e535ce 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -1,7 +1,6 @@ """Module contains classes used by the Nginx Configurator.""" import re - from certbot.plugins import common ADD_HEADER_DIRECTIVE = 'add_header' diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index 8b353c095..28833b1f7 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -18,6 +18,7 @@ from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser from certbot_nginx._internal import obj +from certbot_nginx._internal.nginxparser import UnspacedList logger = logging.getLogger(__name__) @@ -243,6 +244,8 @@ class NginxParser: tree = self.parsed[filename] if ext: filename = filename + os.path.extsep + ext + if not isinstance(tree, UnspacedList): + raise ValueError(f"Error tree {tree} is not an UnspacedList") try: if lazy and not tree.is_dirty(): continue diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 37392ea80..838a2e382 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -1,3 +1,5 @@ +# type: ignore +# This module is not used for now, so we just skip type check for the sake of simplicity. """ This file contains parsing routines and object classes to help derive meaning from raw lists of tokens from pyparsing. """ @@ -204,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all(isinstance(elem, str) for elem in lists) + all(isinstance(elem, str) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -212,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any(not isinstance(elem, str) for elem in raw_list): + any(not isinstance(elem, str) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list @@ -283,7 +285,7 @@ class Block(Parsable): :returns: whether this lists is parseable by `Block`. """ return isinstance(lists, list) and len(lists) == 2 and \ - Sentence.should_parse(lists[0]) and isinstance(lists[1], list) + Sentence.should_parse(lists[0]) and isinstance(lists[1], list) def set_tabs(self, tabs=" "): """ Sets tabs by setting equivalent tabbing on names, then adding tabbing diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index d0982f989..d835d0f13 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -44,7 +44,6 @@ from certbot._internal.plugins import disco as plugins_disco import certbot._internal.plugins.selection as plugin_selection import certbot.plugins.enhancements as enhancements - logger = logging.getLogger(__name__) diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index f185bdc26..73872584c 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -116,6 +116,8 @@ class HelpfulArgumentParser: # This is the only way to turn off overly verbose config flag documentation self.parser._add_config_file_help = False + self.verb: str + # Help that are synonyms for --help subcommands COMMANDS_TOPICS = ["command", "commands", "subcommand", "subcommands", "verbs"] diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index ba696bf85..796f05599 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -2,7 +2,6 @@ import datetime import logging import platform -from typing import List from typing import Optional from cryptography.hazmat.backends import default_backend @@ -391,7 +390,7 @@ class Client: return cert, chain, key, csr def _get_order_and_authorizations(self, csr_pem: str, - best_effort: bool) -> List[messages.OrderResource]: + best_effort: bool) -> messages.OrderResource: """Request a new order and complete its authorizations. :param str csr_pem: A CSR in PEM format. diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index d8a55e27d..fb0a996f7 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -109,11 +109,12 @@ def post_arg_parse_setup(config): root_logger.addHandler(file_handler) root_logger.removeHandler(memory_handler) - temp_handler = memory_handler.target # pylint: disable=no-member + temp_handler = getattr(memory_handler, 'target', None) memory_handler.setTarget(file_handler) # pylint: disable=no-member memory_handler.flush(force=True) # pylint: disable=unexpected-keyword-arg memory_handler.close() - temp_handler.close() + if temp_handler: + temp_handler.close() if config.quiet: level = constants.QUIET_LOGGING_LEVEL @@ -205,7 +206,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. - target = self.target + target = getattr(self, 'target') super(MemoryHandler, self).close() self.target = target diff --git a/certbot/certbot/_internal/reporter.py b/certbot/certbot/_internal/reporter.py index 0268fbf34..295a2d4c5 100644 --- a/certbot/certbot/_internal/reporter.py +++ b/certbot/certbot/_internal/reporter.py @@ -29,7 +29,7 @@ class Reporter: LOW_PRIORITY = 2 """Low priority constant. See `add_message`.""" - _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash') + _msg_type = collections.namedtuple('_msg_type', 'priority text on_crash') def __init__(self, config): self.messages: queue.PriorityQueue[Reporter._msg_type] = queue.PriorityQueue() diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 218a5dd92..11dae33e9 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -5,14 +5,14 @@ import logging import re import shutil import stat - from typing import Optional + import configobj -import parsedatetime -import pytz from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import load_pem_private_key +import parsedatetime +import pytz import certbot from certbot import crypto_util diff --git a/certbot/certbot/achallenges.py b/certbot/certbot/achallenges.py index 7171c271c..a3ddbdcd3 100644 --- a/certbot/certbot/achallenges.py +++ b/certbot/certbot/achallenges.py @@ -18,15 +18,16 @@ Note, that all annotated challenges act as a proxy objects:: """ import logging +from typing import Type import josepy as jose from acme import challenges +from acme.challenges import Challenge logger = logging.getLogger(__name__) - class AnnotatedChallenge(jose.ImmutableMap): """Client annotated challenge. @@ -37,7 +38,7 @@ class AnnotatedChallenge(jose.ImmutableMap): """ __slots__ = ('challb',) - acme_type = NotImplemented + _acme_type: Type[Challenge] = NotImplemented def __getattr__(self, name): return getattr(self.challb, name) diff --git a/certbot/certbot/compat/_path.py b/certbot/certbot/compat/_path.py index 44c5e300f..a5db3a575 100644 --- a/certbot/certbot/compat/_path.py +++ b/certbot/certbot/compat/_path.py @@ -3,6 +3,10 @@ This compat module wraps os.path to forbid some functions. isort:skip_file """ + +# NB: Each function defined in compat._path is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os.path). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -29,7 +33,7 @@ del ourselves, std_os_path, std_sys # Function os.path.realpath is broken on some versions of Python for Windows. -def realpath(*unused_args, **unused_kwargs): +def realpath(*unused_args, **unused_kwargs): # type: ignore """Method os.path.realpath() is forbidden""" raise RuntimeError('Usage of os.path.realpath() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/compat/os.py b/certbot/certbot/compat/os.py index 2f0899bd7..653877b50 100644 --- a/certbot/certbot/compat/os.py +++ b/certbot/certbot/compat/os.py @@ -6,12 +6,15 @@ This module is intended to replace standard os module throughout certbot project This module has the same API as the os module in the Python standard library except for the functions defined below. +isort:skip_file """ -# NOTE: If adding a new documented function to compat.os, ensure that it is added to the +# NB1: If adding a new documented function to compat.os, ensure that it is added to the # ':members:' list in certbot/docs/api/certbot.compat.os.rst. -# isort:skip_file +# NB2: Each function defined in compat.os is marked with "type: ignore" to avoid mypy +# to complain that a function is redefined (because we imported if first from os). + # pylint: disable=function-redefined from __future__ import absolute_import @@ -60,7 +63,7 @@ del ourselves, std_os, std_sys # Basically, it states that appropriate permissions will be set for the owner, nothing for the # group, appropriate permissions for the "Everyone" group, and all permissions to the # "Administrators" group + "System" user, as they can do everything anyway. -def chmod(*unused_args, **unused_kwargs): +def chmod(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.chmod() is forbidden. ' 'Use certbot.compat.filesystem.chmod() instead.') @@ -70,7 +73,7 @@ def chmod(*unused_args, **unused_kwargs): # this platform. In order to have a consistent behavior between Linux and Windows on Certbot files # and directories, the filesystem umask method must be used instead, since it implements umask for # Windows. -def umask(*unused_args, **unused_kwargs): +def umask(*unused_args, **unused_kwargs): # type: ignore """Method os.chmod() is forbidden""" raise RuntimeError('Usage of os.umask() is forbidden. ' 'Use certbot.compat.filesystem.umask() instead.') @@ -79,7 +82,7 @@ def umask(*unused_args, **unused_kwargs): # Because uid is not a concept on Windows, chown is useless. In fact, it is not even available # on Python for Windows. So to be consistent on both platforms for Certbot, this method is # always forbidden. -def chown(*unused_args, **unused_kwargs): +def chown(*unused_args, **unused_kwargs): # type: ignore """Method os.chown() is forbidden""" raise RuntimeError('Usage of os.chown() is forbidden.' 'Use certbot.compat.filesystem.copy_ownership_and_apply_mode() instead.') @@ -90,7 +93,7 @@ def chown(*unused_args, **unused_kwargs): # filesystem.open invokes the Windows native API `CreateFile` to ensure that permissions are # atomically set in case of file creation, or invokes filesystem.chmod to properly set the # permissions for the other cases. -def open(*unused_args, **unused_kwargs): +def open(*unused_args, **unused_kwargs): # type: ignore """Method os.open() is forbidden""" raise RuntimeError('Usage of os.open() is forbidden. ' 'Use certbot.compat.filesystem.open() instead.') @@ -98,7 +101,7 @@ def open(*unused_args, **unused_kwargs): # Very similarly to os.open, os.mkdir has the same effects on Windows and creates an unsecured # folder. So a similar mitigation to security.chmod is provided on this platform. -def mkdir(*unused_args, **unused_kwargs): +def mkdir(*unused_args, **unused_kwargs): # type: ignore """Method os.mkdir() is forbidden""" raise RuntimeError('Usage of os.mkdir() is forbidden. ' 'Use certbot.compat.filesystem.mkdir() instead.') @@ -109,7 +112,7 @@ def mkdir(*unused_args, **unused_kwargs): # that our modified os.mkdir is called on Windows, by monkey patching temporarily the mkdir method # on the original os module, executing the modified logic to correctly protect newly created # folders, then restoring original mkdir method in the os module. -def makedirs(*unused_args, **unused_kwargs): +def makedirs(*unused_args, **unused_kwargs): # type: ignore """Method os.makedirs() is forbidden""" raise RuntimeError('Usage of os.makedirs() is forbidden. ' 'Use certbot.compat.filesystem.makedirs() instead.') @@ -117,7 +120,7 @@ def makedirs(*unused_args, **unused_kwargs): # Because of the blocking strategy on file handlers on Windows, rename does not behave as expected # with POSIX systems: an exception will be raised if dst already exists. -def rename(*unused_args, **unused_kwargs): +def rename(*unused_args, **unused_kwargs): # type: ignore """Method os.rename() is forbidden""" raise RuntimeError('Usage of os.rename() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -125,7 +128,7 @@ def rename(*unused_args, **unused_kwargs): # Behavior of os.replace is consistent between Windows and Linux. However, it is not supported on # Python 2.x. So, as for os.rename, we forbid it in favor of filesystem.replace. -def replace(*unused_args, **unused_kwargs): +def replace(*unused_args, **unused_kwargs): # type: ignore """Method os.replace() is forbidden""" raise RuntimeError('Usage of os.replace() is forbidden. ' 'Use certbot.compat.filesystem.replace() instead.') @@ -133,7 +136,7 @@ def replace(*unused_args, **unused_kwargs): # Results given by os.access are inconsistent or partial on Windows, because this platform is not # following the POSIX approach. -def access(*unused_args, **unused_kwargs): +def access(*unused_args, **unused_kwargs): # type: ignore """Method os.access() is forbidden""" raise RuntimeError('Usage of os.access() is forbidden. ' 'Use certbot.compat.filesystem.check_mode() or ' @@ -142,7 +145,7 @@ def access(*unused_args, **unused_kwargs): # On Windows os.stat call result is inconsistent, with a lot of flags that are not set or # meaningless. We need to use specialized functions from the certbot.compat.filesystem module. -def stat(*unused_args, **unused_kwargs): +def stat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.stat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -151,7 +154,7 @@ def stat(*unused_args, **unused_kwargs): # Method os.fstat has the same problem than os.stat, since it is the same function, # but accepting a file descriptor instead of a path. -def fstat(*unused_args, **unused_kwargs): +def fstat(*unused_args, **unused_kwargs): # type: ignore """Method os.stat() is forbidden""" raise RuntimeError('Usage of os.fstat() is forbidden. ' 'Use certbot.compat.filesystem functions instead ' @@ -163,7 +166,7 @@ def fstat(*unused_args, **unused_kwargs): # unconditionally, which allows to use more than 259 characters, and its string # representation is prepended with "\\?\". Problem is that it does it for any path, # and will make equality comparison fail with paths that will use the simple form. -def readlink(*unused_args, **unused_kwargs): +def readlink(*unused_args, **unused_kwargs): # type: ignore """Method os.readlink() is forbidden""" raise RuntimeError('Usage of os.readlink() is forbidden. ' 'Use certbot.compat.filesystem.realpath() instead.') diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index ddbee8ddc..3830e2849 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -349,7 +349,7 @@ class IInstaller(IPlugin): """ - def rollback_checkpoints(rollback=1): + def rollback_checkpoints(rollback: int = 1): """Revert `rollback` number of configuration checkpoints. :raises .PluginError: when configuration cannot be fully reverted diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 5c0fbcba9..52328ad85 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -43,6 +43,9 @@ class DNSAuthenticator(common.Plugin): def prepare(self): # pylint: disable=missing-function-docstring pass + def more_info(self) -> str: # pylint: disable=missing-function-docstring + raise NotImplementedError() + def perform(self, achalls): # pylint: disable=missing-function-docstring self._setup_credentials() diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 1affcae4e..f6d7bdca8 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -1,22 +1,52 @@ """Base test class for DNS authenticators.""" +import typing import configobj import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from acme import challenges from certbot import achallenges from certbot.compat import filesystem +from certbot.plugins.dns_common import DNSAuthenticator from certbot.tests import acme_util from certbot.tests import util as test_util +if typing.TYPE_CHECKING: + from typing import Protocol +else: + Protocol = object # type: ignore + + + +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) +class _AuthenticatorCallableTestCase(Protocol): + """Protocol describing a TestCase able to call a real DNSAuthenticator instance.""" + auth: DNSAuthenticator + + def assertTrue(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertTrue + """ + ... + + def assertEqual(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertEqual + """ + ... + + class BaseAuthenticatorTest: """ A base test class to reduce duplication between test code for DNS Authenticator Plugins. @@ -29,13 +59,13 @@ class BaseAuthenticatorTest: achall = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) - def test_more_info(self): + def test_more_info(self: _AuthenticatorCallableTestCase): self.assertTrue(isinstance(self.auth.more_info(), str)) # pylint: disable=no-member - def test_get_chall_pref(self): + def test_get_chall_pref(self: _AuthenticatorCallableTestCase): self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) # pylint: disable=no-member - def test_parser_arguments(self): + def test_parser_arguments(self: _AuthenticatorCallableTestCase): m = mock.MagicMock() self.auth.add_parser_arguments(m) # pylint: disable=no-member diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index adeb1a268..48261d395 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,32 +1,79 @@ """Base test class for DNS authenticators built on Lexicon.""" +import typing +from unittest.mock import MagicMock import josepy as jose -try: - import mock -except ImportError: # pragma: no cover - from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException +from acme.challenges import Challenge from certbot import errors from certbot.plugins import dns_test_common +from certbot.plugins.dns_common_lexicon import LexiconClient +from certbot.plugins.dns_test_common import _AuthenticatorCallableTestCase from certbot.tests import util as test_util +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore +if typing.TYPE_CHECKING: + from typing import Protocol +else: + Protocol = object # type: ignore + + + + DOMAIN = 'example.com' KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + +class _AuthenticatorCallableLexiconTestCase(_AuthenticatorCallableTestCase, Protocol): + """ + Protocol describing a TestCase suitable to test challenges against + a mocked LexiconClient instance. + """ + mock_client: MagicMock + achall: Challenge + + +class _LexiconAwareTestCase(Protocol): + """ + Protocol describing a TestCase suitable to test a real LexiconClient instance. + """ + client: LexiconClient + provider_mock: MagicMock + + record_prefix: str + record_name: str + record_content: str + + DOMAIN_NOT_FOUND: Exception + GENERIC_ERROR: Exception + LOGIN_ERROR: Exception + UNKNOWN_LOGIN_ERROR: Exception + + def assertRaises(self, *unused_args) -> None: + """ + See + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertRaises + """ + ... + + # These classes are intended to be subclassed/mixed in, so not all members are defined. # pylint: disable=no-member class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): - def test_perform(self): + def test_perform(self: _AuthenticatorCallableLexiconTestCase): self.auth.perform([self.achall]) expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] self.assertEqual(expected, self.mock_client.mock_calls) - def test_cleanup(self): + def test_cleanup(self: _AuthenticatorCallableLexiconTestCase): self.auth._attempt_cleanup = True # _attempt_cleanup | pylint: disable=protected-access self.auth.cleanup([self.achall]) @@ -44,14 +91,14 @@ class BaseLexiconClientTest: record_name = record_prefix + "." + DOMAIN record_content = "bar" - def test_add_txt_record(self): + def test_add_txt_record(self: _LexiconAwareTestCase): self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) self.provider_mock.create_record.assert_called_with(type='TXT', name=self.record_name, content=self.record_content) - def test_add_txt_record_try_twice_to_find_domain(self): + def test_add_txt_record_try_twice_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, ''] self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) @@ -60,7 +107,7 @@ class BaseLexiconClientTest: name=self.record_name, content=self.record_content) - def test_add_txt_record_fail_to_find_domain(self): + def test_add_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND,] @@ -69,64 +116,64 @@ class BaseLexiconClientTest: self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate(self): + def test_add_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_add_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_finding_domain(self): + def test_add_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_add_txt_record_error_adding_record(self): + def test_add_txt_record_error_adding_record(self: _LexiconAwareTestCase): self.provider_mock.create_record.side_effect = self.GENERIC_ERROR self.assertRaises(errors.PluginError, self.client.add_txt_record, DOMAIN, self.record_name, self.record_content) - def test_del_txt_record(self): + def test_del_txt_record(self: _LexiconAwareTestCase): self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) self.provider_mock.delete_record.assert_called_with(type='TXT', name=self.record_name, content=self.record_content) - def test_del_txt_record_fail_to_find_domain(self): + def test_del_txt_record_fail_to_find_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, self.DOMAIN_NOT_FOUND, ] self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate(self): + def test_del_txt_record_fail_to_authenticate(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_fail_to_authenticate_with_unknown_error(self): + def test_del_txt_record_fail_to_authenticate_with_unknown_error(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_finding_domain(self): + def test_del_txt_record_error_finding_domain(self: _LexiconAwareTestCase): self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - def test_del_txt_record_error_deleting_record(self): + def test_del_txt_record_error_deleting_record(self: _LexiconAwareTestCase): self.provider_mock.delete_record.side_effect = self.GENERIC_ERROR self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) diff --git a/certbot/certbot/reverter.py b/certbot/certbot/reverter.py index e6777a7da..ebc69fcbe 100644 --- a/certbot/certbot/reverter.py +++ b/certbot/certbot/reverter.py @@ -6,7 +6,6 @@ import shutil import time import traceback - from certbot import errors from certbot import util from certbot._internal import constants diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 2cc2039ea..f6bdea58c 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -60,7 +60,7 @@ MarkupSafe==1.1.1 mccabe==0.6.1 more-itertools==5.0.0 msrest==0.6.18 -mypy==0.710 +mypy==0.812 mypy-extensions==0.4.3 ndg-httpsclient==0.3.2 oauth2client==4.0.0 @@ -120,6 +120,7 @@ traitlets==4.3.3 twine==1.11.0 typed-ast==1.4.1 typing==3.6.4 +typing-extensions==3.7.4.3 uritemplate==3.0.0 virtualenv==16.6.2 wcwidth==0.1.8 diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 9112fb054..0107733d5 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -35,8 +35,6 @@ acme = {path = "../../acme", extras = ["dev", "docs"]} windows-installer = {path = "../../windows-installer"} # Extra dependencies -# See https://github.com/certbot/certbot/issues/8425. -mypy = "0.710" # Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many # test failures so let's pin these packages back for now. coverage = "4.5.4" diff --git a/tools/requirements.txt b/tools/requirements.txt index 4c6c4825e..707cc64b9 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -83,7 +83,7 @@ mccabe==0.6.1; python_version >= "3.6" msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" msrest==0.6.21; python_version >= "3.6" mypy-extensions==0.4.3; python_version >= "3.6" -mypy==0.710 +mypy==0.812; python_version >= "3.6" oauth2client==4.1.3; python_version >= "3.6" oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" @@ -160,7 +160,8 @@ tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python tqdm==4.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" -typed-ast==1.4.2; python_version >= "3.6" and implementation_name == "cpython" and python_version < "3.8" +typed-ast==1.4.2; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" +typing-extensions==3.7.4.3; python_version >= "3.6" uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" diff --git a/tox.ini b/tox.ini index 9f63b897c..18d0d7db3 100644 --- a/tox.ini +++ b/tox.ini @@ -151,11 +151,12 @@ commands = {[base]install_packages} python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} +// TODO: Re-enable strict checks for optionals with appropriate type corrections or code redesign. [testenv:mypy] basepython = python3 commands = {[base]install_packages} - mypy {[base]source_paths} + mypy --no-strict-optional {[base]source_paths} [testenv:apacheconftest] commands = From 2622a700e0a83e0de0994c970929b624b98dad40 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 2 Apr 2021 16:19:30 -0700 Subject: [PATCH 54/99] Update a few type ignore comments (#8767) Some are no longer needed and other's comments are out of date. For the changes to the acme nonce errors, `Exception` doesn't take kwargs. The error message about this our own classes isn't super helpful: ``` In [2]: BadNonce('nonce', 'error', foo='bar') --------------------------------------------------------------------------- TypeError Traceback (most recent call last) in ----> 1 BadNonce('nonce', 'error', foo='bar') TypeError: __init__() got an unexpected keyword argument 'foo' ``` but if you try this on `Exception` which these classes inherit from, you get: ``` In [4]: Exception(foo='bar') --------------------------------------------------------------------------- TypeError Traceback (most recent call last) in ----> 1 Exception(foo='bar') TypeError: Exception() takes no keyword arguments ``` See https://github.com/python/typeshed/pull/2348 for more info. * remove outdated ignores * update locking ignore comment * don't accept kwargs --- acme/acme/errors.py | 14 ++++---------- certbot/certbot/_internal/lock.py | 14 ++++++-------- certbot/certbot/crypto_util.py | 3 +-- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 5ca5a4fa2..84b7704dc 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -28,13 +28,8 @@ class NonceError(ClientError): class BadNonce(NonceError): """Bad nonce error.""" - def __init__(self, nonce, error, *args, **kwargs): - # MyPy complains here that there is too many arguments for BaseException constructor. - # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183 - # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to - # new types definitions. So we ignore the error until the code base is fixed to match - # with MyPy>=0.740 referential. - super(BadNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, nonce, error, *args): + super(BadNonce, self).__init__(*args) self.nonce = nonce self.error = error @@ -52,9 +47,8 @@ class MissingNonce(NonceError): :ivar requests.Response ~.response: HTTP Response """ - def __init__(self, response, *args, **kwargs): - # See comment in BadNonce constructor above for an explanation of type: ignore here. - super(MissingNonce, self).__init__(*args, **kwargs) # type: ignore + def __init__(self, response, *args): + super(MissingNonce, self).__init__(*args) self.response = response def __str__(self): diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index d00302598..d0b66e5e2 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -205,10 +205,9 @@ class _WindowsLockMechanism(_BaseLockMechanism): # Under Windows, filesystem.open will raise directly an EACCES error # if the lock file is already locked. fd = filesystem.open(self._path, open_mode, 0o600) - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) # type: ignore except (IOError, OSError) as err: if fd: @@ -224,10 +223,9 @@ class _WindowsLockMechanism(_BaseLockMechanism): def release(self): """Release the lock.""" try: - # The need for this "type: ignore" was fixed in - # https://github.com/python/typeshed/pull/3607 and included in - # newer versions of mypy so it can be removed when mypy is - # upgraded. + # This "type: ignore" is currently needed because msvcrt methods + # are only defined on Windows. See + # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1) # type: ignore os.close(self._fd) diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 445618ea0..5592722dd 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -301,8 +301,7 @@ def verify_signed_payload(public_key, signature, payload, signature_hash_algorit with warnings.catch_warnings(): warnings.simplefilter("ignore") if isinstance(public_key, RSAPublicKey): - # https://github.com/python/typeshed/blob/master/third_party/2/cryptography/hazmat/primitives/asymmetric/rsa.pyi - verifier = public_key.verifier( # type: ignore + verifier = public_key.verifier( signature, PKCS1v15(), signature_hash_algorithm ) verifier.update(payload) From 69479b72771fceabc3caa93152c8b2e1a7d1ae6b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Apr 2021 10:42:18 -0700 Subject: [PATCH 55/99] use standard errno (#8768) We were originally using `socket.errno` with a `type: ignore` and a comment suggesting that this attribute needs to be included in the typeshed. This is incorrect. While it's true that [socket imports errno](https://github.com/python/cpython/blob/43682f1e39a3c61f0e8a638b887bcdcbfef766c5/Lib/socket.py#L58), it's not intended to be part of its API. https://docs.python.org/3/library/socket.html has no mention of it. Instead, we should be using the standard `errno` module and remove this `type: ignore`. --- .../certbot/_internal/plugins/standalone.py | 7 +++--- certbot/tests/plugins/standalone_test.py | 23 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index 833651346..c75ba36a9 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -1,9 +1,8 @@ """Standalone Authenticator.""" import collections +import errno import logging import socket -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi -from socket import errno as socket_errors # type: ignore from typing import DefaultDict from typing import Dict from typing import Set @@ -187,13 +186,13 @@ class Authenticator(common.Plugin): def _handle_perform_error(error): - if error.socket_error.errno == socket_errors.EACCES: + if error.socket_error.errno == errno.EACCES: raise errors.PluginError( "Could not bind TCP port {0} because you don't have " "the appropriate permissions (for example, you " "aren't running this program as " "root).".format(error.port)) - if error.socket_error.errno == socket_errors.EADDRINUSE: + if error.socket_error.errno == errno.EADDRINUSE: display = zope.component.getUtility(interfaces.IDisplay) msg = ( "Could not bind TCP port {0} because it is already in " diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 636b251de..5b3fffd9d 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -1,7 +1,6 @@ """Tests for certbot._internal.plugins.standalone.""" -# https://github.com/python/typeshed/blob/master/stdlib/2and3/socket.pyi +import errno import socket -from socket import errno as socket_errors # type: ignore import unittest from typing import Dict, Set, Tuple @@ -106,8 +105,8 @@ class AuthenticatorTest(unittest.TestCase): @test_util.patch_get_utility() def test_perform_eaddrinuse_retry(self, mock_get_utility): mock_utility = mock_get_utility() - errno = socket_errors.EADDRINUSE - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + encountered_errno = errno.EADDRINUSE + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = [error] + 2 * [mock.MagicMock()] mock_yesno = mock_utility.yesno mock_yesno.return_value = True @@ -121,8 +120,8 @@ class AuthenticatorTest(unittest.TestCase): mock_yesno = mock_utility.yesno mock_yesno.return_value = False - errno = socket_errors.EADDRINUSE - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EADDRINUSE + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) self._assert_correct_yesno_call(mock_yesno) def _assert_correct_yesno_call(self, mock_yesno): @@ -131,16 +130,16 @@ class AuthenticatorTest(unittest.TestCase): self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): - errno = socket_errors.EACCES - self.assertRaises(errors.PluginError, self._fail_perform, errno) + encountered_errno = errno.EACCES + self.assertRaises(errors.PluginError, self._fail_perform, encountered_errno) def test_perform_unexpected_socket_error(self): - errno = socket_errors.ENOTCONN + encountered_errno = errno.ENOTCONN self.assertRaises( - errors.StandaloneBindError, self._fail_perform, errno) + errors.StandaloneBindError, self._fail_perform, encountered_errno) - def _fail_perform(self, errno): - error = errors.StandaloneBindError(mock.MagicMock(errno=errno), -1) + def _fail_perform(self, encountered_errno): + error = errors.StandaloneBindError(mock.MagicMock(errno=encountered_errno), -1) self.auth.servers.run.side_effect = error self.auth.perform(self._get_achalls()) From 33f177b361bfa5938da0c726dc3ee7582faf878e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Apr 2021 11:15:09 -0700 Subject: [PATCH 56/99] Upgrade Python to 3.8.9. (#8775) Over the weekend, Python released new versions of Python 3.8 and Python 3.9 partially in response to the OpenSSL CVEs discussed at https://github.com/certbot/certbot/pull/8741#issuecomment-809644789. You can see this mentioned in their changelog at https://docs.python.org/release/3.8.9/whatsnew/changelog.html#build. This PR updates the windows installer to use that new release so all of our distribution methods that contain their own copy of OpenSSL are patched for the release tomorrow. You can see tests passing with this change at https://dev.azure.com/certbot/certbot/_build/results?buildId=3751&view=results. You can see Python 3.8.9 being downloaded instead of an older version at https://dev.azure.com/certbot/certbot/_build/results?buildId=3751&view=logs&j=ad29f110-3cce-5317-4ef2-0a692ae1dee7&t=901eeead-396c-5477-aba2-f402fdcfb885&l=1055. --- windows-installer/windows_installer/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-installer/windows_installer/construct.py b/windows-installer/windows_installer/construct.py index 983923049..91df6b714 100644 --- a/windows-installer/windows_installer/construct.py +++ b/windows-installer/windows_installer/construct.py @@ -7,7 +7,7 @@ import subprocess import sys import time -PYTHON_VERSION = (3, 8, 8) +PYTHON_VERSION = (3, 8, 9) PYTHON_BITNESS = 32 NSIS_VERSION = '3.06.1' From 0f9f902b6eb32bab609922cdcc78649d78c7b7c0 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 5 Apr 2021 20:53:57 +0200 Subject: [PATCH 57/99] Use typing-extensions to ensure certbot dev environment is compatible with Python 3.6/3.7 (#8776) Fixes #8773 I took option 2 from the issue mentionned above (importing `typing-extensions` on dev dependencies) to avoid modifying certbot runtime requirements given that what needs to be added is useful for mypy only. I did not change the Python version used to execute the linting and mypy on the standard tests, given that the tox `docker_dev` target already checks if the development environment is working for Python < 3.8. --- certbot/certbot/plugins/dns_test_common.py | 2 +- certbot/certbot/plugins/dns_test_common_lexicon.py | 2 +- certbot/setup.py | 3 +++ tools/requirements.txt | 12 ++++++------ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index f6d7bdca8..7a8df9329 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -12,7 +12,7 @@ from certbot.tests import acme_util from certbot.tests import util as test_util if typing.TYPE_CHECKING: - from typing import Protocol + from typing_extensions import Protocol else: Protocol = object # type: ignore diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 48261d395..5c6f09d20 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -18,7 +18,7 @@ try: except ImportError: # pragma: no cover from unittest import mock # type: ignore if typing.TYPE_CHECKING: - from typing import Protocol + from typing_extensions import Protocol else: Protocol = object # type: ignore diff --git a/certbot/setup.py b/certbot/setup.py index 8843a35a2..6913d8384 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -77,6 +77,9 @@ dev_extras = [ 'pytest', 'pytest-cov', 'pytest-xdist', + # typing-extensions is required to import typing.Protocol and make the mypy checks + # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7 + 'typing-extensions', 'tox', 'twine', 'wheel', diff --git a/tools/requirements.txt b/tools/requirements.txt index 707cc64b9..e5a6d3be7 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -18,8 +18,8 @@ backcall==0.2.0 bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.42; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.42; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.44; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.44; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -36,7 +36,7 @@ configobj==5.0.6; python_version >= "3.6" coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" -decorator==4.4.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.2.0" +decorator==5.0.5 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -99,7 +99,7 @@ ply==3.11; python_version >= "3.6" poetry-core==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" poetry==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" prompt-toolkit==3.0.3 -protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" @@ -124,7 +124,7 @@ pytest==3.2.5 python-augeas==0.5.0 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" python-digitalocean==1.16.0; python_version >= "3.6" -python-dotenv==0.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +python-dotenv==0.17.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") @@ -144,7 +144,7 @@ shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" snowballstemmer==2.1.0; python_version >= "3.6" soupsieve==2.2.1; python_version >= "3.6" -sphinx-rtd-theme==0.5.1; python_version >= "3.6" +sphinx-rtd-theme==0.5.2; python_version >= "3.6" sphinx==3.5.3; python_version >= "3.6" sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" From c438a397a01659912060e2e4606cb9262f40d29d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 6 Apr 2021 00:04:21 +0200 Subject: [PATCH 58/99] Enable mypy strict mode (#8766) Built on top of #8748, this PR reenables mypy strict mode and adds the appropriate corrections to pass the types checks. * Upgrade mypy * First step for acme * Cast for the rescue * Fixing types for certbot * Fix typing for certbot-nginx * Finalize type fixes, configure no optional strict check for mypy in tox * Align requirements * Isort * Pylint * Protocol for python 3.6 * Use Python 3.9 for mypy, make code compatible with Python 3.8< * Pylint and mypy * Pragma no cover * Pythonic NotImplemented constant * More type definitions * Add comments * Simplify typing logic * Use vararg tuple * Relax constraints on mypy * Add more type * Do not silence error if target is not defined * Conditionally import Protocol for type checking only * Clean up imports * Add comments * Align python version linting with mypy and coverage * Just ignore types in an unused module * Add comments * Fix lint * Work in progress * Finish type control * Isort * Fix pylint * Fix imports * Fix cli subparser * Some fixes * Coverage * Remove --no-strict-optional (obviously...) * Update certbot-apache/certbot_apache/_internal/configurator.py Co-authored-by: Brad Warren * Update certbot/certbot/_internal/display/completer.py Co-authored-by: Brad Warren * Cleanup dns_google * Improve lock controls and fix subparser * Use the expected interfaces * Fix code Co-authored-by: Brad Warren --- .../certbot_apache/_internal/configurator.py | 20 ++++++--- .../_internal/override_centos.py | 5 ++- .../certbot_apache/_internal/parser.py | 44 ++++++++++--------- certbot-apache/tests/parser_test.py | 2 +- .../rfc2136_tests/context.py | 2 - .../utils/dns_server.py | 6 ++- .../configurators/common.py | 6 ++- .../_internal/dns_cloudflare.py | 6 ++- .../_internal/dns_cloudxns.py | 6 ++- .../_internal/dns_digitalocean.py | 6 ++- .../_internal/dns_dnsimple.py | 6 ++- .../_internal/dns_dnsmadeeasy.py | 6 ++- .../_internal/dns_gehirn.py | 7 ++- .../_internal/dns_google.py | 4 -- .../_internal/dns_linode.py | 6 ++- .../_internal/dns_luadns.py | 6 ++- .../certbot_dns_nsone/_internal/dns_nsone.py | 6 ++- .../certbot_dns_ovh/_internal/dns_ovh.py | 6 ++- .../_internal/dns_rfc2136.py | 6 ++- .../_internal/dns_sakuracloud.py | 7 ++- .../certbot_nginx/_internal/configurator.py | 2 +- certbot/certbot/_internal/cli/subparsers.py | 3 +- certbot/certbot/_internal/client.py | 4 ++ .../certbot/_internal/display/completer.py | 7 ++- certbot/certbot/_internal/lock.py | 2 + certbot/certbot/_internal/plugins/disco.py | 36 +++++++++------ certbot/certbot/interfaces.py | 3 +- certbot/certbot/plugins/dns_common.py | 3 +- certbot/certbot/plugins/dns_common_lexicon.py | 4 +- certbot/certbot/plugins/storage.py | 4 +- certbot/tests/plugins/storage_test.py | 5 ++- tox.ini | 2 +- 32 files changed, 164 insertions(+), 74 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index c24a646db..28f73b32d 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -12,6 +12,7 @@ from typing import cast from typing import DefaultDict from typing import Dict from typing import List +from typing import Optional from typing import Set from typing import Union @@ -36,6 +37,9 @@ from certbot_apache._internal import dualparser from certbot_apache._internal import http_01 from certbot_apache._internal import obj from certbot_apache._internal import parser +from certbot_apache._internal.dualparser import DualBlockNode +from certbot_apache._internal.obj import VirtualHost +from certbot_apache._internal.parser import ApacheParser try: import apacheconfig @@ -232,11 +236,11 @@ class ApacheConfigurator(common.Installer): self.parsed_paths: List[str] = [] # These will be set in the prepare function self._prepared = False - self.parser = None - self.parser_root = None + self.parser: ApacheParser + self.parser_root: Optional[DualBlockNode] = None self.version = version self._openssl_version = openssl_version - self.vhosts = None + self.vhosts: List[VirtualHost] self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, @@ -345,8 +349,9 @@ class ApacheConfigurator(common.Installer): "augeaspath": self.parser.get_root_augpath(), "ac_ast": None} if self.USE_PARSERNODE: - self.parser_root = self.get_parsernode_root(pn_meta) - self.parsed_paths = self.parser_root.parsed_paths() + parser_root = self.get_parsernode_root(pn_meta) + self.parser_root = parser_root + self.parsed_paths = parser_root.parsed_paths() # Check for errors in parsing files with Augeas self.parser.check_parsing_errors("httpd.aug") @@ -408,7 +413,7 @@ class ApacheConfigurator(common.Installer): super(ApacheConfigurator, self).recovery_routine() # Reload configuration after these changes take effect if needed # ie. ApacheParser has been initialized. - if self.parser: + if hasattr(self, "parser"): # TODO: wrap into non-implementation specific parser interface self.parser.aug.load() @@ -1051,6 +1056,9 @@ class ApacheConfigurator(common.Installer): :rtype: list """ + if not self.parser_root: + raise errors.Error("This ApacheConfigurator instance is not" # pragma: no cover + " configured to use a node parser.") vhs = [] vhosts = self.parser_root.find_blocks("VirtualHost", exclude=False) for vhblock in vhosts: diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index fc77aaafc..bd9a47b52 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -119,8 +119,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator): else: loadmod_args = path_args - if self.parser.not_modssl_ifmodule(noarg_path): # pylint: disable=no-member - if self.parser.loc["default"] in noarg_path: + centos_parser: CentOSParser = cast(CentOSParser, self.parser) + if centos_parser.not_modssl_ifmodule(noarg_path): + if centos_parser.loc["default"] in noarg_path: # LoadModule already in the main configuration file if ("ifmodule/" in noarg_path.lower() or "ifmodule[1]" in noarg_path.lower()): diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 1a6af4c4b..ff7e90f3b 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -5,12 +5,18 @@ import logging import re from typing import Dict from typing import List +from typing import Optional from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import constants +try: + from augeas import Augeas +except ImportError: # pragma: no cover + Augeas = None # type: ignore + logger = logging.getLogger(__name__) @@ -39,8 +45,7 @@ class ApacheParser: self.configurator = configurator # Initialize augeas - self.aug = None - self.init_augeas() + self.aug = init_augeas() if not self.check_aug_version(): raise errors.NotSupportedError( @@ -48,7 +53,7 @@ class ApacheParser: "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules: Dict[str, str] = {} + self.modules: Dict[str, Optional[str]] = {} self.parser_paths: Dict[str, List[str]] = {} self.variables: Dict[str, str] = {} @@ -83,23 +88,6 @@ class ApacheParser: if self.find_dir("Define", exclude=False): raise errors.PluginError("Error parsing runtime variables") - def init_augeas(self): - """ Initialize the actual Augeas instance """ - - try: - import augeas - except ImportError: # pragma: no cover - raise errors.NoInstallationError("Problem in Augeas installation") - - self.aug = augeas.Augeas( - # specify a directory to load our preferred lens from - loadpath=constants.AUGEAS_LENS_DIR, - # Do not save backup (we do it ourselves), do not load - # anything by default - flags=(augeas.Augeas.NONE | - augeas.Augeas.NO_MODL_AUTOLOAD | - augeas.Augeas.ENABLE_SPAN)) - def check_parsing_errors(self, lens): """Verify Augeas can parse all of the lens files. @@ -949,3 +937,19 @@ def get_aug_path(file_path): """ return "/files%s" % file_path + + +def init_augeas() -> Augeas: + """ Initialize the actual Augeas instance """ + + if not Augeas: # pragma: no cover + raise errors.NoInstallationError("Problem in Augeas installation") + + return Augeas( + # specify a directory to load our preferred lens from + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(Augeas.NONE | + Augeas.NO_MODL_AUTOLOAD | + Augeas.ENABLE_SPAN)) diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 7aedec31d..2eff9a9dd 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -339,7 +339,7 @@ class ParserInitTest(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - @mock.patch("certbot_apache._internal.parser.ApacheParser.init_augeas") + @mock.patch("certbot_apache._internal.parser.init_augeas") def test_prepare_no_augeas(self, mock_init_augeas): from certbot_apache._internal.parser import ApacheParser mock_init_augeas.side_effect = errors.NoInstallationError diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index 8df89f473..a4f62b5e4 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -17,7 +17,6 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): self.request = request - self._dns_xdist = None if hasattr(request.config, 'workerinput'): # Worker node self._dns_xdist = request.config.workerinput['dns_xdist'] else: # Primary node @@ -45,7 +44,6 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): src_file = resource_filename('certbot_integration_tests', 'assets/bind-config/rfc2136-credentials-{}.ini.tpl' .format(label)) - contents = None with open(src_file, 'r') as f: contents = f.read().format( diff --git a/certbot-ci/certbot_integration_tests/utils/dns_server.py b/certbot-ci/certbot_integration_tests/utils/dns_server.py index 31e7aee18..48f5a533e 100644 --- a/certbot-ci/certbot_integration_tests/utils/dns_server.py +++ b/certbot-ci/certbot_integration_tests/utils/dns_server.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile import time +from typing import Optional from pkg_resources import resource_filename @@ -38,7 +39,7 @@ class DNSServer: self.bind_root = tempfile.mkdtemp() - self.process: subprocess.Popen = None + self.process: Optional[subprocess.Popen] = None self.dns_xdist = {"address": BIND_BIND_ADDRESS[0], "port": BIND_BIND_ADDRESS[1]} @@ -119,6 +120,9 @@ class DNSServer: but otherwise the contents are ignored. :param int attempts: The number of attempts to make. """ + if not self.process: + raise ValueError("DNS server has not been started. Please run start() first.") + for _ in range(attempts): if self.process.poll(): raise ValueError("BIND9 server stopped unexpectedly") diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index bf768f8f8..bf26291ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -33,7 +33,9 @@ class Proxy: self.args = args self.http_port = 80 self.https_port = 443 - self._configurator = self._all_names = self._test_names = None + self._configurator = None + self._all_names = None + self._test_names = None def __getattr__(self, name): """Wraps the configurator methods""" @@ -93,5 +95,7 @@ class Proxy: """Installs cert""" cert_path, key_path, chain_path = self.copy_certs_and_keys( cert_path, key_path, chain_path) + if not self._configurator: + raise ValueError("Configurator plugin is not set.") self._configurator.deploy_cert( domain, cert_path, key_path, chain_path, fullchain_path) diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index 034d7bbb2..ceee39f86 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -3,6 +3,7 @@ import logging from typing import Any from typing import Dict from typing import List +from typing import Optional import CloudFlare import zope.interface @@ -10,6 +11,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -30,7 +32,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -79,6 +81,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudflare_client().del_txt_record(domain, validation_name, validation) def _get_cloudflare_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") if self.credentials.conf('api-token'): return _CloudflareClient(None, self.credentials.conf('api-token')) return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py index 654c04c70..7fe52b618 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py @@ -1,5 +1,6 @@ """DNS Authenticator for CloudXNS DNS.""" import logging +from typing import Optional from lexicon.providers import cloudxns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_cloudxns_client().del_txt_record(domain, validation_name, validation) def _get_cloudxns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _CloudXNSLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index cb5012fb7..85eb1159e 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -1,5 +1,6 @@ """DNS Authenticator for DigitalOcean.""" import logging +from typing import Optional import digitalocean import zope.interface @@ -7,6 +8,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -25,7 +27,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -53,6 +55,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) def _get_digitalocean_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DigitalOceanClient(self.credentials.conf('token')) diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py index 9f7f100d7..318bc25fb 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNSimple DNS.""" import logging +from typing import Optional from lexicon.providers import dnsimple import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsimple_client().del_txt_record(domain, validation_name, validation) def _get_dnsimple_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSimpleLexiconClient(self.credentials.conf('token'), self.ttl) diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py index 4a1fcffc3..3a805f507 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py @@ -1,5 +1,6 @@ """DNS Authenticator for DNS Made Easy DNS.""" import logging +from typing import Optional from lexicon.providers import dnsmadeeasy import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -28,7 +30,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -58,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_dnsmadeeasy_client().del_txt_record(domain, validation_name, validation) def _get_dnsmadeeasy_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _DNSMadeEasyLexiconClient(self.credentials.conf('api-key'), self.credentials.conf('secret-key'), self.ttl) diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py index 39deddae5..dd9665c2f 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py @@ -1,12 +1,15 @@ """DNS Authenticator for Gehirn Infrastructure Service DNS.""" import logging +from typing import Optional from lexicon.providers import gehirn import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +30,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -57,6 +60,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_gehirn_client().del_txt_record(domain, validation_name, validation) def _get_gehirn_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _GehirnLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 363c5e079..39f26c7d0 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -32,10 +32,6 @@ class Authenticator(dns_common.DNSAuthenticator): 'for DNS).') ttl = 60 - def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None - @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index c1b5e066f..c9e6a42b5 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -1,6 +1,7 @@ """DNS Authenticator for Linode.""" import logging import re +from typing import Optional from lexicon.providers import linode from lexicon.providers import linode4 @@ -10,6 +11,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -28,7 +30,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -56,6 +58,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_linode_client().del_txt_record(domain, validation_name, validation) def _get_linode_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") api_key = self.credentials.conf('key') api_version = self.credentials.conf('version') if api_version == '': diff --git a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py index d5b499c72..6595c3165 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py @@ -1,5 +1,6 @@ """DNS Authenticator for LuaDNS DNS.""" import logging +from typing import Optional from lexicon.providers import luadns import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -55,6 +57,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_luadns_client().del_txt_record(domain, validation_name, validation) def _get_luadns_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _LuaDNSLexiconClient(self.credentials.conf('email'), self.credentials.conf('token'), self.ttl) diff --git a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py index d328d80ce..dc4a0d0ee 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py @@ -1,5 +1,6 @@ """DNS Authenticator for NS1 DNS.""" import logging +from typing import Optional from lexicon.providers import nsone import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -54,6 +56,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_nsone_client().del_txt_record(domain, validation_name, validation) def _get_nsone_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _NS1LexiconClient(self.credentials.conf('api-key'), self.ttl) diff --git a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py index 11ae6b8f0..13f5b2805 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py @@ -1,5 +1,6 @@ """DNS Authenticator for OVH DNS.""" import logging +from typing import Optional from lexicon.providers import ovh import zope.interface @@ -8,6 +9,7 @@ from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -60,6 +62,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_ovh_client().del_txt_record(domain, validation_name, validation) def _get_ovh_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _OVHLexiconClient( self.credentials.conf('endpoint'), self.credentials.conf('application-key'), diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index adebdd963..a3ee1e0e3 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -1,5 +1,6 @@ """DNS Authenticator using RFC 2136 Dynamic Updates.""" import logging +from typing import Optional import dns.flags import dns.message @@ -15,6 +16,7 @@ import zope.interface from certbot import errors from certbot import interfaces from certbot.plugins import dns_common +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -44,7 +46,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -80,6 +82,8 @@ class Authenticator(dns_common.DNSAuthenticator): self._get_rfc2136_client().del_txt_record(validation_name, validation) def _get_rfc2136_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _RFC2136Client(self.credentials.conf('server'), int(self.credentials.conf('port') or self.PORT), self.credentials.conf('name'), diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py index 67cfb2e97..eb78735f5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py @@ -1,12 +1,15 @@ """DNS Authenticator for Sakura Cloud DNS.""" import logging +from typing import Optional from lexicon.providers import sakuracloud import zope.interface +from certbot import errors from certbot import interfaces from certbot.plugins import dns_common from certbot.plugins import dns_common_lexicon +from certbot.plugins.dns_common import CredentialsConfiguration logger = logging.getLogger(__name__) @@ -27,7 +30,7 @@ class Authenticator(dns_common.DNSAuthenticator): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self.credentials = None + self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ @@ -60,6 +63,8 @@ class Authenticator(dns_common.DNSAuthenticator): domain, validation_name, validation) def _get_sakuracloud_client(self): + if not self.credentials: # pragma: no cover + raise errors.Error("Plugin has not been prepared.") return _SakuraCloudLexiconClient( self.credentials.conf('api-token'), self.credentials.conf('api-secret'), diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index aab489315..c05ddceba 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -119,7 +119,6 @@ class NginxConfigurator(common.Installer): self._chall_out = 0 # These will be set in the prepare function - self.parser: Optional[parser.NginxParser] = None self.version = version self.openssl_version = openssl_version self._enhance_func = {"redirect": self._enable_redirect, @@ -127,6 +126,7 @@ class NginxConfigurator(common.Installer): "staple-ocsp": self._enable_ocsp_stapling} self.reverter.recovery_routine() + self.parser: parser.NginxParser @property def mod_ssl_conf_src(self): diff --git a/certbot/certbot/_internal/cli/subparsers.py b/certbot/certbot/_internal/cli/subparsers.py index 822381d21..dabb26661 100644 --- a/certbot/certbot/_internal/cli/subparsers.py +++ b/certbot/certbot/_internal/cli/subparsers.py @@ -32,8 +32,7 @@ def _create_subparsers(helpful): " Currently --csr only works with the 'certonly' subcommand.") helpful.add("revoke", "--reason", dest="reason", - choices=CaseInsensitiveList(sorted(constants.REVOCATION_REASONS, - key=constants.REVOCATION_REASONS.get)), + choices=CaseInsensitiveList(constants.REVOCATION_REASONS.keys()), action=_EncodeReasonAction, default=flag_default("reason"), help="Specify reason for revoking certificate. (default: unspecified)") helpful.add("revoke", diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 796f05599..0442febda 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -254,6 +254,7 @@ class Client: acme = acme_from_config_key(config, self.account.key, self.account.regr) self.acme = acme + self.auth_handler: Optional[auth_handler.AuthHandler] if auth is not None: self.auth_handler = auth_handler.AuthHandler( auth, self.acme, self.account, self.config.pref_challs) @@ -407,6 +408,9 @@ class Client: raise errors.Error("The currently selected ACME CA endpoint does" " not support issuing wildcard certificates.") + if not self.auth_handler: + raise errors.Error("No authorization handler has been set.") + # For a dry run, ensure we have an order with fresh authorizations if orderr and self.config.dry_run: deactivated, failed = self.auth_handler.deactivate_valid_authorizations(orderr) diff --git a/certbot/certbot/_internal/display/completer.py b/certbot/certbot/_internal/display/completer.py index a6c984195..b43859b19 100644 --- a/certbot/certbot/_internal/display/completer.py +++ b/certbot/certbot/_internal/display/completer.py @@ -1,5 +1,8 @@ """Provides Tab completion when prompting users for a path.""" import glob +from typing import Callable +from typing import Iterator +from typing import Optional # readline module is not available on all systems try: @@ -26,7 +29,9 @@ class Completer: """ def __init__(self): - self._iter = self._original_completer = self._original_delims = None + self._iter: Iterator[str] + self._original_completer: Optional[Callable] + self._original_delims: str def complete(self, text, state): """Provides path completion for use with readline. diff --git a/certbot/certbot/_internal/lock.py b/certbot/certbot/_internal/lock.py index d0b66e5e2..80b79d993 100644 --- a/certbot/certbot/_internal/lock.py +++ b/certbot/certbot/_internal/lock.py @@ -223,6 +223,8 @@ class _WindowsLockMechanism(_BaseLockMechanism): def release(self): """Release the lock.""" try: + if not self._fd: + raise errors.Error("The lock has not been acquired first.") # This "type: ignore" is currently needed because msvcrt methods # are only defined on Windows. See # https://github.com/python/typeshed/blob/16ae4c61201cd8b96b8b22cdfb2ab9e89ba5bcf2/stdlib/msvcrt.pyi. diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index 4097544e8..744062968 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -1,18 +1,20 @@ """Utilities for plugins discovery and selection.""" -from collections.abc import Mapping import itertools import logging import sys +from collections.abc import Mapping from typing import Dict +from typing import Optional +from typing import Union import pkg_resources import zope.interface import zope.interface.verify - from certbot import errors from certbot import interfaces from certbot._internal import constants from certbot.compat import os +from certbot.errors import Error logger = logging.getLogger(__name__) @@ -44,15 +46,15 @@ class PluginEntryPoint: # this object is mutable, don't allow it to be hashed! __hash__ = None # type: ignore - def __init__(self, entry_point, with_prefix=False): + def __init__(self, entry_point: pkg_resources.EntryPoint, with_prefix=False): self.name = self.entry_point_to_plugin_name(entry_point, with_prefix) - self.plugin_cls = entry_point.load() + self.plugin_cls: interfaces.IPluginFactory = entry_point.load() self.entry_point = entry_point - self.warning_message = None - self._initialized = None - self._prepared = None + self.warning_message: Optional[str] = None + self._initialized: Optional[interfaces.IPlugin] = None + self._prepared: Optional[Union[bool, Error]] = None self._hidden = False - self._long_description = None + self._long_description: Optional[str] = None def check_name(self, name): """Check if the name refers to this plugin.""" @@ -118,12 +120,15 @@ class PluginEntryPoint: """Memoized plugin initialization.""" if not self.initialized: self.entry_point.require() # fetch extras! - self._initialized = self.plugin_cls(config, self.name) + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized = self.plugin_cls(config, self.name) # type: ignore return self._initialized def verify(self, ifaces): """Verify that the plugin conforms to the specified interfaces.""" - assert self.initialized + if not self.initialized: + raise ValueError("Plugin is not initialized.") for iface in ifaces: # zope.interface.providedBy(plugin) try: zope.interface.verify.verifyObject(iface, self.init()) @@ -144,10 +149,13 @@ class PluginEntryPoint: def prepare(self): """Memoized plugin preparation.""" - assert self.initialized + if self._initialized is None: + raise ValueError("Plugin is not initialized.") if self._prepared is None: try: - self._initialized.prepare() + # TODO: remove type ignore once the interface becomes a proper + # abstract class (using abc) that mypy understands. + self._initialized.prepare() # type: ignore except errors.MisconfigurationError as error: logger.debug("Misconfigured %r: %s", self, error, exc_info=True) self._prepared = error @@ -247,8 +255,10 @@ class PluginsRegistry(Mapping): plugin_ep = PluginEntryPoint(entry_point, with_prefix) if plugin_ep.name in plugins: other_ep = plugins[plugin_ep.name] + plugin1 = plugin_ep.entry_point.dist.key if plugin_ep.entry_point.dist else "unknown" + plugin2 = other_ep.entry_point.dist.key if other_ep.entry_point.dist else "unknown" raise Exception("Duplicate plugin name {0} from {1} and {2}.".format( - plugin_ep.name, plugin_ep.entry_point.dist.key, other_ep.entry_point.dist.key)) + plugin_ep.name, plugin1, plugin2)) if interfaces.IPluginFactory.providedBy(plugin_ep.plugin_cls): plugins[plugin_ep.name] = plugin_ep else: # pragma: no cover diff --git a/certbot/certbot/interfaces.py b/certbot/certbot/interfaces.py index 3830e2849..de9175def 100644 --- a/certbot/certbot/interfaces.py +++ b/certbot/certbot/interfaces.py @@ -1,5 +1,6 @@ """Certbot client interfaces.""" import abc +from typing import Optional import zope.interface @@ -327,7 +328,7 @@ class IInstaller(IPlugin): """ - def save(title=None, temporary=False): + def save(title: Optional[str] = None, temporary: bool = False): """Saves all changes to the configuration files. Both title and temporary are needed because a save may be diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 52328ad85..4b621ceac 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -142,7 +142,8 @@ class DNSAuthenticator(common.Plugin): setattr(self.config, self.dest(key), os.path.abspath(os.path.expanduser(new_value))) - def _configure_credentials(self, key, label, required_variables=None, validator=None): + def _configure_credentials(self, key, label, required_variables=None, + validator=None) -> 'CredentialsConfiguration': """ As `_configure_file`, but for a credential configuration file. diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index 10efa5f85..a4d46587e 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -17,8 +17,10 @@ from certbot.plugins import dns_common # if Lexicon is not available, obviously. try: from lexicon.config import ConfigResolver + from lexicon.providers.base import Provider except ImportError: ConfigResolver = None # type: ignore + Provider = None # type: ignore logger = logging.getLogger(__name__) @@ -29,7 +31,7 @@ class LexiconClient: """ def __init__(self): - self.provider = None + self.provider: Provider def add_txt_record(self, domain, record_name, record_content): """ diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index 0d32d6850..602c62d1e 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -26,8 +26,8 @@ class PluginStorage: self._config = config self._classkey = classkey self._initialized = False - self._data = None - self._storagepath = None + self._data: Dict + self._storagepath: str def _initialize_storage(self): """Initializes PluginStorage data and reads current state from the disk diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 2999d306e..4eb5f0912 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -31,7 +31,7 @@ class PluginStorageTest(test_util.ConfigTestCase): # When unable to read file that exists mock_open = mock.mock_open() mock_open.side_effect = IOError - self.plugin.storage.storagepath = os.path.join(self.config.config_dir, + self.plugin.storage._storagepath = os.path.join(self.config.config_dir, ".pluginstorage.json") with mock.patch("builtins.open", mock_open): with mock.patch('certbot.compat.os.path.isfile', return_value=True): @@ -67,7 +67,7 @@ class PluginStorageTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: # Set data as something that can't be serialized self.plugin.storage._initialized = True # pylint: disable=protected-access - self.plugin.storage.storagepath = "/tmp/whatever" + self.plugin.storage._storagepath = "/tmp/whatever" self.plugin.storage._data = self.plugin_cls # pylint: disable=protected-access self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) @@ -80,6 +80,7 @@ class PluginStorageTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: self.plugin.storage._data = {"valid": "data"} # pylint: disable=protected-access self.plugin.storage._initialized = True # pylint: disable=protected-access + self.plugin.storage._storagepath = "/tmp/whatever" self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) self.assertTrue("Could not write" in mock_log.call_args[0][0]) diff --git a/tox.ini b/tox.ini index 18d0d7db3..ada7002da 100644 --- a/tox.ini +++ b/tox.ini @@ -156,7 +156,7 @@ commands = basepython = python3 commands = {[base]install_packages} - mypy --no-strict-optional {[base]source_paths} + mypy {[base]source_paths} [testenv:apacheconftest] commands = From a12d91aef6c48765b1783f0741f55202343d7a9b Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 6 Apr 2021 08:50:12 +1000 Subject: [PATCH 59/99] fix various fd leaks (#8747) * fix various fd leaks * use context manager for display provider --- certbot/certbot/_internal/main.py | 40 ++++++++++++++++++--------- certbot/tests/log_test.py | 3 +- certbot/tests/plugins/webroot_test.py | 3 +- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index ac4d18449..aed265ba3 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -4,6 +4,9 @@ import functools import logging.handlers import sys +from contextlib import contextmanager +from typing import Generator +from typing import IO from typing import Iterable from typing import List from typing import Optional @@ -1347,26 +1350,36 @@ def make_or_verify_needed_dirs(config): util.make_or_verify_dir(hook_dir, strict=config.strict_permissions) -def set_displayer(config): - """Set the displayer +@contextmanager +def make_displayer(config: configuration.NamespaceConfig + ) -> Generator[Union[display_util.NoninteractiveDisplay, + display_util.FileDisplay], None, None]: + """Creates a display object appropriate to the flags in the supplied config. :param config: Configuration object - :type config: interfaces.IConfig - :returns: `None` - :rtype: None + :returns: Display object implementing :class:`certbot.interfaces.IDisplay` """ + displayer: Union[None, display_util.NoninteractiveDisplay, + display_util.FileDisplay] = None + devnull: Optional[IO] = None + if config.quiet: config.noninteractive_mode = True - displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\ - display_util.NoninteractiveDisplay(open(os.devnull, "w")) + devnull = open(os.devnull, "w") + displayer = display_util.NoninteractiveDisplay(devnull) elif config.noninteractive_mode: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: - displayer = display_util.FileDisplay(sys.stdout, - config.force_interactive) - zope.component.provideUtility(displayer) + displayer = display_util.FileDisplay( + sys.stdout, config.force_interactive) + + try: + yield displayer + finally: + if devnull: + devnull.close() def main(cli_args=None): @@ -1411,11 +1424,12 @@ def main(cli_args=None): if config.func != plugins_cmd: # pylint: disable=comparison-with-callable raise - set_displayer(config) - # Reporter report = reporter.Reporter(config) zope.component.provideUtility(report) util.atexit_register(report.print_messages) - return config.func(config, plugins) + with make_displayer(config) as displayer: + zope.component.provideUtility(displayer) + + return config.func(config, plugins) diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 7b1aacd5c..5960b37cf 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -28,7 +28,8 @@ class PreArgParseSetupTest(unittest.TestCase): @classmethod def _call(cls, *args, **kwargs): # pylint: disable=unused-argument from certbot._internal.log import pre_arg_parse_setup - return pre_arg_parse_setup() + with mock.patch('builtins.open', mock.mock_open()): + return pre_arg_parse_setup() @mock.patch('certbot._internal.log.sys') @mock.patch('certbot._internal.log.pre_arg_parse_except_hook') diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index e6fbd8e88..5792924a9 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -141,7 +141,8 @@ class AuthenticatorTest(unittest.TestCase): f.write("thingimy") filesystem.chmod(self.path, 0o000) try: - open(permission_canary, "r") + with open(permission_canary, "r"): + pass print("Warning, running tests as root skips permissions tests...") except IOError: # ok, permissions work, test away... From f15d10abc827b59d06418926621cd26abfcf8534 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 5 Apr 2021 16:02:14 -0700 Subject: [PATCH 60/99] Update Dockerfile-dev (#8774) * switch Dockerfile-dev to Ubuntu Focal * Make apt noninteractive * add --no-install-recommends --- Dockerfile-dev | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 86847f8fd..895dbdc0b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,5 +1,5 @@ # This Dockerfile builds an image for development. -FROM debian:buster +FROM ubuntu:focal # Note: this only exposes the port to other docker containers. EXPOSE 80 443 @@ -8,8 +8,9 @@ WORKDIR /opt/certbot/src COPY . . RUN apt-get update && \ - apt-get install apache2 git python3-dev python3-venv gcc libaugeas0 \ - libssl-dev libffi-dev ca-certificates openssl nginx-light -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install apache2 git python3-dev \ + python3-venv gcc libaugeas0 libssl-dev libffi-dev ca-certificates \ + openssl nginx-light -y --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ From 1b6e4028dcc01ae8f23d83393626e8dbde85fd28 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Apr 2021 10:17:01 -0700 Subject: [PATCH 61/99] Update changelog for 1.14.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 9b37e9a1a..e0ebcfd81 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.14.0 - master +## 1.14.0 - 2021-04-06 ### Added From bf40b81b5abcc4fef0f61770310bd56b529fa27f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Apr 2021 10:24:32 -0700 Subject: [PATCH 62/99] Release 1.14.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 27 ++++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 2 +- letsencrypt-auto | 27 ++++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++------ letsencrypt-auto-source/letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/letsencrypt-auto.sig | 4 +-- .../pieces/certbot-requirements.txt | 24 ++++++++--------- 26 files changed, 82 insertions(+), 82 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 90ab5c9f3..1170e7393 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 43165a9dd..eb5e59130 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 24af0b4c3..c37c45596 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1488,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 3e9e6a7c2..f1e0b4425 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 2e5215f9c..f2daa8067 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 44bada6bc..c276beff2 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 106eb3cac..5f2b34ca3 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 99a3c92a3..fa2d2a53b 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index b126b4e8f..d1c26f6d1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index cada7a85e..9c059a82c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index e8b26eceb..6e059eef8 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 63da31df2..780eb3a58 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 61b9dfa78..9537d261a 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d60412f27..4c7bba724 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 4d6131ff7..8d1dcab2a 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 18d006732..86eef3f3d 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index fdc931bbe..f52ba6e08 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 55f140d76..0ac3019cf 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 8b638f28e..217c59373 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0.dev0' +version = '1.14.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 790443bcc..672b4ed1d 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.14.0.dev0' +__version__ = '1.14.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index 27511777b..c5490c100 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,7 +118,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.13.0 + "". (default: CertbotACMEClient/1.14.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the diff --git a/letsencrypt-auto b/letsencrypt-auto index 24af0b4c3..c37c45596 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.13.0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -800,6 +800,7 @@ BootstrapMageiaCommon() { # packages BOOTSTRAP_VERSION is not set. if [ -f /etc/debian_version ]; then DEPRECATED_OS=1 + NO_SELF_UPGRADE=1 elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release DEPRECATED_OS=1 @@ -1488,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index fbd16f2b0..c0cf63418 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmA+sv4ACgkQTRfJlc2X -dfLG+ggAvwUCqy06UZd8jZhOZFUoi8nWHG2TMnm/4CW4G9PPCjsCoQplhaAaUCrR -PAv0vtsG1rBKWekICg6IBTzLVioH9xRPUkpfbVQhT1c1T/5CqMsFFXR5p9YAKRe7 -hlOb7VRN10bdCS4JThRPNhdWFdWKZXYKcIOObWA/FX2GacxMHuLwPpSsbt2NRffy -qz1ZOWxvr289aWEbZWfyBiI2bxQ7wlMEbZ/JLUXDe46ETDxzENu+c0x2109ha6m4 -wHmUS0r16ps/n9DueTZGJf3C26mU+cIB+LgNvOcibBo+0Ly7t+OiBHYbkFXS2KTQ -RbH4bPZrsduUOzhE8wIsSUIsVGDleQ== -=jYnL +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAmBsmUkACgkQTRfJlc2X +dfI7Bwf9FkNrf1HEh2G3uk1p+qLMd/s5kcVV2udK2FkRELee5nHlLZx2YmHA/8ID +gqsk8EsyRZNMX374nGrPm0syykdEsyVtMJTbHCEr+Ms3l54ZgE3HV6ywnhWSlAFo +Za50kdzhodBVTS5AEADbCKLKObVAWwO3fFKtKyv/iY29ykpHK0KSHCKRII3iQU7l +dnR6u35Z0wgfEmDxsH27K6uo0YepZaEL70qHHFk93MhCh9Z15rO17gRpsVzz7Z1j +YClI6h2K/VOfZtbkoQvoks7s+xd75Kjr3GNH+cznkJx8gNWSZLfkc1XX4Bjdm4GG +IWz3Ezy8tFg6PtITb7y+aIg75kWx4w== +=zEy4 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b0b25759f..c37c45596 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.14.0.dev0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1489,18 +1489,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.13.0 \ - --hash=sha256:082eb732e1318bb9605afa7aea8db2c2f4c5029d523c73f24c6aa98f03caff76 \ - --hash=sha256:64cf41b57df7667d9d849fcaa9031a4f151788246733d1f4c3f37a5aa5e2f458 -acme==1.13.0 \ - --hash=sha256:93b6365c9425de03497a6b8aee1107814501d2974499b42e9bcc9a7378771143 \ - --hash=sha256:6b4257dfd6a6d5f01e8cd4f0b10422c17836bed7c67e9c5b0a0ad6c7d651c088 -certbot-apache==1.13.0 \ - --hash=sha256:36ed02ac7d2d91febee8dd3181ae9095b3f06434c9ed8959fbc6db24ab4da2e8 \ - --hash=sha256:4b5a16e80c1418e2edc05fc2578f522fb24974b2c13eb747cdfeef69e5bd5ae1 -certbot-nginx==1.13.0 \ - --hash=sha256:3ff271f65321b25c77a868af21f76f58754a7d61529ad565a1d66e29c711120f \ - --hash=sha256:9e972cc19c0fa9e5b7863da0423b156fbfb5623fd30b558fd2fd6d21c24c0b08 +certbot==1.14.0 \ + --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ + --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb +acme==1.14.0 \ + --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ + --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 +certbot-apache==1.14.0 \ + --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ + --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 +certbot-nginx==1.14.0 \ + --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ + --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 3516f77b0..b297bdfb3 100644 --- a/letsencrypt-auto-source/letsencrypt-auto.sig +++ b/letsencrypt-auto-source/letsencrypt-auto.sig @@ -1,3 +1 @@ -_̊П‡åNþÌ*–¿”ŽšJsµ¤‰Q…j ÜS¢llªY3Q]Âê«x+/V`$4xM…ñï晾èUăÚí«Sí³•Pl ¶Hbƒ a<£{p~`ÐŒó&a1E@²Ú;RNiL¬J¹@îpÌÏwk!Š ‡&Æ, Date: Tue, 6 Apr 2021 10:24:34 -0700 Subject: [PATCH 63/99] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index e0ebcfd81..7648d0ec3 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.15.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.14.0 - 2021-04-06 ### Added From 04a85742c1529167772e326ba904eca3d6aa7f4d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 6 Apr 2021 10:24:35 -0700 Subject: [PATCH 64/99] Bump version to 1.15.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 1170e7393..443aab6e8 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index eb5e59130..76ebbddfd 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index f1e0b4425..f1bf28596 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index f2daa8067..fcc7c3036 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index c276beff2..6452d325d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 5f2b34ca3..33f16314e 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index fa2d2a53b..14fcd4a69 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index d1c26f6d1..dd02d96b5 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 9c059a82c..00ff0834a 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 6e059eef8..4933ffb0b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 780eb3a58..55dad1d60 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 9537d261a..46c71c46b 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4c7bba724..6d3218221 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8d1dcab2a..e8e44db30 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 86eef3f3d..6b4648d91 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index f52ba6e08..30c45dc84 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 0ac3019cf..c328429c9 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 217c59373..11df5eddf 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.14.0' +version = '1.15.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 672b4ed1d..fbf8160fc 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.14.0' +__version__ = '1.15.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c37c45596..9ddc7e076 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.14.0" +LE_AUTO_VERSION="1.15.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 459a254aea3a9a5a6ac9f069ba446d93ce307a5c Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Apr 2021 14:52:15 -0700 Subject: [PATCH 65/99] Improve tools/snap/build_remote.py output (#8780) I think this PR improves tools/snap/build_remote.py's output in a number of ways such as: * Logs of snap builds were being deleted because they weren't being copied out of the temporary directory added in https://github.com/certbot/certbot/pull/8719. * The lock should now always be acquired before printing output when multiple processes are running which helps prevent processes mixing their output with each other. * Output is never buffered which ensures that repeated calls to `print` from the same process while it holds the output lock is kept together. * The case where we printed output about the "chroot problem" and stopped retrying the build has been deleted because with the fix in https://github.com/certbot/certbot/pull/8719, we should be able to recover in this case. * If the build failed for any reason, we dump as much output about the problem as we can. I think most times we won't need to read this output, but I personally prefer it being there in case we want it for some reason. Due to this change, I also simplified `_build_snap` and `_dump_results` a bit since `_build_snap` handles printing logs as needed. * print more output * lock when printing output * clarify purpose of lock * preserve logfiles * python better * consistently flush output * remove workspaces dict * rename variable * remove unused variable * don't use all which exits early * fix typo --- tools/snap/build_remote.py | 156 +++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index b12799dc3..91854e9e0 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse import datetime +import functools import glob from multiprocessing import Manager from multiprocessing import Pool @@ -28,21 +29,34 @@ CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] +# In Python, stdout and stderr are buffered in each process by default. When +# printing output from multiple processes, this can cause delays in printing +# output with lines from different processes being interleaved depending +# on when the output for that process is flushed. To prevent this, we override +# print so that it always flushes its output. Disabling output buffering can +# also be done through command line flags or environment variables set when the +# Python process starts, but this approach was taken instead to ensure +# consistent behavior regardless of how the script is invoked. +print = functools.partial(print, flush=True) + + +def _snap_log_name(target: str, arch: str): + return f'{target}_{arch}.txt' + + def _execute_build( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], workspace: str) -> Tuple[int, List[str]]: - temp_workspace = None - try: - # Snapcraft remote-build has a recover feature, that will make it reconnect to an existing - # build on Launchpad if possible. However, the signature used to retrieve a potential - # build is not based on the content of the sources used to build a snap, but on a hash - # of the snapcraft current working directory (the path itself, not the content). - # It means that every build started from /my/path/to/certbot will always be considered - # as the same build, whatever the actual sources are. - # To circumvent this, we create a temporary folder and use it as a workspace to build - # the snap: this path is random, making the recover feature effectively noop. - temp_workspace = tempfile.mkdtemp() + # Snapcraft remote-build has a recover feature, that will make it reconnect to an existing + # build on Launchpad if possible. However, the signature used to retrieve a potential + # build is not based on the content of the sources used to build a snap, but on a hash + # of the snapcraft current working directory (the path itself, not the content). + # It means that every build started from /my/path/to/certbot will always be considered + # as the same build, whatever the actual sources are. + # To circumvent this, we create a temporary folder and use it as a workspace to build + # the snap: this path is random, making the recover feature effectively noop. + with tempfile.TemporaryDirectory() as temp_workspace: ignore = None if target == 'certbot': ignore = shutil.ignore_patterns(".git", "venv*", ".tox") @@ -71,20 +85,18 @@ def _execute_build( for path in glob.glob(join(temp_workspace, '*.snap')): shutil.copy(path, workspace) + for arch in archs: + log_name = _snap_log_name(target, arch) + log_path = join(temp_workspace, log_name) + if exists(log_path): + shutil.copy(log_path, workspace) return process_state, process_output - except BaseException as e: - print(e) - sys.stdout.flush() - raise e - finally: - if temp_workspace: - shutil.rmtree(temp_workspace, ignore_errors=True) def _build_snap( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], - running: Dict[str, bool], lock: Lock) -> Dict[str, str]: + running: Dict[str, bool], output_lock: Lock) -> bool: status[target] = {arch: '...' for arch in archs} if target == 'certbot': @@ -92,24 +104,19 @@ def _build_snap( else: workspace = join(CERTBOT_DIR, target) + build_success = False retry = 3 while retry: exit_code, process_output = _execute_build(target, archs, status, workspace) + with output_lock: + print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' + f'exit code {exit_code}.') - print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' - f'exit code {exit_code}.') - sys.stdout.flush() - - with lock: - dump_output = exit_code != 0 failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] - if any(arch for arch in archs if status[target][arch] == 'Chroot problem'): - print('Some builds failed with the status "Chroot problem".') - print('This status is known to make any future build fail until either ' - 'the source code changes or the build on Launchpad is deleted.') - print('Please fix the build appropriately before trying a new one.') - # It is useless to retry in this situation. - retry = 0 + # If the command failed or any architecture wasn't built + # successfully, let's try to print all the output about the problem + # that we can. + dump_output = exit_code != 0 or failed_archs if exit_code == 0 and not failed_archs: # We expect to have all target snaps available, or something bad happened. snaps_list = glob.glob(join(workspace, '*.snap')) @@ -118,16 +125,12 @@ def _build_snap( f'(current list: {snaps_list}).') dump_output = True else: + build_success = True break - if failed_archs: - # We expect each failed build to have a log file, or something bad happened. - for arch in failed_archs: - if not exists(join(workspace, f'{target}_{arch}.txt')): - dump_output = True - print(f'Missing output on a failed build {target} for {arch}.') if dump_output: print(f'Dumping snapcraft remote-build output build for {target}:') print('\n'.join(process_output)) + _dump_failed_build_logs(target, archs, status, workspace) # Retry the remote build if it has been interrupted (non zero status code) # or if some builds have failed. @@ -135,7 +138,7 @@ def _build_snap( running[target] = False - return {target: workspace} + return build_success def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: @@ -167,53 +170,45 @@ def _dump_status_helper(archs: Set[str], status: Dict[str, Dict[str, str]]) -> N print(f'|{"-" * 26}' * len(headers)) print() - sys.stdout.flush() - def _dump_status( archs: Set[str], status: Dict[str, Dict[str, str]], - running: Dict[str, bool]) -> None: + running: Dict[str, bool], output_lock: Lock) -> None: while any(running.values()): - print(f'Remote build status at {datetime.datetime.now()}') - _dump_status_helper(archs, status) + with output_lock: + print(f'Remote build status at {datetime.datetime.now()}') + _dump_status_helper(archs, status) time.sleep(10) -def _dump_results( - targets: Set[str], archs: Set[str], status: Dict[str, Dict[str, str]], - workspaces: Dict[str, str]) -> bool: - failures = False - for target in targets: - for arch in archs: - result = status[target][arch] +def _dump_failed_build_logs( + target: str, archs: Set[str], status: Dict[str, Dict[str, str]], + workspace: str) -> None: + for arch in archs: + result = status[target][arch] - if result != 'Successfully built': - failures = True + if result != 'Successfully built': + failures = True - build_output_path = join(workspaces[target], f'{target}_{arch}.txt') - if not exists(build_output_path): - build_output = f'No output has been dumped by snapcraft remote-build.' - else: - with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h: - build_output = file_h.read() + build_output_name = _snap_log_name(target, arch) + build_output_path = join(workspace, build_output_name) + if not exists(build_output_path): + build_output = f'No output has been dumped by snapcraft remote-build.' + else: + with open(build_output_path) as file_h: + build_output = file_h.read() - print(f'Output for failed build target={target} arch={arch}') - print('-------------------------------------------') - print(build_output) - print('-------------------------------------------') - print() + print(f'Output for failed build target={target} arch={arch}') + print('-------------------------------------------') + print(build_output) + print('-------------------------------------------') + print() - if not failures: - print('All builds succeeded.') - else: - print('Some builds failed.') - print() +def _dump_results(archs: Set[str], status: Dict[str, Dict[str, str]]) -> None: print(f'Results for remote build finished at {datetime.datetime.now()}') _dump_status_helper(archs, status) - return failures - def main(): parser = argparse.ArgumentParser() @@ -252,12 +247,14 @@ def main(): with manager, pool: status: Dict[str, Dict[str, str]] = manager.dict() running = manager.dict({target: True for target in targets}) - lock = manager.Lock() + # While multiple processes are running, this lock should be acquired + # before printing output. + output_lock = manager.Lock() - async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock)) + async_results = [pool.apply_async(_build_snap, (target, archs, status, running, output_lock)) for target in targets] - process = Process(target=_dump_status, args=(archs, status, running)) + process = Process(target=_dump_status, args=(archs, status, running, output_lock)) process.start() try: @@ -266,11 +263,16 @@ def main(): if process.is_alive(): raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!") - workspaces = {} + build_success = True for async_result in async_results: - workspaces.update(async_result.get()) + if not async_result.get(): + build_success = False - if _dump_results(targets, archs, status, workspaces): + _dump_results(archs, status) + if build_success: + print('All builds succeeded.') + else: + print('Some builds failed.') raise ValueError("There were failures during the build!") finally: process.terminate() From 7f9857a81b7064d767f694e033b29dc4bb942406 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 8 Apr 2021 13:04:51 -0700 Subject: [PATCH 66/99] Use Python 3 style super (#8777) This is one of the things that newer versions of `pylint` complains about. * git grep -l super\( | xargs sed -i 's/super([^)]*)/super()/g' * fix spacing --- acme/acme/challenges.py | 6 ++--- acme/acme/client.py | 6 ++--- acme/acme/errors.py | 12 +++++----- acme/acme/fields.py | 4 ++-- acme/acme/jws.py | 2 +- acme/acme/messages.py | 22 +++++++++---------- acme/acme/mixins.py | 10 ++++----- acme/tests/client_test.py | 6 ++--- .../certbot_apache/_internal/apacheparser.py | 8 +++---- .../certbot_apache/_internal/augeasparser.py | 8 +++---- .../certbot_apache/_internal/configurator.py | 6 ++--- .../certbot_apache/_internal/http_01.py | 2 +- .../certbot_apache/_internal/interfaces.py | 4 ++-- .../certbot_apache/_internal/obj.py | 2 +- .../_internal/override_centos.py | 12 +++++----- .../_internal/override_debian.py | 2 +- .../_internal/override_fedora.py | 10 ++++----- .../_internal/override_gentoo.py | 4 ++-- certbot-apache/tests/augeasnode_test.py | 2 +- certbot-apache/tests/autohsts_test.py | 2 +- certbot-apache/tests/centos6_test.py | 6 ++--- certbot-apache/tests/centos_test.py | 12 +++++----- certbot-apache/tests/complex_parsing_test.py | 2 +- .../tests/configurator_reverter_test.py | 2 +- certbot-apache/tests/configurator_test.py | 16 +++++++------- certbot-apache/tests/debian_test.py | 2 +- certbot-apache/tests/fedora_test.py | 12 +++++----- certbot-apache/tests/gentoo_test.py | 6 ++--- certbot-apache/tests/http_01_test.py | 2 +- certbot-apache/tests/parser_test.py | 4 ++-- .../tests/parsernode_configurator_test.py | 2 +- certbot-apache/tests/parsernode_test.py | 6 ++--- certbot-apache/tests/util.py | 2 +- .../nginx_tests/context.py | 4 ++-- .../rfc2136_tests/context.py | 2 +- .../configurators/apache/common.py | 6 ++--- .../configurators/nginx/common.py | 2 +- .../_internal/dns_cloudflare.py | 4 ++-- .../tests/dns_cloudflare_test.py | 2 +- .../_internal/dns_cloudxns.py | 6 ++--- .../tests/dns_cloudxns_test.py | 2 +- .../_internal/dns_digitalocean.py | 4 ++-- .../tests/dns_digitalocean_test.py | 2 +- .../_internal/dns_dnsimple.py | 6 ++--- .../tests/dns_dnsimple_test.py | 2 +- .../_internal/dns_dnsmadeeasy.py | 6 ++--- .../tests/dns_dnsmadeeasy_test.py | 2 +- .../_internal/dns_gehirn.py | 8 +++---- certbot-dns-gehirn/tests/dns_gehirn_test.py | 2 +- .../_internal/dns_google.py | 2 +- certbot-dns-google/tests/dns_google_test.py | 4 ++-- .../_internal/dns_linode.py | 6 ++--- certbot-dns-linode/tests/dns_linode_test.py | 2 +- .../_internal/dns_luadns.py | 6 ++--- certbot-dns-luadns/tests/dns_luadns_test.py | 2 +- .../certbot_dns_nsone/_internal/dns_nsone.py | 6 ++--- certbot-dns-nsone/tests/dns_nsone_test.py | 2 +- .../certbot_dns_ovh/_internal/dns_ovh.py | 8 +++---- certbot-dns-ovh/tests/dns_ovh_test.py | 2 +- .../_internal/dns_rfc2136.py | 4 ++-- certbot-dns-rfc2136/tests/dns_rfc2136_test.py | 2 +- .../_internal/dns_route53.py | 2 +- .../certbot_dns_route53/authenticator.py | 2 +- certbot-dns-route53/tests/dns_route53_test.py | 2 +- .../_internal/dns_sakuracloud.py | 8 +++---- .../tests/dns_sakuracloud_test.py | 2 +- .../certbot_nginx/_internal/configurator.py | 6 ++--- .../certbot_nginx/_internal/http_01.py | 2 +- certbot-nginx/certbot_nginx/_internal/obj.py | 6 ++--- .../certbot_nginx/_internal/parser_obj.py | 6 ++--- certbot-nginx/tests/configurator_test.py | 4 ++-- certbot-nginx/tests/display_ops_test.py | 2 +- certbot-nginx/tests/http_01_test.py | 2 +- certbot-nginx/tests/test_util.py | 2 +- certbot/certbot/_internal/cli/cli_utils.py | 2 +- certbot/certbot/_internal/log.py | 16 +++++++------- certbot/certbot/_internal/plugins/manual.py | 2 +- .../certbot/_internal/plugins/standalone.py | 2 +- certbot/certbot/_internal/plugins/webroot.py | 4 ++-- certbot/certbot/_internal/snap_config.py | 4 ++-- certbot/certbot/display/util.py | 4 ++-- certbot/certbot/errors.py | 4 ++-- certbot/certbot/plugins/common.py | 2 +- certbot/certbot/plugins/dns_common.py | 2 +- certbot/certbot/tests/util.py | 2 +- certbot/tests/account_test.py | 2 +- certbot/tests/cert_manager_test.py | 10 ++++----- certbot/tests/client_test.py | 8 +++---- certbot/tests/compat/filesystem_test.py | 10 ++++----- certbot/tests/configuration_test.py | 2 +- certbot/tests/crypto_util_test.py | 6 ++--- certbot/tests/display/completer_test.py | 2 +- certbot/tests/display/ops_test.py | 2 +- certbot/tests/display/util_test.py | 2 +- certbot/tests/eff_test.py | 2 +- certbot/tests/error_handler_test.py | 2 +- certbot/tests/hook_test.py | 16 +++++++------- certbot/tests/lock_test.py | 2 +- certbot/tests/log_test.py | 6 ++--- certbot/tests/main_test.py | 16 +++++++------- certbot/tests/plugins/common_test.py | 4 ++-- .../tests/plugins/dns_common_lexicon_test.py | 2 +- certbot/tests/plugins/dns_common_test.py | 4 ++-- certbot/tests/plugins/enhancements_test.py | 2 +- certbot/tests/plugins/manual_test.py | 2 +- certbot/tests/plugins/selection_test.py | 2 +- certbot/tests/plugins/storage_test.py | 2 +- certbot/tests/renewupdater_test.py | 2 +- certbot/tests/reverter_test.py | 4 ++-- certbot/tests/storage_test.py | 6 ++--- certbot/tests/util_test.py | 8 +++---- 111 files changed, 263 insertions(+), 263 deletions(-) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 4fad9a421..2c8190be5 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -30,7 +30,7 @@ class Challenge(jose.TypedJSONObjectWithFields): @classmethod def from_json(cls, jobj): try: - return super(Challenge, cls).from_json(jobj) + return super().from_json(jobj) except jose.UnrecognizedTypeError as error: logger.debug(error) return UnrecognizedChallenge.from_json(jobj) @@ -58,7 +58,7 @@ class UnrecognizedChallenge(Challenge): """ def __init__(self, jobj): - super(UnrecognizedChallenge, self).__init__() + super().__init__() object.__setattr__(self, "jobj", jobj) def to_partial_json(self): @@ -141,7 +141,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse): return True def to_partial_json(self): - jobj = super(KeyAuthorizationChallengeResponse, self).to_partial_json() + jobj = super().to_partial_json() jobj.pop('keyAuthorization', None) return jobj diff --git a/acme/acme/client.py b/acme/acme/client.py index ae4a64b23..548c3d548 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -253,7 +253,7 @@ class Client(ClientBase): if isinstance(directory, str): directory = messages.Directory.from_json( net.get(directory).json()) - super(Client, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=1) def register(self, new_reg=None): @@ -577,7 +577,7 @@ class ClientV2(ClientBase): :param .messages.Directory directory: Directory Resource :param .ClientNetwork net: Client network. """ - super(ClientV2, self).__init__(directory=directory, + super().__init__(directory=directory, net=net, acme_version=2) def new_account(self, new_account): @@ -627,7 +627,7 @@ class ClientV2(ClientBase): """ # https://github.com/certbot/certbot/issues/6155 new_regr = self._get_v2_account(regr) - return super(ClientV2, self).update_registration(new_regr, update) + return super().update_registration(new_regr, update) def _get_v2_account(self, regr): self.net.account = None diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 84b7704dc..b47ed88da 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -29,7 +29,7 @@ class NonceError(ClientError): class BadNonce(NonceError): """Bad nonce error.""" def __init__(self, nonce, error, *args): - super(BadNonce, self).__init__(*args) + super().__init__(*args) self.nonce = nonce self.error = error @@ -48,7 +48,7 @@ class MissingNonce(NonceError): """ def __init__(self, response, *args): - super(MissingNonce, self).__init__(*args) + super().__init__(*args) self.response = response def __str__(self): @@ -72,7 +72,7 @@ class PollError(ClientError): def __init__(self, exhausted, updated): self.exhausted = exhausted self.updated = updated - super(PollError, self).__init__() + super().__init__() @property def timeout(self): @@ -90,7 +90,7 @@ class ValidationError(Error): """ def __init__(self, failed_authzrs): self.failed_authzrs = failed_authzrs - super(ValidationError, self).__init__() + super().__init__() class TimeoutError(Error): # pylint: disable=redefined-builtin @@ -106,7 +106,7 @@ class IssuanceError(Error): :param messages.Error error: The error provided by the server. """ self.error = error - super(IssuanceError, self).__init__() + super().__init__() class ConflictError(ClientError): @@ -119,7 +119,7 @@ class ConflictError(ClientError): """ def __init__(self, location): self.location = location - super(ConflictError, self).__init__() + super().__init__() class WildcardUnsupportedError(Error): diff --git a/acme/acme/fields.py b/acme/acme/fields.py index 3b5672283..bd915e47d 100644 --- a/acme/acme/fields.py +++ b/acme/acme/fields.py @@ -12,7 +12,7 @@ class Fixed(jose.Field): def __init__(self, json_name, value): self.value = value - super(Fixed, self).__init__( + super().__init__( json_name=json_name, default=value, omitempty=False) def decode(self, value): @@ -53,7 +53,7 @@ class Resource(jose.Field): def __init__(self, resource_type, *args, **kwargs): self.resource_type = resource_type - super(Resource, self).__init__( + super().__init__( 'resource', default=resource_type, *args, **kwargs) def decode(self, value): diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 7eccf0fdf..d9d3e12c3 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -50,7 +50,7 @@ class JWS(jose.JWS): # Per ACME spec, jwk and kid are mutually exclusive, so only include a # jwk field if kid is not provided. include_jwk = kid is None - return super(JWS, cls).sign(payload, key=key, alg=alg, + return super().sign(payload, key=key, alg=alg, protect=frozenset(['nonce', 'url', 'kid', 'jwk', 'alg']), nonce=nonce, url=url, kid=kid, include_jwk=include_jwk) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index c8b971b8f..36207dba0 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -132,7 +132,7 @@ class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore POSSIBLE_NAMES: Dict[str, '_Constant'] = NotImplemented def __init__(self, name): - super(_Constant, self).__init__() + super().__init__() self.POSSIBLE_NAMES[name] = self # pylint: disable=unsupported-assignment-operation self.name = name @@ -201,7 +201,7 @@ class Directory(jose.JSONDeSerializable): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(Directory.Meta, self).__init__(**kwargs) + super().__init__(**kwargs) @property def terms_of_service(self): @@ -211,7 +211,7 @@ class Directory(jose.JSONDeSerializable): def __iter__(self): # When iterating over fields, use the external name 'terms_of_service' instead of # the internal '_terms_of_service'. - for name in super(Directory.Meta, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_terms_of_service' else name def _internal_name(self, name): @@ -357,7 +357,7 @@ class Registration(ResourceBody): if 'contact' in kwargs: # Avoid the __setattr__ used by jose.TypedJSONObjectWithFields object.__setattr__(self, '_add_contact', True) - super(Registration, self).__init__(**kwargs) + super().__init__(**kwargs) def _filter_contact(self, prefix): return tuple( @@ -383,12 +383,12 @@ class Registration(ResourceBody): def to_partial_json(self): """Modify josepy.JSONDeserializable.to_partial_json()""" - jobj = super(Registration, self).to_partial_json() + jobj = super().to_partial_json() return self._add_contact_if_appropriate(jobj) def fields_to_partial_json(self): """Modify josepy.JSONObjectWithFields.fields_to_partial_json()""" - jobj = super(Registration, self).fields_to_partial_json() + jobj = super().fields_to_partial_json() return self._add_contact_if_appropriate(jobj) @property @@ -460,19 +460,19 @@ class ChallengeBody(ResourceBody): def __init__(self, **kwargs): kwargs = {self._internal_name(k): v for k, v in kwargs.items()} - super(ChallengeBody, self).__init__(**kwargs) + super().__init__(**kwargs) def encode(self, name): - return super(ChallengeBody, self).encode(self._internal_name(name)) + return super().encode(self._internal_name(name)) def to_partial_json(self): - jobj = super(ChallengeBody, self).to_partial_json() + jobj = super().to_partial_json() jobj.update(self.chall.to_partial_json()) return jobj @classmethod def fields_from_json(cls, jobj): - jobj_fields = super(ChallengeBody, cls).fields_from_json(jobj) + jobj_fields = super().fields_from_json(jobj) jobj_fields['chall'] = challenges.Challenge.from_json(jobj) return jobj_fields @@ -487,7 +487,7 @@ class ChallengeBody(ResourceBody): def __iter__(self): # When iterating over fields, use the external name 'uri' instead of # the internal '_uri'. - for name in super(ChallengeBody, self).__iter__(): + for name in super().__iter__(): yield name[1:] if name == '_uri' else name def _internal_name(self, name): diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py index 0e1e0977c..6d58e0889 100644 --- a/acme/acme/mixins.py +++ b/acme/acme/mixins.py @@ -20,7 +20,7 @@ class VersionedLEACMEMixin: # Required for @property to operate properly. See comment above. object.__setattr__(self, key, value) else: - super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + super().__setattr__(key, value) # pragma: no cover class ResourceMixin(VersionedLEACMEMixin): @@ -30,12 +30,12 @@ class ResourceMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'resource') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(ResourceMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'resource') @@ -46,12 +46,12 @@ class TypeMixin(VersionedLEACMEMixin): """ def to_partial_json(self): """See josepy.JSONDeserializable.to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'to_partial_json', 'type') def fields_to_partial_json(self): """See josepy.JSONObjectWithFields.fields_to_partial_json()""" - return _safe_jobj_compliance(super(TypeMixin, self), + return _safe_jobj_compliance(super(), 'fields_to_partial_json', 'type') diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 6addb09f3..89e66c6d6 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -90,7 +90,7 @@ class BackwardsCompatibleClientV2Test(ClientTestBase): """Tests for acme.client.BackwardsCompatibleClientV2.""" def setUp(self): - super(BackwardsCompatibleClientV2Test, self).setUp() + super().setUp() # contains a loaded cert self.certr = messages.CertificateResource( body=messages_test.CERT) @@ -319,7 +319,7 @@ class ClientTest(ClientTestBase): """Tests for acme.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.directory = DIRECTORY_V1 @@ -716,7 +716,7 @@ class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" def setUp(self): - super(ClientV2Test, self).setUp() + super().setUp() self.directory = DIRECTORY_V2 diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index adfbc4442..d3bd1a4bf 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -15,7 +15,7 @@ class ApacheParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty @@ -39,7 +39,7 @@ class ApacheCommentNode(ApacheParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(ApacheCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.comment = comment def __eq__(self, other): # pragma: no cover @@ -57,7 +57,7 @@ class ApacheDirectiveNode(ApacheParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(ApacheDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.parameters = parameters self.enabled = enabled @@ -83,7 +83,7 @@ class ApacheBlockNode(ApacheDirectiveNode): """ apacheconfig implementation of BlockNode interface """ def __init__(self, **kwargs): - super(ApacheBlockNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.children: Tuple[ApacheParserNode, ...] = () def __eq__(self, other): # pragma: no cover diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index 4549e935a..896e17cf8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -80,7 +80,7 @@ class AugeasParserNode(interfaces.ParserNode): def __init__(self, **kwargs): ancestor, dirty, filepath, metadata = util.parsernode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.ancestor = ancestor self.filepath = filepath self.dirty = dirty @@ -169,7 +169,7 @@ class AugeasCommentNode(AugeasParserNode): def __init__(self, **kwargs): comment, kwargs = util.commentnode_kwargs(kwargs) # pylint: disable=unused-variable - super(AugeasCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) # self.comment = comment self.comment = comment @@ -188,7 +188,7 @@ class AugeasDirectiveNode(AugeasParserNode): def __init__(self, **kwargs): name, parameters, enabled, kwargs = util.directivenode_kwargs(kwargs) - super(AugeasDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.name = name self.enabled = enabled if parameters: @@ -245,7 +245,7 @@ class AugeasBlockNode(AugeasDirectiveNode): """ Augeas implementation of BlockNode interface """ def __init__(self, **kwargs): - super(AugeasBlockNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.children = () def __eq__(self, other): diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 28f73b32d..5ef9083e6 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -214,7 +214,7 @@ class ApacheConfigurator(common.Installer): version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) openssl_version = kwargs.pop("openssl_version", None) - super(ApacheConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Add name_server association dict self.assoc: Dict[str, obj.VirtualHost] = {} @@ -410,7 +410,7 @@ class ApacheConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(ApacheConfigurator, self).recovery_routine() + super().recovery_routine() # Reload configuration after these changes take effect if needed # ie. ApacheParser has been initialized. if hasattr(self, "parser"): @@ -435,7 +435,7 @@ class ApacheConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(ApacheConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.parser.aug.load() def _verify_exe_availability(self, exe): diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 66d888e97..83a1a8e08 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -47,7 +47,7 @@ class ApacheHttp01(common.ChallengePerformer): """ def __init__(self, *args, **kwargs): - super(ApacheHttp01, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.challenge_conf_pre = os.path.join( self.configurator.conf("challenge-location"), "le_http_01_challenge_pre.conf") diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 38c21b785..106e2778a 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -238,7 +238,7 @@ class CommentNode(ParserNode, metaclass=abc.ABCMeta): created or changed after the last save. Default: False. :type dirty: bool """ - super(CommentNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover @@ -302,7 +302,7 @@ class DirectiveNode(ParserNode, metaclass=abc.ABCMeta): :type enabled: bool """ - super(DirectiveNode, self).__init__(ancestor=kwargs['ancestor'], + super().__init__(ancestor=kwargs['ancestor'], dirty=kwargs.get('dirty', False), filepath=kwargs['filepath'], metadata=kwargs.get('metadata', {})) # pragma: no cover diff --git a/certbot-apache/certbot_apache/_internal/obj.py b/certbot-apache/certbot_apache/_internal/obj.py index 21fd042f6..9001a860d 100644 --- a/certbot-apache/certbot_apache/_internal/obj.py +++ b/certbot-apache/certbot_apache/_internal/obj.py @@ -26,7 +26,7 @@ class Addr(common.Addr): def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ if __eq__ or # __cmp__ is overridden. See https://bugs.python.org/issue2235 - return super(Addr, self).__hash__() + return super().__hash__() def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index bd9a47b52..98dc80e0b 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -51,7 +51,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): fedora = os_info[0].lower() == "fedora" try: - super(CentOSConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: if fedora: self._try_restart_fedora() @@ -69,14 +69,14 @@ class CentOSConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(CentOSConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ Override the options dictionary initialization in order to support alternative restart cmd used in CentOS. """ - super(CentOSConfigurator, self)._prepare_options() + super()._prepare_options() cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") def get_parser(self): @@ -91,7 +91,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): has "LoadModule ssl_module..." before parsing the VirtualHost configuration that was created by Certbot """ - super(CentOSConfigurator, self)._deploy_cert(*args, **kwargs) + super()._deploy_cert(*args, **kwargs) if self.version < (2, 4, 0): self._deploy_loadmodule_ssl_if_needed() @@ -169,12 +169,12 @@ class CentOSParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # CentOS specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(CentOSParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(CentOSParser, self).update_runtime_variables() + super().update_runtime_variables() self.parse_sysconfig_var() def parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 9f938046b..c5f3b6e1a 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -58,7 +58,7 @@ class DebianConfigurator(configurator.ApacheConfigurator): if not os.path.isdir(os.path.dirname(enabled_path)): # For some reason, sites-enabled / sites-available do not exist # Call the parent method - return super(DebianConfigurator, self).enable_site(vhost) + return super().enable_site(vhost) self.reverter.register_file_creation(False, enabled_path) try: os.symlink(vhost.filep, enabled_path) diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 0dc3df66b..0f7970460 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -43,7 +43,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): during the first (re)start of httpd. """ try: - super(FedoraConfigurator, self).config_test() + super().config_test() except errors.MisconfigurationError: self._try_restart_fedora() @@ -63,7 +63,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): raise errors.MisconfigurationError(str(err)) # Finish with actual config check to see if systemctl restart helped - super(FedoraConfigurator, self).config_test() + super().config_test() def _prepare_options(self): """ @@ -71,7 +71,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): instead of httpd and so take advantages of this new bash script in newer versions of Fedora to restart httpd. """ - super(FedoraConfigurator, self)._prepare_options() + super()._prepare_options() cast(List[str], self.options["restart_cmd"])[0] = 'apachectl' cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl' cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl' @@ -82,12 +82,12 @@ class FedoraParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Fedora 29+ specific configuration file for Apache self.sysconfig_filep = "/etc/sysconfig/httpd" - super(FedoraParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ # Opportunistic, works if SELinux not enforced - super(FedoraParser, self).update_runtime_variables() + super().update_runtime_variables() self._parse_sysconfig_var() def _parse_sysconfig_var(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index 773fdd568..484e15532 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -38,7 +38,7 @@ class GentooConfigurator(configurator.ApacheConfigurator): Override the options dictionary initialization in order to support alternative restart cmd used in Gentoo. """ - super(GentooConfigurator, self)._prepare_options() + super()._prepare_options() cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") def get_parser(self): @@ -53,7 +53,7 @@ class GentooParser(parser.ApacheParser): def __init__(self, *args, **kwargs): # Gentoo specific configuration file for Apache2 self.apacheconfig_filep = "/etc/conf.d/apache2" - super(GentooParser, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def update_runtime_variables(self): """ Override for update_runtime_variables for custom parsing """ diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 42126a5a9..85a17ab31 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -29,7 +29,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(AugeasParserNodeTest, self).setUp() + super().setUp() with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: mock_parsernode.side_effect = _get_augeasnode_mock( diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index db1c46b52..8c8ba4873 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -18,7 +18,7 @@ class AutoHSTSTest(util.ApacheTest): # pylint: disable=protected-access def setUp(self): # pylint: disable=arguments-differ - super(AutoHSTSTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 27b4f8e80..bfbe22ad0 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -36,9 +36,9 @@ class CentOS6Tests(util.ApacheTest): test_dir = "centos6_apache/apache" config_root = "centos6_apache/apache/httpd" vhost_root = "centos6_apache/apache/httpd/conf.d" - super(CentOS6Tests, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index b7e9c1cb6..a92c37979 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -41,9 +41,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora_old") @@ -96,9 +96,9 @@ class MultipleVhostsTestCentOS(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestCentOS, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/complex_parsing_test.py b/certbot-apache/tests/complex_parsing_test.py index 8b795b0b6..e36bd85d1 100644 --- a/certbot-apache/tests/complex_parsing_test.py +++ b/certbot-apache/tests/complex_parsing_test.py @@ -11,7 +11,7 @@ class ComplexParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(ComplexParserTest, self).setUp( + super().setUp( "complex_parsing", "complex_parsing") self.setup_variables() diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index 8596195d8..d8f5ddd05 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -16,7 +16,7 @@ class ConfiguratorReverterTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorReverterTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index 302bf0023..ad1f5f04d 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -30,7 +30,7 @@ class MultipleVhostsTest(util.ApacheTest): """Test two standard well-configured HTTP vhosts.""" def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) @@ -1477,9 +1477,9 @@ class AugeasVhostsTest(util.ApacheTest): td = "debian_apache_2_4/augeas_vhosts" cr = "debian_apache_2_4/augeas_vhosts/apache2" vr = "debian_apache_2_4/augeas_vhosts/apache2/sites-available" - super(AugeasVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, @@ -1556,9 +1556,9 @@ class MultiVhostsTest(util.ApacheTest): td = "debian_apache_2_4/multi_vhosts" cr = "debian_apache_2_4/multi_vhosts/apache2" vr = "debian_apache_2_4/multi_vhosts/apache2/sites-available" - super(MultiVhostsTest, self).setUp(test_dir=td, - config_root=cr, - vhost_root=vr) + super().setUp(test_dir=td, + config_root=cr, + vhost_root=vr) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, @@ -1661,7 +1661,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): # pylint: disable=arguments-differ - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 192e3cba5..c72b8b6ae 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): _multiprocess_can_split_ = True def setUp(self): # pylint: disable=arguments-differ - super(MultipleVhostsTestDebian, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="debian") diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index 50831802b..d48559cee 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -46,9 +46,9 @@ class FedoraRestartTest(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(FedoraRestartTest, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, os_info="fedora") @@ -90,9 +90,9 @@ class MultipleVhostsTestFedora(util.ApacheTest): test_dir = "centos7_apache/apache" config_root = "centos7_apache/apache/httpd" vhost_root = "centos7_apache/apache/httpd/conf.d" - super(MultipleVhostsTestFedora, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir, diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index 64f7d1062..cf39ff5cb 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -50,9 +50,9 @@ class MultipleVhostsTestGentoo(util.ApacheTest): test_dir = "gentoo_apache/apache" config_root = "gentoo_apache/apache/apache2" vhost_root = "gentoo_apache/apache/apache2/vhosts.d" - super(MultipleVhostsTestGentoo, self).setUp(test_dir=test_dir, - config_root=config_root, - vhost_root=vhost_root) + super().setUp(test_dir=test_dir, + config_root=config_root, + vhost_root=vhost_root) # pylint: disable=line-too-long with mock.patch("certbot_apache._internal.override_gentoo.GentooParser.update_runtime_variables"): diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 9d66c3aa0..71f2db500 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -24,7 +24,7 @@ class ApacheHttp01Test(util.ApacheTest): """Test for certbot_apache._internal.http_01.ApacheHttp01.""" def setUp(self, *args, **kwargs): # pylint: disable=arguments-differ - super(ApacheHttp01Test, self).setUp(*args, **kwargs) + super().setUp(*args, **kwargs) self.account_key = self.rsa512jwk self.achalls: List[achallenges.KeyAuthorizationAnnotatedChallenge] = [] diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 2eff9a9dd..37d4eb782 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -16,7 +16,7 @@ class BasicParserTest(util.ParserTest): """Apache Parser Test.""" def setUp(self): # pylint: disable=arguments-differ - super(BasicParserTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) @@ -332,7 +332,7 @@ class BasicParserTest(util.ParserTest): class ParserInitTest(util.ApacheTest): def setUp(self): # pylint: disable=arguments-differ - super(ParserInitTest, self).setUp() + super().setUp() def tearDown(self): shutil.rmtree(self.temp_dir) diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 7fbec2540..411871a43 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -20,7 +20,7 @@ class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-p """Test AugeasParserNode using available test configurations""" def setUp(self): # pylint: disable=arguments-differ - super(ConfiguratorParserNodeTest, self).setUp() + super().setUp() self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, diff --git a/certbot-apache/tests/parsernode_test.py b/certbot-apache/tests/parsernode_test.py index a86952f53..4ea5f8415 100644 --- a/certbot-apache/tests/parsernode_test.py +++ b/certbot-apache/tests/parsernode_test.py @@ -18,7 +18,7 @@ class DummyParserNode(interfaces.ParserNode): self.dirty = dirty self.filepath = filepath self.metadata = metadata - super(DummyParserNode, self).__init__(**kwargs) + super().__init__(**kwargs) def save(self, msg): # pragma: no cover """Save""" @@ -38,7 +38,7 @@ class DummyCommentNode(DummyParserNode): """ comment, kwargs = util.commentnode_kwargs(kwargs) self.comment = comment - super(DummyCommentNode, self).__init__(**kwargs) + super().__init__(**kwargs) class DummyDirectiveNode(DummyParserNode): @@ -54,7 +54,7 @@ class DummyDirectiveNode(DummyParserNode): self.parameters = parameters self.enabled = enabled - super(DummyDirectiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) def set_parameters(self, parameters): # pragma: no cover """Set parameters""" diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index 18c7e5aca..bd522d736 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -67,7 +67,7 @@ class ParserTest(ApacheTest): def setUp(self, test_dir="debian_apache_2_4/multiple_vhosts", config_root="debian_apache_2_4/multiple_vhosts/apache2", vhost_root="debian_apache_2_4/multiple_vhosts/apache2/sites-available"): - super(ParserTest, self).setUp(test_dir, config_root, vhost_root) + super().setUp(test_dir, config_root, vhost_root) zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py index 6f0f833a0..3ee1766a1 100644 --- a/certbot-ci/certbot_integration_tests/nginx_tests/context.py +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -11,7 +11,7 @@ from certbot_integration_tests.utils import misc class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """General fixture describing a certbot-nginx integration tests context""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.nginx_root = os.path.join(self.workspace, 'nginx') os.mkdir(self.nginx_root) @@ -29,7 +29,7 @@ class IntegrationTestsContext(certbot_context.IntegrationTestsContext): def cleanup(self): self._stop_nginx() - super(IntegrationTestsContext, self).cleanup() + super().cleanup() def certbot_test_nginx(self, args): """ diff --git a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py index a4f62b5e4..3d4147313 100644 --- a/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py +++ b/certbot-ci/certbot_integration_tests/rfc2136_tests/context.py @@ -13,7 +13,7 @@ from certbot_integration_tests.utils import certbot_call class IntegrationTestsContext(certbot_context.IntegrationTestsContext): """Integration test context for certbot-dns-rfc2136""" def __init__(self, request): - super(IntegrationTestsContext, self).__init__(request) + super().__init__(request) self.request = request diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 2b3f94581..f62635610 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -22,7 +22,7 @@ class Proxy(configurators_common.Proxy): def __init__(self, args): """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) + super().__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" self.modules = self.server_root = self.test_conf = self.version = None @@ -34,7 +34,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) @@ -65,7 +65,7 @@ class Proxy(configurators_common.Proxy): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - super(Proxy, self).cleanup_from_tests() + super().cleanup_from_tests() mock.patch.stopall() diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py index d7b7a120e..e209480e3 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/nginx/common.py @@ -21,7 +21,7 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - config = super(Proxy, self).load_config() + config = super().load_config() self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py index ceee39f86..8f42b3ce9 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/_internal/dns_cloudflare.py @@ -31,12 +31,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='Cloudflare credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index e9adf4ed9..d7075a84d 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -27,7 +27,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_cloudflare._internal.dns_cloudflare import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"cloudflare_email": EMAIL, "cloudflare_api_key": API_KEY}, path) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py index 7fe52b618..1ec5cb5ab 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/_internal/dns_cloudxns.py @@ -28,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='CloudXNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -71,7 +71,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_CloudXNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('cloudxns', { 'ttl': ttl, diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index 43c69790f..81dea5ca4 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -26,7 +26,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_cloudxns._internal.dns_cloudxns import Authenticator diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py index 85eb1159e..23b669847 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/_internal/dns_digitalocean.py @@ -26,12 +26,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 30 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add) + super().add_parser_arguments(add) add('credentials', help='DigitalOcean credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index 84bb35b1f..6088262ee 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_digitalocean._internal.dns_digitalocean import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"digitalocean_token": TOKEN}, path) diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py index 318bc25fb..858ee8925 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/_internal/dns_dnsimple.py @@ -28,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='DNSimple credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -67,7 +67,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, token, ttl): - super(_DNSimpleLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index 40eba4754..fc3dc5b1f 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -20,7 +20,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsimple._internal.dns_dnsimple import Authenticator diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py index 3a805f507..67903e19d 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/_internal/dns_dnsmadeeasy.py @@ -29,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='DNS Made Easy credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -73,7 +73,7 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, secret_key, ttl): - super(_DNSMadeEasyLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { 'ttl': ttl, diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index 4a69e977c..a04716d95 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_dnsmadeeasy._internal.dns_dnsmadeeasy import Authenticator diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py index dd9665c2f..57ff01671 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/_internal/dns_gehirn.py @@ -29,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='Gehirn Infrastructure Service credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -75,7 +75,7 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_GehirnLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('gehirn', { 'ttl': ttl, @@ -89,4 +89,4 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_GehirnLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index 0598a5eb5..1310f74ca 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_gehirn._internal.dns_gehirn import Authenticator diff --git a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py index 39f26c7d0..3a2686a63 100644 --- a/certbot-dns-google/certbot_dns_google/_internal/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/_internal/dns_google.py @@ -34,7 +34,7 @@ class Authenticator(dns_common.DNSAuthenticator): @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help=('Path to Google Cloud DNS service account JSON file. (See {0} for' + 'information about creating a service account and {1} for information about the' + diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 7de5f1d67..aa3ff35b5 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -26,14 +26,14 @@ PROJECT_ID = "test-test-1" class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_google._internal.dns_google import Authenticator path = os.path.join(self.tempdir, 'file.json') open(path, "wb").close() - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock(google_credentials=path, google_propagation_seconds=0) # don't wait during tests diff --git a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py index c9e6a42b5..b1649cf61 100644 --- a/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/_internal/dns_linode.py @@ -29,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): description = 'Obtain certificates using a DNS TXT record (if you are using Linode for DNS).' def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=120) + super().add_parser_arguments(add, default_propagation_seconds=120) add('credentials', help='Linode credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -85,7 +85,7 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, api_version): - super(_LinodeLexiconClient, self).__init__() + super().__init__() self.api_version = api_version diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index fb9b1aa93..9861d2ab0 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -22,7 +22,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write({"linode_key": TOKEN}, path) diff --git a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py index 6595c3165..ed90b63d9 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/_internal/dns_luadns.py @@ -28,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='LuaDNS credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -70,7 +70,7 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, email, token, ttl): - super(_LuaDNSLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('luadns', { 'ttl': ttl, diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index a1242582f..7592e2323 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_luadns._internal.dns_luadns import Authenticator diff --git a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py index dc4a0d0ee..ce46ad835 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/_internal/dns_nsone.py @@ -28,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='NS1 credentials file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -67,7 +67,7 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_key, ttl): - super(_NS1LexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index 83371252f..3754f9811 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_nsone._internal.dns_nsone import Authenticator diff --git a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py index 13f5b2805..54fd6d791 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/_internal/dns_ovh.py @@ -28,12 +28,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + super().add_parser_arguments(add, default_propagation_seconds=30) add('credentials', help='OVH credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring @@ -79,7 +79,7 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): - super(_OVHLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('ovh', { 'ttl': ttl, @@ -106,4 +106,4 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): if domain_name in str(e) and str(e).endswith('not found'): return - super(_OVHLexiconClient, self)._handle_general_error(e, domain_name) + super()._handle_general_error(e, domain_name) diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index dd0f3058b..7f93967eb 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -23,7 +23,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_ovh._internal.dns_ovh import Authenticator diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py index a3ee1e0e3..28fd27eec 100644 --- a/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py +++ b/certbot-dns-rfc2136/certbot_dns_rfc2136/_internal/dns_rfc2136.py @@ -45,12 +45,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 120 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + super().add_parser_arguments(add, default_propagation_seconds=60) add('credentials', help='RFC 2136 credentials INI file.') def more_info(self): # pylint: disable=missing-function-docstring diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index dc4a73a04..c2b80defe 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -28,7 +28,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic def setUp(self): from certbot_dns_rfc2136._internal.dns_rfc2136 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() path = os.path.join(self.tempdir, 'file.ini') dns_test_common.write(VALID_CONFIG, path) diff --git a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py index 4dda13f1d..9e470bdde 100644 --- a/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py +++ b/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py @@ -37,7 +37,7 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 10 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.r53 = boto3.client("route53") self._resource_records: DefaultDict[str, List[Dict[str, str]]] = collections.defaultdict(list) diff --git a/certbot-dns-route53/certbot_dns_route53/authenticator.py b/certbot-dns-route53/certbot_dns_route53/authenticator.py index 2987934a1..060d2fa38 100644 --- a/certbot-dns-route53/certbot_dns_route53/authenticator.py +++ b/certbot-dns-route53/certbot_dns_route53/authenticator.py @@ -18,4 +18,4 @@ class Authenticator(dns_route53.Authenticator): def __init__(self, *args, **kwargs): warnings.warn("The 'authenticator' module was renamed 'dns_route53'", DeprecationWarning) - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 1fd191c69..69b6b115d 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(unittest.TestCase, dns_test_common.BaseAuthenticatorTest def setUp(self): from certbot_dns_route53._internal.dns_route53 import Authenticator - super(AuthenticatorTest, self).setUp() + super().setUp() self.config = mock.MagicMock() diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py index eb78735f5..668c5e1f8 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/_internal/dns_sakuracloud.py @@ -29,12 +29,12 @@ class Authenticator(dns_common.DNSAuthenticator): ttl = 60 def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.credentials: Optional[CredentialsConfiguration] = None @classmethod def add_parser_arguments(cls, add): # pylint: disable=arguments-differ - super(Authenticator, cls).add_parser_arguments( + super().add_parser_arguments( add, default_propagation_seconds=90) add('credentials', help='Sakura Cloud credentials file.') @@ -78,7 +78,7 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): """ def __init__(self, api_token, api_secret, ttl): - super(_SakuraCloudLexiconClient, self).__init__() + super().__init__() config = dns_common_lexicon.build_lexicon_config('sakuracloud', { 'ttl': ttl, @@ -92,4 +92,4 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return None # Expected errors when zone name guess is wrong - return super(_SakuraCloudLexiconClient, self)._handle_http_error(e, domain_name) + return super()._handle_http_error(e, domain_name) diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index af94336b3..1c64df372 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -21,7 +21,7 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common_lexicon.BaseLexiconAuthenticatorTest): def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() from certbot_dns_sakuracloud._internal.dns_sakuracloud import Authenticator diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index c05ddceba..9fb81efaa 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -102,7 +102,7 @@ class NginxConfigurator(common.Installer): """ version = kwargs.pop("version", None) openssl_version = kwargs.pop("openssl_version", None) - super(NginxConfigurator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Files to save self.save_notes = "" @@ -1110,7 +1110,7 @@ class NginxConfigurator(common.Installer): :raises .errors.PluginError: If unable to recover the configuration """ - super(NginxConfigurator, self).recovery_routine() + super().recovery_routine() self.new_vhost = None self.parser.load() @@ -1133,7 +1133,7 @@ class NginxConfigurator(common.Installer): the function is unable to correctly revert the configuration """ - super(NginxConfigurator, self).rollback_checkpoints(rollback) + super().rollback_checkpoints(rollback) self.new_vhost = None self.parser.load() diff --git a/certbot-nginx/certbot_nginx/_internal/http_01.py b/certbot-nginx/certbot_nginx/_internal/http_01.py index abcdfbd53..716af0898 100644 --- a/certbot-nginx/certbot_nginx/_internal/http_01.py +++ b/certbot-nginx/certbot_nginx/_internal/http_01.py @@ -37,7 +37,7 @@ class NginxHttp01(common.ChallengePerformer): """ def __init__(self, configurator): - super(NginxHttp01, self).__init__(configurator) + super().__init__(configurator) self.challenge_conf = os.path.join( configurator.config.config_dir, "le_http_01_cert_challenge.conf") diff --git a/certbot-nginx/certbot_nginx/_internal/obj.py b/certbot-nginx/certbot_nginx/_internal/obj.py index f77e535ce..44be0e598 100644 --- a/certbot-nginx/certbot_nginx/_internal/obj.py +++ b/certbot-nginx/certbot_nginx/_internal/obj.py @@ -35,7 +35,7 @@ class Addr(common.Addr): CANONICAL_UNSPECIFIED_ADDRESS = UNSPECIFIED_IPV4_ADDRESSES[0] def __init__(self, host, port, ssl, default, ipv6, ipv6only): - super(Addr, self).__init__((host, port)) + super().__init__((host, port)) self.ssl = ssl self.default = default self.ipv6 = ipv6 @@ -120,7 +120,7 @@ class Addr(common.Addr): def __hash__(self): # pylint: disable=useless-super-delegation # Python 3 requires explicit overridden for __hash__ # See certbot-apache/certbot_apache/_internal/obj.py for more information - return super(Addr, self).__hash__() + return super().__hash__() def super_eq(self, other): """Check ip/port equality, with IPv6 support. @@ -132,7 +132,7 @@ class Addr(common.Addr): self.tup[1]), self.ipv6) == \ common.Addr((other.CANONICAL_UNSPECIFIED_ADDRESS, other.tup[1]), other.ipv6) - return super(Addr, self).__eq__(other) + return super().__eq__(other) def __eq__(self, other): if isinstance(other, self.__class__): diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 838a2e382..d4d332c47 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -122,7 +122,7 @@ class Statements(Parsable): precede any more statements. """ def __init__(self, parent=None): - super(Statements, self).__init__(parent) + super().__init__(parent) self._trailing_whitespace = None # ======== Begin overridden functions @@ -167,7 +167,7 @@ class Statements(Parsable): def dump(self, include_spaces=False): """ Dumps this object by first dumping each statement, then appending its trailing whitespace (if `include_spaces` is set) """ - data = super(Statements, self).dump(include_spaces) + data = super().dump(include_spaces) if include_spaces and self._trailing_whitespace is not None: return data + [self._trailing_whitespace] return data @@ -271,7 +271,7 @@ class Block(Parsable): contents = [["\n ", "content", " ", "1"], ["\n ", "content", " ", "2"], "\n"] """ def __init__(self, parent=None): - super(Block, self).__init__(parent) + super().__init__(parent) self.names: Sentence = None self.contents: Block = None diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 9ccc3fc9e..e6ceca344 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -26,7 +26,7 @@ class NginxConfiguratorTest(util.NginxTest): def setUp(self): - super(NginxConfiguratorTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) @@ -954,7 +954,7 @@ class InstallSslOptionsConfTest(util.NginxTest): """Test that the options-ssl-nginx.conf file is installed and updated properly.""" def setUp(self): - super(InstallSslOptionsConfTest, self).setUp() + super().setUp() self.config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) diff --git a/certbot-nginx/tests/display_ops_test.py b/certbot-nginx/tests/display_ops_test.py index 377255441..bcd455410 100644 --- a/certbot-nginx/tests/display_ops_test.py +++ b/certbot-nginx/tests/display_ops_test.py @@ -12,7 +12,7 @@ class SelectVhostMultiTest(util.NginxTest): """Tests for certbot_nginx._internal.display_ops.select_vhost_multiple.""" def setUp(self): - super(SelectVhostMultiTest, self).setUp() + super().setUp() nparser = parser.NginxParser(self.config_path) self.vhosts = nparser.get_vhosts() diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index d0e84fc83..f10e44859 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -51,7 +51,7 @@ class HttpPerformTest(util.NginxTest): ] def setUp(self): - super(HttpPerformTest, self).setUp() + super().setUp() config = self.get_nginx_configurator( self.config_path, self.config_dir, self.work_dir, self.logs_dir) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index ee53e8e1e..383a15753 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -22,7 +22,7 @@ from certbot_nginx._internal import nginxparser class NginxTest(test_util.ConfigTestCase): def setUp(self): - super(NginxTest, self).setUp() + super().setUp() self.configuration = self.config self.config = None diff --git a/certbot/certbot/_internal/cli/cli_utils.py b/certbot/certbot/_internal/cli/cli_utils.py index 1f30261c1..8060b5e21 100644 --- a/certbot/certbot/_internal/cli/cli_utils.py +++ b/certbot/certbot/_internal/cli/cli_utils.py @@ -140,7 +140,7 @@ class CaseInsensitiveList(list): through the `helpful` wrapper. It is necessary due to special handling of command line arguments by `set_by_cli` in which the `type_func` is not applied.""" def __contains__(self, element): - return super(CaseInsensitiveList, self).__contains__(element.lower()) + return super().__contains__(element.lower()) def _user_agent_comment_type(value): diff --git a/certbot/certbot/_internal/log.py b/certbot/certbot/_internal/log.py index fb0a996f7..7338578a7 100644 --- a/certbot/certbot/_internal/log.py +++ b/certbot/certbot/_internal/log.py @@ -171,7 +171,7 @@ class ColoredStreamHandler(logging.StreamHandler): """ def __init__(self, stream=None): - super(ColoredStreamHandler, self).__init__(stream) + super().__init__(stream) self.colored = (sys.stderr.isatty() if stream is None else stream.isatty()) self.red_level = logging.WARNING @@ -185,7 +185,7 @@ class ColoredStreamHandler(logging.StreamHandler): :rtype: str """ - out = super(ColoredStreamHandler, self).format(record) + out = super().format(record) if self.colored and record.levelno >= self.red_level: return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) return out @@ -200,14 +200,14 @@ class MemoryHandler(logging.handlers.MemoryHandler): """ def __init__(self, target=None, capacity=10000): # capacity doesn't matter because should_flush() is overridden - super(MemoryHandler, self).__init__(capacity, target=target) + super().__init__(capacity, target=target) def close(self): """Close the memory handler, but don't set the target to None.""" # This allows the logging module which may only have a weak # reference to the target handler to properly flush and close it. target = getattr(self, 'target') - super(MemoryHandler, self).close() + super().close() self.target = target def flush(self, force=False): # pylint: disable=arguments-differ @@ -221,7 +221,7 @@ class MemoryHandler(logging.handlers.MemoryHandler): # This method allows flush() calls in logging.shutdown to be a # noop so we can control when this handler is flushed. if force: - super(MemoryHandler, self).flush() + super().flush() def shouldFlush(self, record): """Should the buffer be automatically flushed? @@ -249,7 +249,7 @@ class TempHandler(logging.StreamHandler): self._workdir = tempfile.mkdtemp() self.path = os.path.join(self._workdir, 'log') stream = util.safe_open(self.path, mode='w', chmod=0o600) - super(TempHandler, self).__init__(stream) + super().__init__(stream) self._delete = True def emit(self, record): @@ -259,7 +259,7 @@ class TempHandler(logging.StreamHandler): """ self._delete = False - super(TempHandler, self).emit(record) + super().emit(record) def close(self): """Close the handler and the temporary log file. @@ -275,7 +275,7 @@ class TempHandler(logging.StreamHandler): if self._delete: shutil.rmtree(self._workdir) self._delete = False - super(TempHandler, self).close() + super().close() finally: self.release() diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index ed2e0559e..5423ec1d9 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -71,7 +71,7 @@ permitted by DNS standards.) """ def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() self.env: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] = {} diff --git a/certbot/certbot/_internal/plugins/standalone.py b/certbot/certbot/_internal/plugins/standalone.py index c75ba36a9..5fb29671f 100644 --- a/certbot/certbot/_internal/plugins/standalone.py +++ b/certbot/certbot/_internal/plugins/standalone.py @@ -119,7 +119,7 @@ class Authenticator(common.Plugin): description = "Spin up a temporary webserver" def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.served: ServedType = collections.defaultdict(set) diff --git a/certbot/certbot/_internal/plugins/webroot.py b/certbot/certbot/_internal/plugins/webroot.py index c5b436b65..3473d3c8e 100644 --- a/certbot/certbot/_internal/plugins/webroot.py +++ b/certbot/certbot/_internal/plugins/webroot.py @@ -66,7 +66,7 @@ to serve all files under specified web root ({0}).""" return [challenges.HTTP01] def __init__(self, *args, **kwargs): - super(Authenticator, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.full_roots: Dict[str, str] = {} self.performed: DefaultDict[str, Set[AnnotatedChallenge]] = collections.defaultdict(set) # stack of dirs successfully created by this authenticator @@ -250,7 +250,7 @@ class _WebrootPathAction(argparse.Action): """Action class for parsing webroot_path.""" def __init__(self, *args, **kwargs): - super(_WebrootPathAction, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._domain_before_webroot = False def __call__(self, parser, namespace, webroot_path, option_string=None): diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index 1ba53cad4..f7832cd55 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -80,7 +80,7 @@ def prepare_env(cli_args: List[str]) -> List[str]: class _SnapdConnection(HTTPConnection): def __init__(self): - super(_SnapdConnection, self).__init__("localhost") + super().__init__("localhost") self.sock = None def connect(self): @@ -90,7 +90,7 @@ class _SnapdConnection(HTTPConnection): class _SnapdConnectionPool(HTTPConnectionPool): def __init__(self): - super(_SnapdConnectionPool, self).__init__("localhost") + super().__init__("localhost") def _new_conn(self): return _SnapdConnection() diff --git a/certbot/certbot/display/util.py b/certbot/certbot/display/util.py index 9443ae266..dc642586c 100644 --- a/certbot/certbot/display/util.py +++ b/certbot/certbot/display/util.py @@ -115,7 +115,7 @@ class FileDisplay: # see https://github.com/certbot/certbot/issues/3915 def __init__(self, outfile, force_interactive): - super(FileDisplay, self).__init__() + super().__init__() self.outfile = outfile self.force_interactive = force_interactive self.skipped_interaction = False @@ -481,7 +481,7 @@ class NoninteractiveDisplay: """An iDisplay implementation that never asks for interactive user input""" def __init__(self, outfile, *unused_args, **unused_kwargs): - super(NoninteractiveDisplay, self).__init__() + super().__init__() self.outfile = outfile def _interaction_fail(self, message, cli_flag, extra=""): diff --git a/certbot/certbot/errors.py b/certbot/certbot/errors.py index 48aebc267..cf0d3d283 100644 --- a/certbot/certbot/errors.py +++ b/certbot/certbot/errors.py @@ -53,7 +53,7 @@ class FailedChallenges(AuthorizationError): def __init__(self, failed_achalls): assert failed_achalls self.failed_achalls = failed_achalls - super(FailedChallenges, self).__init__() + super().__init__() def __str__(self): return "Failed authorization procedure. {0}".format( @@ -95,7 +95,7 @@ class StandaloneBindError(Error): """Standalone plugin bind error.""" def __init__(self, socket_error, port): - super(StandaloneBindError, self).__init__( + super().__init__( "Problem binding to port {0}: {1}".format(port, socket_error)) self.socket_error = socket_error self.port = port diff --git a/certbot/certbot/plugins/common.py b/certbot/certbot/plugins/common.py index 06acefc69..4c33acbab 100644 --- a/certbot/certbot/plugins/common.py +++ b/certbot/certbot/plugins/common.py @@ -105,7 +105,7 @@ class Installer(Plugin): """ def __init__(self, *args, **kwargs): - super(Installer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.storage = PluginStorage(self.config, self.name) self.reverter = reverter.Reverter(self.config) diff --git a/certbot/certbot/plugins/dns_common.py b/certbot/certbot/plugins/dns_common.py index 4b621ceac..4459ac0fa 100644 --- a/certbot/certbot/plugins/dns_common.py +++ b/certbot/certbot/plugins/dns_common.py @@ -25,7 +25,7 @@ class DNSAuthenticator(common.Plugin): """Base class for DNS Authenticators""" def __init__(self, config, name): - super(DNSAuthenticator, self).__init__(config, name) + super().__init__(config, name) self._attempt_cleanup = False diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 78236fa06..b6a27a7c8 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -331,7 +331,7 @@ class TempDirTestCase(unittest.TestCase): class ConfigTestCase(TempDirTestCase): """Test class which sets up a NamespaceConfig object.""" def setUp(self): - super(ConfigTestCase, self).setUp() + super().setUp() self.config = configuration.NamespaceConfig( mock.MagicMock(**constants.CLI_DEFAULTS) ) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index d0448a1db..7c8f52273 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -106,7 +106,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): """Tests for certbot._internal.account.AccountFileStorage.""" def setUp(self): - super(AccountFileStorageTest, self).setUp() + super().setUp() from certbot._internal.account import AccountFileStorage self.storage = AccountFileStorage(self.config) diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index ba6cfddc3..918459256 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -26,7 +26,7 @@ class BaseCertManagerTest(test_util.ConfigTestCase): """Base class for setting up Cert Manager tests. """ def setUp(self): - super(BaseCertManagerTest, self).setUp() + super().setUp() self.config.quiet = False filesystem.makedirs(self.config.renewal_configs_dir) @@ -396,7 +396,7 @@ class RenameLineageTest(BaseCertManagerTest): """Tests for certbot._internal.cert_manager.rename_lineage""" def setUp(self): - super(RenameLineageTest, self).setUp() + super().setUp() self.config.certname = "example.org" self.config.new_certname = "after" @@ -483,7 +483,7 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): """Test to avoid duplicate lineages.""" def setUp(self): - super(DuplicativeCertsTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() @@ -521,7 +521,7 @@ class CertPathToLineageTest(storage_test.BaseRenewableCertTest): """Tests for certbot._internal.cert_manager.cert_path_to_lineage""" def setUp(self): - super(CertPathToLineageTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', @@ -581,7 +581,7 @@ class MatchAndCheckOverlaps(storage_test.BaseRenewableCertTest): archive dirs.""" # A test with real overlapping archive dirs can be found in tests/boulder_integration.sh def setUp(self): - super(MatchAndCheckOverlaps, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index f058cb658..2acaa71b1 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -58,7 +58,7 @@ class RegisterTest(test_util.ConfigTestCase): """Tests for certbot._internal.client.register.""" def setUp(self): - super(RegisterTest, self).setUp() + super().setUp() self.config.rsa_key_size = 1024 self.config.register_unsafely_without_email = False self.config.email = "alias@example.com" @@ -217,7 +217,7 @@ class ClientTestCommon(test_util.ConfigTestCase): """Common base class for certbot._internal.client.Client tests.""" def setUp(self): - super(ClientTestCommon, self).setUp() + super().setUp() self.config.no_verify_ssl = False self.config.allow_subset_of_names = False @@ -236,7 +236,7 @@ class ClientTest(ClientTestCommon): """Tests for certbot._internal.client.Client.""" def setUp(self): - super(ClientTest, self).setUp() + super().setUp() self.config.allow_subset_of_names = False self.config.dry_run = False @@ -582,7 +582,7 @@ class EnhanceConfigTest(ClientTestCommon): """Tests for certbot._internal.client.Client.enhance_config.""" def setUp(self): - super(EnhanceConfigTest, self).setUp() + super().setUp() self.config.hsts = False self.config.redirect = False diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index ea48c9d1c..47da2b415 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -34,7 +34,7 @@ ADMINS_SID = 'S-1-5-32-544' class WindowsChmodTests(TempDirTestCase): """Unit tests for Windows chmod function in filesystem module""" def setUp(self): - super(WindowsChmodTests, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): @@ -203,7 +203,7 @@ class UmaskTest(TempDirTestCase): class ComputePrivateKeyModeTest(TempDirTestCase): def setUp(self): - super(ComputePrivateKeyModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_compute_private_key_mode(self): @@ -351,7 +351,7 @@ class MakedirsTests(test_util.TempDirTestCase): class CopyOwnershipAndModeTest(test_util.TempDirTestCase): """Tests about copy_ownership_and_apply_mode, copy_ownership_and_mode and has_same_ownership""" def setUp(self): - super(CopyOwnershipAndModeTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') @@ -424,7 +424,7 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): class CheckPermissionsTest(test_util.TempDirTestCase): """Tests relative to functions that check modes.""" def setUp(self): - super(CheckPermissionsTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_check_mode(self): @@ -506,7 +506,7 @@ class OsReplaceTest(test_util.TempDirTestCase): class RealpathTest(test_util.TempDirTestCase): """Tests for realpath method""" def setUp(self): - super(RealpathTest, self).setUp() + super().setUp() self.probe_path = _create_probe(self.tempdir) def test_symlink_resolution(self): diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index 1f8a0e803..e23c50afb 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -17,7 +17,7 @@ class NamespaceConfigTest(test_util.ConfigTestCase): """Tests for certbot._internal.configuration.NamespaceConfig.""" def setUp(self): - super(NamespaceConfigTest, self).setUp() + super().setUp() self.config.foo = 'bar' # pylint: disable=blacklisted-name self.config.server = 'https://acme-server.org:443/new' self.config.https_port = 1234 diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 3b9c973f7..a2b89ee17 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -36,7 +36,7 @@ CERT_ALT_ISSUER = test_util.load_vector('cert_intermediate_2.pem') class InitSaveKeyTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_key.""" def setUp(self): - super(InitSaveKeyTest, self).setUp() + super().setUp() self.workdir = os.path.join(self.tempdir, 'workdir') filesystem.mkdir(self.workdir, mode=0o700) @@ -46,7 +46,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock.Mock(strict_permissions=True), interfaces.IConfig) def tearDown(self): - super(InitSaveKeyTest, self).tearDown() + super().tearDown() logging.disable(logging.NOTSET) @@ -73,7 +73,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): """Tests for certbot.crypto_util.init_save_csr.""" def setUp(self): - super(InitSaveCSRTest, self).setUp() + super().setUp() zope.component.provideUtility( mock.Mock(strict_permissions=True), interfaces.IConfig) diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 86f939491..087617310 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -25,7 +25,7 @@ class CompleterTest(test_util.TempDirTestCase): """Test certbot._internal.display.completer.Completer.""" def setUp(self): - super(CompleterTest, self).setUp() + super().setUp() # directories must end with os.sep for completer to # search inside the directory for possible completions diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index ce60a5f0b..1e50ba03b 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -90,7 +90,7 @@ class GetEmailTest(unittest.TestCase): class ChooseAccountTest(test_util.TempDirTestCase): """Tests for certbot.display.ops.choose_account.""" def setUp(self): - super(ChooseAccountTest, self).setUp() + super().setUp() zope.component.provideUtility(display_util.FileDisplay(sys.stdout, False)) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 30b33bbc9..5f1fac8c0 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -65,7 +65,7 @@ class FileOutputDisplayTest(unittest.TestCase): """ def setUp(self): - super(FileOutputDisplayTest, self).setUp() + super().setUp() self.mock_stdout = mock.MagicMock() self.displayer = display_util.FileDisplay(self.mock_stdout, False) diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index dc6b6b944..5da1e6fb6 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -22,7 +22,7 @@ _KEY = josepy.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) class SubscriptionTest(test_util.ConfigTestCase): """Abstract class for subscription tests.""" def setUp(self): - super(SubscriptionTest, self).setUp() + super().setUp() self.account = account.Account( regr=messages.RegistrationResource( uri=None, body=messages.Registration(), diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 521b6976b..e04fe0742 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -136,7 +136,7 @@ class ExitHandlerTest(ErrorHandlerTest): def setUp(self): from certbot._internal import error_handler - super(ExitHandlerTest, self).setUp() + super().setUp() self.handler = error_handler.ExitHandler(self.init_func, *self.init_args, **self.init_kwargs) diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index b0522bd52..76d460ae9 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -93,7 +93,7 @@ class PreHookTest(HookTest): return pre_hook(*args, **kwargs) def setUp(self): - super(PreHookTest, self).setUp() + super().setUp() self.config.pre_hook = "foo" filesystem.makedirs(self.config.renewal_pre_hooks_dir) @@ -106,7 +106,7 @@ class PreHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_pre_hook_already() - super(PreHookTest, self).tearDown() + super().tearDown() def _reset_pre_hook_already(self): from certbot._internal.hooks import executed_pre_hooks @@ -171,7 +171,7 @@ class PostHookTest(HookTest): return post_hook(*args, **kwargs) def setUp(self): - super(PostHookTest, self).setUp() + super().setUp() self.config.post_hook = "bar" filesystem.makedirs(self.config.renewal_post_hooks_dir) @@ -184,7 +184,7 @@ class PostHookTest(HookTest): def tearDown(self): # Reset this value so it's unmodified for future tests self._reset_post_hook_eventually() - super(PostHookTest, self).tearDown() + super().tearDown() def _reset_post_hook_eventually(self): from certbot._internal.hooks import post_hooks @@ -266,7 +266,7 @@ class RunSavedPostHooksTest(HookTest): return self._call_with_mock_execute(*args, **kwargs) def setUp(self): - super(RunSavedPostHooksTest, self).setUp() + super().setUp() self.eventually: List[str] = [] def test_empty(self): @@ -319,7 +319,7 @@ class RenewalHookTest(HookTest): return mock_execute def setUp(self): - super(RenewalHookTest, self).setUp() + super().setUp() self.vars_to_clear = set( var for var in ("RENEWED_DOMAINS", "RENEWED_LINEAGE",) if var not in os.environ) @@ -327,7 +327,7 @@ class RenewalHookTest(HookTest): def tearDown(self): for var in self.vars_to_clear: os.environ.pop(var, None) - super(RenewalHookTest, self).tearDown() + super().tearDown() class DeployHookTest(RenewalHookTest): @@ -373,7 +373,7 @@ class RenewHookTest(RenewalHookTest): return renew_hook(*args, **kwargs) def setUp(self): - super(RenewHookTest, self).setUp() + super().setUp() self.config.renew_hook = "foo" filesystem.makedirs(self.config.renewal_deploy_hooks_dir) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 2f887d33e..ac6b539a4 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -44,7 +44,7 @@ class LockFileTest(test_util.TempDirTestCase): return LockFile(*args, **kwargs) def setUp(self): - super(LockFileTest, self).setUp() + super().setUp() self.lock_path = os.path.join(self.tempdir, 'test.lock') def test_acquire_without_deletion(self): diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 5960b37cf..37660f51e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -70,7 +70,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): return post_arg_parse_setup(*args, **kwargs) def setUp(self): - super(PostArgParseSetupTest, self).setUp() + super().setUp() self.config.debug = False self.config.max_log_backups = 1000 self.config.quiet = False @@ -91,7 +91,7 @@ class PostArgParseSetupTest(test_util.ConfigTestCase): self.stream_handler.close() self.temp_handler.close() self.devnull.close() - super(PostArgParseSetupTest, self).tearDown() + super().tearDown() def test_common(self): with mock.patch('certbot._internal.log.logging.getLogger') as mock_get_logger: @@ -136,7 +136,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): return setup_log_file_handler(*args, **kwargs) def setUp(self): - super(SetupLogFileHandlerTest, self).setUp() + super().setUp() self.config.max_log_backups = 42 @mock.patch('certbot._internal.main.logging.handlers.RotatingFileHandler') diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index e093edd20..7908cf804 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -101,7 +101,7 @@ class RunTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.run.""" def setUp(self): - super(RunTest, self).setUp() + super().setUp() self.domain = 'example.org' patches = [ mock.patch('certbot._internal.main._get_and_save_cert'), @@ -283,7 +283,7 @@ class RevokeTest(test_util.TempDirTestCase): """Tests for certbot._internal.main.revoke.""" def setUp(self): - super(RevokeTest, self).setUp() + super().setUp() shutil.copy(CERT_PATH, self.tempdir) self.tmp_cert_path = os.path.abspath(os.path.join(self.tempdir, 'cert_512.pem')) @@ -512,7 +512,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._determine_account.""" def setUp(self): - super(DetermineAccountTest, self).setUp() + super().setUp() self.config.account = None self.config.email = None self.config.register_unsafely_without_email = False @@ -584,7 +584,7 @@ class MainTest(test_util.ConfigTestCase): """Tests for different commands.""" def setUp(self): - super(MainTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.logs_dir) self.standard_args = ['--config-dir', self.config.config_dir, @@ -597,7 +597,7 @@ class MainTest(test_util.ConfigTestCase): # Reset globals in cli reload_module(cli) - super(MainTest, self).tearDown() + super().tearDown() def _call(self, args, stdout=None, mockisfile=False): """Run the cli with output streams, actual client and optionally @@ -1576,7 +1576,7 @@ class EnhanceTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.enhance.""" def setUp(self): - super(EnhanceTest, self).setUp() + super().setUp() self.get_utility_patch = test_util.patch_get_utility() self.mock_get_utility = self.get_utility_patch.start() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @@ -1720,7 +1720,7 @@ class InstallTest(test_util.ConfigTestCase): """Tests for certbot._internal.main.install.""" def setUp(self): - super(InstallTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1765,7 +1765,7 @@ class UpdateAccountTest(test_util.ConfigTestCase): for patch in patches.values(): self.addCleanup(patch.stop) - return super(UpdateAccountTest, self).setUp() + return super().setUp() def _call(self, args): with mock.patch('certbot._internal.main.sys.stdout'), \ diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index bcd190e9b..7de3134fa 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -88,7 +88,7 @@ class InstallerTest(test_util.ConfigTestCase): """Tests for certbot.plugins.common.Installer.""" def setUp(self): - super(InstallerTest, self).setUp() + super().setUp() filesystem.mkdir(self.config.config_dir) from certbot.plugins.common import Installer @@ -282,7 +282,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): """Tests for certbot.plugins.common.install_version_controlled_file.""" def setUp(self): - super(InstallVersionControlledFileTest, self).setUp() + super().setUp() self.hashes = ["someotherhash"] self.dest_path = os.path.join(self.tempdir, "options-ssl-dest.conf") self.hash_path = os.path.join(self.tempdir, ".options-ssl-conf.txt") diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index a67430f3e..40afd107b 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -17,7 +17,7 @@ class LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconCl pass def setUp(self): - super(LexiconClientTest, self).setUp() + super().setUp() self.client = LexiconClientTest._FakeLexiconClient() self.provider_mock = mock.MagicMock() diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 31761e986..377244c4c 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -36,7 +36,7 @@ class DNSAuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthen fake_file_path = None def setUp(self): - super(DNSAuthenticatorTest, self).setUp() + super().setUp() self.config = DNSAuthenticatorTest._FakeConfig() @@ -164,7 +164,7 @@ class CredentialsConfigurationTest(test_util.TempDirTestCase): class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): def setUp(self): - super(CredentialsConfigurationRequireTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, 'file.ini') diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index a20a6864f..0aa1512b4 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -15,7 +15,7 @@ class EnhancementTest(test_util.ConfigTestCase): """Tests for new style enhancements in certbot.plugins.enhancements""" def setUp(self): - super(EnhancementTest, self).setUp() + super().setUp() self.mockinstaller = mock.MagicMock(spec=enhancements.AutoHSTSEnhancement) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 0e552e0c5..08ab20456 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -19,7 +19,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): """Tests for certbot._internal.plugins.manual.Authenticator.""" def setUp(self): - super(AuthenticatorTest, self).setUp() + super().setUp() self.http_achall = acme_util.HTTP01_A self.dns_achall = acme_util.DNS01_A self.dns_achall_2 = acme_util.DNS01_A_2 diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index eed07ae76..7fcd213a0 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -160,7 +160,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): """Tests for certbot._internal.plugins.selection.get_unprepared_installer.""" def setUp(self): - super(GetUnpreparedInstallerTest, self).setUp() + super().setUp() self.mock_apache_fail_ep = mock.Mock( description_with_name="afail") self.mock_apache_fail_ep.check_name = lambda name: name == "afail" diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 4eb5f0912..36159e44f 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -18,7 +18,7 @@ class PluginStorageTest(test_util.ConfigTestCase): """Test for certbot.plugins.storage.PluginStorage""" def setUp(self): - super(PluginStorageTest, self).setUp() + super().setUp() self.plugin_cls = common.Installer filesystem.mkdir(self.config.config_dir) with mock.patch("certbot.reverter.util"): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index b5ecddb5a..5137c81c3 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -17,7 +17,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): """Tests for interfaces.RenewDeployer and interfaces.GenericUpdater""" def setUp(self): - super(RenewUpdaterTest, self).setUp() + super().setUp() self.generic_updater = mock.MagicMock(spec=interfaces.GenericUpdater) self.generic_updater.restart = mock.MagicMock() self.renew_deployer = mock.MagicMock(spec=interfaces.RenewDeployer) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index af01a9a1b..45479b16d 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -18,7 +18,7 @@ from certbot.tests import util as test_util class ReverterCheckpointLocalTest(test_util.ConfigTestCase): """Test the Reverter Class.""" def setUp(self): - super(ReverterCheckpointLocalTest, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... we are trying to test for them @@ -280,7 +280,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): class TestFullCheckpointsReverter(test_util.ConfigTestCase): """Tests functions having to deal with full checkpoints.""" def setUp(self): - super(TestFullCheckpointsReverter, self).setUp() + super().setUp() from certbot.reverter import Reverter # Disable spurious errors... logging.disable(logging.CRITICAL) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 1e3a1ffd8..d7ef24b43 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -99,7 +99,7 @@ class BaseRenewableCertTest(test_util.ConfigTestCase): def setUp(self): from certbot._internal import storage - super(BaseRenewableCertTest, self).setUp() + super().setUp() # TODO: maybe provide NamespaceConfig.make_dirs? # TODO: main() should create those dirs, c.f. #902 @@ -846,7 +846,7 @@ class RenewableCertTests(BaseRenewableCertTest): class DeleteFilesTest(BaseRenewableCertTest): """Tests for certbot._internal.storage.delete_files""" def setUp(self): - super(DeleteFilesTest, self).setUp() + super().setUp() for kind in ALL_FOUR: kind_path = os.path.join(self.config.config_dir, "live", "example.org", @@ -936,7 +936,7 @@ class DeleteFilesTest(BaseRenewableCertTest): class CertPathForCertNameTest(BaseRenewableCertTest): """Test for certbot._internal.storage.cert_path_for_cert_name""" def setUp(self): - super(CertPathForCertNameTest, self).setUp() + super().setUp() self.config_file.write() self._write_out_ex_kinds() self.fullchain = os.path.join(self.config.config_dir, 'live', 'example.org', diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 18947c342..e2f0218b8 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -108,7 +108,7 @@ class LockDirUntilExit(test_util.TempDirTestCase): return lock_dir_until_exit(*args, **kwargs) def setUp(self): - super(LockDirUntilExit, self).setUp() + super().setUp() # reset global state from other tests import certbot.util reload_module(certbot.util) @@ -164,7 +164,7 @@ class MakeOrVerifyDirTest(test_util.TempDirTestCase): """ def setUp(self): - super(MakeOrVerifyDirTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") filesystem.mkdir(self.path, 0o600) @@ -196,7 +196,7 @@ class UniqueFileTest(test_util.TempDirTestCase): """Tests for certbot.util.unique_file.""" def setUp(self): - super(UniqueFileTest, self).setUp() + super().setUp() self.default_name = os.path.join(self.tempdir, "foo.txt") @@ -284,7 +284,7 @@ class SafelyRemoveTest(test_util.TempDirTestCase): """Tests for certbot.util.safely_remove.""" def setUp(self): - super(SafelyRemoveTest, self).setUp() + super().setUp() self.path = os.path.join(self.tempdir, "foo") From 06bece36de0898c7cf22046ddc140c7dd5bcc8f5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 9 Apr 2021 14:34:50 -0700 Subject: [PATCH 67/99] Ensure that mock is pinned (#8786) * List mock as a dependency in pyproject.toml * Add a code comment to help us remember to remove it when we can * Run pin.sh --- certbot/certbot/tests/util.py | 3 +++ tools/pinning/pyproject.toml | 7 +++++++ tools/requirements.txt | 11 ++++++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index b6a27a7c8..b892ad0a1 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -27,6 +27,9 @@ from certbot.compat import os from certbot.display import util as display_util try: + # When we remove this deprecated import, we should also remove the + # "external-mock" test environment and the mock dependency listed in + # tools/pinning/pyproject.toml. import mock warnings.warn( "The external mock module is being used for backwards compatibility " diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 0107733d5..20a1f3bf2 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -35,6 +35,13 @@ acme = {path = "../../acme", extras = ["dev", "docs"]} windows-installer = {path = "../../windows-installer"} # Extra dependencies +# We install mock in our "external-mock" tox environment to test that we didn't +# break Certbot's test API which used to always use mock objects from the 3rd +# party mock library. We list the mock dependency here so that is pinned, but +# we don't depend on it in Certbot to avoid installing mock when it's not +# needed. This dependency can be removed here once Certbot's support for the +# 3rd party mock library has been dropped. +mock = "*" # Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many # test failures so let's pin these packages back for now. coverage = "4.5.4" diff --git a/tools/requirements.txt b/tools/requirements.txt index e5a6d3be7..498c1e87a 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -18,8 +18,8 @@ backcall==0.2.0 bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.44; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.44; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.47; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.47; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -36,7 +36,7 @@ configobj==5.0.6; python_version >= "3.6" coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" -decorator==5.0.5 +decorator==5.0.6 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -80,6 +80,7 @@ lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0 lockfile==0.12.2 markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" mccabe==0.6.1; python_version >= "3.6" +mock==4.0.3; python_version >= "3.6" msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" msrest==0.6.21; python_version >= "3.6" mypy-extensions==0.4.3; python_version >= "3.6" @@ -99,7 +100,7 @@ ply==3.11; python_version >= "3.6" poetry-core==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" poetry==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" prompt-toolkit==3.0.3 -protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" @@ -157,7 +158,7 @@ tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -tqdm==4.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +tqdm==4.60.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" typed-ast==1.4.2; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" From e33090f2827558962984e0fa6cd16e653307f412 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 12 Apr 2021 13:36:38 -0700 Subject: [PATCH 68/99] Fix homebrew (#8791) The macOS tests run on this PR would fail without this change. * brew update * add link to upstream issue --- .azure-pipelines/templates/steps/tox-steps.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index ecf3d6032..3e5fb995d 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -1,6 +1,10 @@ steps: + # We run brew update because we've seen attempts to install an older version + # of a package fail. See + # https://github.com/actions/virtual-environments/issues/3165. - bash: | set -e + brew update brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') displayName: Install MacOS dependencies From 0dbe17bbd48b97d41754b9618c302ab3c4aa1421 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 13 Apr 2021 20:18:49 +0200 Subject: [PATCH 69/99] Define OS options by a dedicated object in Apache configurator (#8778) In https://github.com/certbot/certbot/pull/8748#discussion_r605457670 we discussed about changing the dict used to set OS options for Apache configurators into a dedicated object. * Create _OsOptions class to configure the os specific options of the Apache configurators * Fix tests * Clean imports * Fix naming * Fix compatibility tests * Rename a class * Ensure restart_cmd_alt is set for specific OSes. * Add docstring * Fix override * Fix coverage --- .../certbot_apache/_internal/configurator.py | 143 ++++++++++-------- .../certbot_apache/_internal/override_arch.py | 9 +- .../_internal/override_centos.py | 15 +- .../_internal/override_darwin.py | 11 +- .../_internal/override_debian.py | 20 +-- .../_internal/override_fedora.py | 22 +-- .../_internal/override_gentoo.py | 25 +-- .../certbot_apache/_internal/override_suse.py | 9 +- .../certbot_apache/_internal/parser.py | 8 +- certbot-apache/tests/configurator_test.py | 15 +- certbot-apache/tests/parser_test.py | 4 +- certbot-apache/tests/util.py | 4 +- .../configurators/apache/common.py | 4 +- 13 files changed, 129 insertions(+), 160 deletions(-) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 5ef9083e6..abf199793 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -8,7 +8,6 @@ import logging import re import socket import time -from typing import cast from typing import DefaultDict from typing import Dict from typing import List @@ -51,6 +50,47 @@ except ImportError: # pragma: no cover logger = logging.getLogger(__name__) +class OsOptions: + """ + Dedicated class to describe the OS specificities (eg. paths, binary names) + that the Apache configurator needs to be aware to operate properly. + """ + def __init__(self, + server_root="/etc/apache2", + vhost_root="/etc/apache2/sites-available", + vhost_files="*", + logs_root="/var/log/apache2", + ctl="apache2ctl", + version_cmd: Optional[List[str]] = None, + restart_cmd: Optional[List[str]] = None, + restart_cmd_alt: Optional[List[str]] = None, + conftest_cmd: Optional[List[str]] = None, + enmod: Optional[str] = None, + dismod: Optional[str] = None, + le_vhost_ext="-le-ssl.conf", + handle_modules=False, + handle_sites=False, + challenge_location="/etc/apache2", + apache_bin: Optional[str] = None, + ): + self.server_root = server_root + self.vhost_root = vhost_root + self.vhost_files = vhost_files + self.logs_root = logs_root + self.ctl = ctl + self.version_cmd = ['apache2ctl', '-v'] if not version_cmd else version_cmd + self.restart_cmd = ['apache2ctl', 'graceful'] if not restart_cmd else restart_cmd + self.restart_cmd_alt = restart_cmd_alt + self.conftest_cmd = ['apache2ctl', 'configtest'] if not conftest_cmd else conftest_cmd + self.enmod = enmod + self.dismod = dismod + self.le_vhost_ext = le_vhost_ext + self.handle_modules = handle_modules + self.handle_sites = handle_sites + self.challenge_location = challenge_location + self.bin = apache_bin + + # TODO: Augeas sections ie. , beginning and closing # tags need to be the same case, otherwise Augeas doesn't recognize them. # This is not able to be completely remedied by regular expressions because @@ -106,27 +146,7 @@ class ApacheConfigurator(common.Installer): " change depending on the operating system Certbot is run on.)" ) - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, - challenge_location="/etc/apache2", - bin=None - ) - - def option(self, key): - """Get a value from options""" - return self.options.get(key) + OS_DEFAULTS = OsOptions() def pick_apache_config(self, warn_on_no_mod_ssl=True): """ @@ -156,14 +176,14 @@ class ApacheConfigurator(common.Installer): for o in opts: # Config options use dashes instead of underscores if self.conf(o.replace("_", "-")) is not None: - self.options[o] = self.conf(o.replace("_", "-")) + setattr(self.options, o, self.conf(o.replace("_", "-"))) else: - self.options[o] = self.OS_DEFAULTS[o] + setattr(self.options, o, getattr(self.OS_DEFAULTS, o)) # Special cases - cast(List[str], self.options["version_cmd"])[0] = self.option("ctl") - cast(List[str], self.options["restart_cmd"])[0] = self.option("ctl") - cast(List[str], self.options["conftest_cmd"])[0] = self.option("ctl") + self.options.version_cmd[0] = self.options.ctl + self.options.restart_cmd[0] = self.options.ctl + self.options.conftest_cmd[0] = self.options.ctl @classmethod def add_parser_arguments(cls, add): @@ -178,30 +198,30 @@ class ApacheConfigurator(common.Installer): else: # cls.OS_DEFAULTS can be distribution specific, see override classes DEFAULTS = cls.OS_DEFAULTS - add("enmod", default=DEFAULTS["enmod"], + add("enmod", default=DEFAULTS.enmod, help="Path to the Apache 'a2enmod' binary") - add("dismod", default=DEFAULTS["dismod"], + add("dismod", default=DEFAULTS.dismod, help="Path to the Apache 'a2dismod' binary") - add("le-vhost-ext", default=DEFAULTS["le_vhost_ext"], + add("le-vhost-ext", default=DEFAULTS.le_vhost_ext, help="SSL vhost configuration extension") - add("server-root", default=DEFAULTS["server_root"], + add("server-root", default=DEFAULTS.server_root, help="Apache server root directory") add("vhost-root", default=None, help="Apache server VirtualHost configuration root") - add("logs-root", default=DEFAULTS["logs_root"], + add("logs-root", default=DEFAULTS.logs_root, help="Apache server logs directory") add("challenge-location", - default=DEFAULTS["challenge_location"], + default=DEFAULTS.challenge_location, help="Directory path for challenge configuration") - add("handle-modules", default=DEFAULTS["handle_modules"], + add("handle-modules", default=DEFAULTS.handle_modules, help="Let installer handle enabling required modules for you " + "(Only Ubuntu/Debian currently)") - add("handle-sites", default=DEFAULTS["handle_sites"], + add("handle-sites", default=DEFAULTS.handle_sites, help="Let installer handle enabling sites for you " + "(Only Ubuntu/Debian currently)") - add("ctl", default=DEFAULTS["ctl"], + add("ctl", default=DEFAULTS.ctl, help="Full path to Apache control script") - add("bin", default=DEFAULTS["bin"], + add("bin", default=DEFAULTS.bin, help="Full path to apache2/httpd binary") def __init__(self, *args, **kwargs): @@ -290,8 +310,8 @@ class ApacheConfigurator(common.Installer): ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) else: # Possibility B: ssl_module is statically linked into Apache - if self.option("bin"): - ssl_module_location = self.option("bin") + if self.options.bin: + ssl_module_location = self.options.bin else: logger.warning("ssl_module is statically linked but --apache-bin is " "missing; not disabling session tickets.") @@ -321,7 +341,7 @@ class ApacheConfigurator(common.Installer): self._prepare_options() # Verify Apache is installed - self._verify_exe_availability(self.option("ctl")) + self._verify_exe_availability(self.options.ctl) # Make sure configuration is valid self.config_test() @@ -361,20 +381,20 @@ class ApacheConfigurator(common.Installer): # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. # This is currently only true for debian/ubuntu. - warn_on_no_mod_ssl = not self.option("handle_modules") + warn_on_no_mod_ssl = not self.options.handle_modules self.install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest, warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: - util.lock_dir_until_exit(self.option("server_root")) + util.lock_dir_until_exit(self.options.server_root) except (OSError, errors.LockError): logger.debug("Encountered error:", exc_info=True) raise errors.PluginError( "Unable to create a lock file in {0}. Are you running" " Certbot with sufficient privileges to modify your" - " Apache configuration?".format(self.option("server_root"))) + " Apache configuration?".format(self.options.server_root)) self._prepared = True def save(self, title=None, temporary=False): @@ -449,7 +469,7 @@ class ApacheConfigurator(common.Installer): """Initializes the ApacheParser""" # If user provided vhost_root value in command line, use it return parser.ApacheParser( - self.option("server_root"), self.conf("vhost-root"), + self.options.server_root, self.conf("vhost-root"), self.version, configurator=self) def get_parsernode_root(self, metadata): @@ -457,9 +477,9 @@ class ApacheConfigurator(common.Installer): if HAS_APACHECONFIG: apache_vars = {} - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + apache_vars["defines"] = apache_util.parse_defines(self.options.ctl) + apache_vars["includes"] = apache_util.parse_includes(self.options.ctl) + apache_vars["modules"] = apache_util.parse_modules(self.options.ctl) metadata["apache_vars"] = apache_vars with open(self.parser.loc["root"]) as f: @@ -1311,7 +1331,7 @@ class ApacheConfigurator(common.Installer): :param boolean temp: If the change is temporary """ - if self.option("handle_modules"): + if self.options.handle_modules: if self.version >= (2, 4) and ("socache_shmcb_module" not in self.parser.modules): self.enable_mod("socache_shmcb", temp=temp) @@ -1331,7 +1351,7 @@ class ApacheConfigurator(common.Installer): Duplicates vhost and adds default ssl options New vhost will reside as (nonssl_vhost.path) + - ``self.option("le_vhost_ext")`` + ``self.options.le_vhost_ext`` .. note:: This function saves the configuration @@ -1430,15 +1450,15 @@ class ApacheConfigurator(common.Installer): """ if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")): - fp = os.path.join(filesystem.realpath(self.option("vhost_root")), + fp = os.path.join(filesystem.realpath(self.options.vhost_root), os.path.basename(non_ssl_vh_fp)) else: # Use non-ssl filepath fp = filesystem.realpath(non_ssl_vh_fp) if fp.endswith(".conf"): - return fp[:-(len(".conf"))] + self.option("le_vhost_ext") - return fp + self.option("le_vhost_ext") + return fp[:-(len(".conf"))] + self.options.le_vhost_ext + return fp + self.options.le_vhost_ext def _sift_rewrite_rule(self, line): """Decides whether a line should be copied to a SSL vhost. @@ -2282,7 +2302,7 @@ class ApacheConfigurator(common.Installer): addr in self._get_proposed_addrs(ssl_vhost)), servername, serveralias, " ".join(rewrite_rule_args), - self.option("logs_root"))) + self.options.logs_root)) def _write_out_redirect(self, ssl_vhost, text): # This is the default name @@ -2294,7 +2314,7 @@ class ApacheConfigurator(common.Installer): if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)): redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name - redirect_filepath = os.path.join(self.option("vhost_root"), + redirect_filepath = os.path.join(self.options.vhost_root, redirect_filename) # Register the new file that will be created @@ -2414,19 +2434,18 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("restart_cmd")) + util.run_script(self.options.restart_cmd) except errors.SubprocessError as err: logger.info("Unable to restart apache using %s", - self.option("restart_cmd")) - alt_restart = self.option("restart_cmd_alt") + self.options.restart_cmd) + alt_restart = self.options.restart_cmd_alt if alt_restart: logger.debug("Trying alternative restart command: %s", alt_restart) # There is an alternative restart command available # This usually is "restart" verb while original is "graceful" try: - util.run_script(self.option( - "restart_cmd_alt")) + util.run_script(self.options.restart_cmd_alt) return except errors.SubprocessError as secerr: error = str(secerr) @@ -2441,7 +2460,7 @@ class ApacheConfigurator(common.Installer): """ try: - util.run_script(self.option("conftest_cmd")) + util.run_script(self.options.conftest_cmd) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -2457,11 +2476,11 @@ class ApacheConfigurator(common.Installer): """ try: - stdout, _ = util.run_script(self.option("version_cmd")) + stdout, _ = util.run_script(self.options.version_cmd) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % - self.option("version_cmd")) + self.options.version_cmd) regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE) matches = regex.findall(stdout) diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 1c3aed1dc..30d161a4e 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -3,13 +3,14 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class ArchConfigurator(configurator.ApacheConfigurator): """Arch Linux specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf", vhost_files="*.conf", @@ -18,11 +19,5 @@ class ArchConfigurator(configurator.ApacheConfigurator): version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 98dc80e0b..c1a69885c 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -12,6 +12,7 @@ from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ logger = logging.getLogger(__name__) class CentOSConfigurator(configurator.ApacheConfigurator): """CentOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -30,13 +31,7 @@ class CentOSConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -77,12 +72,14 @@ class CentOSConfigurator(configurator.ApacheConfigurator): alternative restart cmd used in CentOS. """ super()._prepare_options() - cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for CentOS.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return CentOSParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _deploy_cert(self, *args, **kwargs): # pylint: disable=arguments-differ diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 106f5fbab..e1dca7f5e 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -3,26 +3,19 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class DarwinConfigurator(configurator.ApacheConfigurator): """macOS specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/other", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/other", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index c5f3b6e1a..7f12f4bbc 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -10,6 +10,7 @@ from certbot.compat import filesystem from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions logger = logging.getLogger(__name__) @@ -18,22 +19,11 @@ logger = logging.getLogger(__name__) class DebianConfigurator(configurator.ApacheConfigurator): """Debian specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", - vhost_root="/etc/apache2/sites-available", - vhost_files="*", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], - conftest_cmd=['apache2ctl', 'configtest'], + OS_DEFAULTS = OsOptions( enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", handle_modules=True, handle_sites=True, - challenge_location="/etc/apache2", - bin=None, ) def enable_site(self, vhost): @@ -132,11 +122,11 @@ class DebianConfigurator(configurator.ApacheConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not util.exe_exists(self.option("dismod")): + if not util.exe_exists(self.options.dismod): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( - temp, [self.option("dismod"), "-f", mod_name]) - util.run_script([self.option("enmod"), mod_name]) + temp, [self.options.dismod, "-f", mod_name]) + util.run_script([self.options.enmod, mod_name]) diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 0f7970460..3b947a823 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -1,7 +1,4 @@ """ Distribution specific override class for Fedora 29+ """ -from typing import cast -from typing import List - import zope.interface from certbot import errors @@ -10,13 +7,14 @@ from certbot import util from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class FedoraConfigurator(configurator.ApacheConfigurator): """Fedora 29+ specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/httpd", vhost_root="/etc/httpd/conf.d", vhost_files="*.conf", @@ -26,13 +24,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): restart_cmd=['apachectl', 'graceful'], restart_cmd_alt=['apachectl', 'restart'], conftest_cmd=['apachectl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/httpd/conf.d", - bin=None, ) def config_test(self): @@ -50,7 +42,7 @@ class FedoraConfigurator(configurator.ApacheConfigurator): def get_parser(self): """Initializes the ApacheParser""" return FedoraParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) def _try_restart_fedora(self): @@ -72,9 +64,11 @@ class FedoraConfigurator(configurator.ApacheConfigurator): of Fedora to restart httpd. """ super()._prepare_options() - cast(List[str], self.options["restart_cmd"])[0] = 'apachectl' - cast(List[str], self.options["restart_cmd_alt"])[0] = 'apachectl' - cast(List[str], self.options["conftest_cmd"])[0] = 'apachectl' + self.options.restart_cmd[0] = 'apachectl' + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Fedora.") + self.options.restart_cmd_alt[0] = 'apachectl' + self.options.conftest_cmd[0] = 'apachectl' class FedoraParser(parser.ApacheParser): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index 484e15532..1b86c925e 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -1,36 +1,23 @@ """ Distribution specific override class for Gentoo Linux """ -from typing import cast -from typing import List - import zope.interface from certbot import interfaces from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class GentooConfigurator(configurator.ApacheConfigurator): """Gentoo specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( + OS_DEFAULTS = OsOptions( server_root="/etc/apache2", vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", - ctl="apache2ctl", - version_cmd=['apache2ctl', '-v'], - restart_cmd=['apache2ctl', 'graceful'], restart_cmd_alt=['apache2ctl', 'restart'], - conftest_cmd=['apache2ctl', 'configtest'], - enmod=None, - dismod=None, - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) def _prepare_options(self): @@ -39,12 +26,14 @@ class GentooConfigurator(configurator.ApacheConfigurator): alternative restart cmd used in Gentoo. """ super()._prepare_options() - cast(List[str], self.options["restart_cmd_alt"])[0] = self.option("ctl") + if not self.options.restart_cmd_alt: # pragma: no cover + raise ValueError("OS option restart_cmd_alt must be set for Gentoo.") + self.options.restart_cmd_alt[0] = self.options.ctl def get_parser(self): """Initializes the ApacheParser""" return GentooParser( - self.option("server_root"), self.option("vhost_root"), + self.options.server_root, self.options.vhost_root, self.version, configurator=self) @@ -69,7 +58,7 @@ class GentooParser(parser.ApacheParser): def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - mod_cmd = [self.configurator.option("ctl"), "modules"] + mod_cmd = [self.configurator.options.ctl, "modules"] matches = apache_util.parse_from_subprocess(mod_cmd, r"(.*)_module") for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index afce98dfa..d692fd239 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -3,26 +3,21 @@ import zope.interface from certbot import interfaces from certbot_apache._internal import configurator +from certbot_apache._internal.configurator import OsOptions @zope.interface.provider(interfaces.IPluginFactory) class OpenSUSEConfigurator(configurator.ApacheConfigurator): """OpenSUSE specific ApacheConfigurator override class""" - OS_DEFAULTS = dict( - server_root="/etc/apache2", + OS_DEFAULTS = OsOptions( vhost_root="/etc/apache2/vhosts.d", vhost_files="*.conf", - logs_root="/var/log/apache2", ctl="apachectl", version_cmd=['apachectl', '-v'], restart_cmd=['apachectl', 'graceful'], conftest_cmd=['apachectl', 'configtest'], enmod="a2enmod", dismod="a2dismod", - le_vhost_ext="-le-ssl.conf", - handle_modules=False, - handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - bin=None, ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index ff7e90f3b..141991ccc 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -81,7 +81,7 @@ class ApacheParser: # Must also attempt to parse additional virtual host root if vhostroot: self.parse_file(os.path.abspath(vhostroot) + "/" + - self.configurator.option("vhost_files")) + self.configurator.options.vhost_files) # check to see if there were unparsed define statements if version < (2, 4): @@ -282,7 +282,7 @@ class ApacheParser: def update_defines(self): """Updates the dictionary of known variables in the configuration""" - self.variables = apache_util.parse_defines(self.configurator.option("ctl")) + self.variables = apache_util.parse_defines(self.configurator.options.ctl) def update_includes(self): """Get includes from httpd process, and add them to DOM if needed""" @@ -292,7 +292,7 @@ class ApacheParser: # configuration files _ = self.find_dir("Include") - matches = apache_util.parse_includes(self.configurator.option("ctl")) + matches = apache_util.parse_includes(self.configurator.options.ctl) if matches: for i in matches: if not self.parsed_in_current(i): @@ -301,7 +301,7 @@ class ApacheParser: def update_modules(self): """Get loaded modules from httpd process, and add them to DOM""" - matches = apache_util.parse_modules(self.configurator.option("ctl")) + matches = apache_util.parse_modules(self.configurator.options.ctl) for mod in matches: self.add_mod(mod.strip()) diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index ad1f5f04d..8620955aa 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -103,9 +103,9 @@ class MultipleVhostsTest(util.ApacheTest): "handle_modules", "handle_sites", "ctl"] exp = {} - for k in ApacheConfigurator.OS_DEFAULTS: + for k in ApacheConfigurator.OS_DEFAULTS.__dict__.keys(): if k in parserargs: - exp[k.replace("_", "-")] = ApacheConfigurator.OS_DEFAULTS[k] + exp[k.replace("_", "-")] = getattr(ApacheConfigurator.OS_DEFAULTS, k) # Special cases exp["vhost-root"] = None @@ -128,14 +128,13 @@ class MultipleVhostsTest(util.ApacheTest): def test_all_configurators_defaults_defined(self): from certbot_apache._internal.entrypoint import OVERRIDE_CLASSES from certbot_apache._internal.configurator import ApacheConfigurator - parameters = set(ApacheConfigurator.OS_DEFAULTS.keys()) + parameters = set(ApacheConfigurator.OS_DEFAULTS.__dict__.keys()) for cls in OVERRIDE_CLASSES.values(): - self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys()))) + self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.__dict__.keys()))) def test_constant(self): self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in - self.config.option("server_root")) - self.assertEqual(self.config.option("nonexistent"), None) + self.config.options.server_root) @certbot_util.patch_get_utility() def test_get_all_names(self, mock_getutility): @@ -1774,7 +1773,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # ssl_module statically linked self.config._openssl_version = None self.config.parser.modules['ssl_module'] = None - self.config.options['bin'] = '/fake/path/to/httpd' + self.config.options.bin = '/fake/path/to/httpd' with mock.patch("certbot_apache._internal.configurator." "ApacheConfigurator._open_module_file") as mock_omf: mock_omf.return_value = some_string_contents @@ -1810,7 +1809,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): # When ssl_module is statically linked but --apache-bin not provided self.config._openssl_version = None - self.config.options['bin'] = None + self.config.options.bin = None self.config.parser.modules['ssl_module'] = None with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: self.assertEqual(self.config.openssl_version(), None) diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index 37d4eb782..e30eca153 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -305,11 +305,9 @@ class BasicParserTest(util.ParserTest): self.assertRaises( errors.PluginError, self.parser.update_runtime_variables) - @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.option") @mock.patch("certbot_apache._internal.apache_util.subprocess.Popen") - def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt): + def test_update_runtime_vars_bad_ctl(self, mock_popen): mock_popen.side_effect = OSError - mock_opt.return_value = "nonexistent" self.assertRaises( errors.MisconfigurationError, self.parser.update_runtime_variables) diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index bd522d736..a0b44d188 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -123,11 +123,11 @@ def get_apache_configurator( version=version, use_parsernode=use_parsernode, openssl_version=openssl_version) if not conf_vhost_path: - config_class.OS_DEFAULTS["vhost_root"] = vhost_path + config_class.OS_DEFAULTS.vhost_root = vhost_path else: # Custom virtualhost path was requested config.config.apache_vhost_root = conf_vhost_path - config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"] + config.config.apache_ctl = config_class.OS_DEFAULTS.ctl config.prepare() return config diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index f62635610..909a9d37c 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -54,9 +54,9 @@ class Proxy(configurators_common.Proxy): def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - for k in entrypoint.ENTRYPOINT.OS_DEFAULTS: + for k in entrypoint.ENTRYPOINT.OS_DEFAULTS.__dict__.keys(): setattr(self.le_config, "apache_" + k, - entrypoint.ENTRYPOINT.OS_DEFAULTS[k]) + getattr(entrypoint.ENTRYPOINT.OS_DEFAULTS, k)) self._configurator = entrypoint.ENTRYPOINT( config=configuration.NamespaceConfig(self.le_config), From 4a404e2a4a251442240896cc985e311f13e283d5 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Thu, 15 Apr 2021 00:36:14 +0200 Subject: [PATCH 70/99] Expand manual DNS challenge instructions to include mention of propagation time and tool to check this (#8770) * Expand manual DNS challenge instructions * Less jargon Co-authored-by: ohemorange * Less is more Co-authored-by: ohemorange * Make more clear where to look at Googles Toolbox * Reshuffle text * Show verify instructions only on last dns-01 challenge * Swap domain and value * Remove '(also)' * Fix DNS verify message for mixed challenge types * Add a lengthy comment about why there's a full stop after `{domain}` * Typo Co-authored-by: ohemorange --- certbot/certbot/_internal/plugins/manual.py | 43 +++++++++++++++++---- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 5423ec1d9..dec73a1ed 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -43,13 +43,30 @@ class Authenticator(common.Plugin): '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' 'list of all domains that are challenged for the current certificate.') + # Include the full stop at the end of the FQDN in the instructions below for the null + # label of the DNS root, as stated in section 3.1 of RFC 1035. While not necessary + # for most day to day usage of hostnames, when adding FQDNs to a DNS zone editor, this + # full stop is often mandatory. Without a full stop, the entered name is often seen as + # relative to the DNS zone origin, which could lead to entries for, e.g.: + # _acme-challenge.example.com.example.com. For users unaware of this subtle detail, + # including the trailing full stop in the DNS instructions below might avert this issue. _DNS_INSTRUCTIONS = """\ -Please deploy a DNS TXT record under the name -{domain} with the following value: +Please deploy a DNS TXT record under the name: + +{domain}. + +with the following value: {validation} - -Before continuing, verify the record is deployed.""" +""" + _DNS_VERIFY_INSTRUCTIONS = """ +Before continuing, verify the TXT record has been deployed. Depending on the DNS +provider, this may take some time, from a few seconds to multiple minutes. You can +check if it has finished deploying with aid of online tools, such as the Google +Admin Toolbox: https://toolbox.googleapps.com/apps/dig/#TXT/{domain}. +Look for one or more bolded line(s) below the line ';ANSWER'. It should show the +value(s) you've just added. +""" _HTTP_INSTRUCTIONS = """\ Create a file containing just this data: @@ -114,11 +131,15 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring responses = [] - for achall in achalls: + last_dns_achall = 0 + for i, achall in enumerate(achalls): + if isinstance(achall.chall, challenges.DNS01): + last_dns_achall = i + for i, achall in enumerate(achalls): if self.conf('auth-hook'): self._perform_achall_with_script(achall, achalls) else: - self._perform_achall_manually(achall) + self._perform_achall_manually(achall, i == last_dns_achall) responses.append(achall.response(achall.account_key)) return responses @@ -136,7 +157,7 @@ permitted by DNS standards.) env['CERTBOT_AUTH_OUTPUT'] = out.strip() self.env[achall] = env - def _perform_achall_manually(self, achall): + def _perform_achall_manually(self, achall, last_dns_achall=False): validation = achall.validation(achall.account_key) if isinstance(achall.chall, challenges.HTTP01): msg = self._HTTP_INSTRUCTIONS.format( @@ -152,7 +173,15 @@ permitted by DNS standards.) if self.subsequent_dns_challenge: # 2nd or later dns-01 challenge msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS + elif self.subsequent_any_challenge: + # 1st dns-01 challenge, but 2nd or later *any* challenge, so + # instruct user not to remove any previous http-01 challenge + msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS self.subsequent_dns_challenge = True + if last_dns_achall: + # last dns-01 challenge + msg += self._DNS_VERIFY_INSTRUCTIONS.format( + domain=achall.validation_domain_name(achall.domain)) elif self.subsequent_any_challenge: # 2nd or later challenge of another type msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS From fb967fda1532222c555a550c88303f02026e0bb1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 20 Apr 2021 12:12:45 -0700 Subject: [PATCH 71/99] pin cython (#8794) --- tools/pinning/pyproject.toml | 7 +++++++ tools/requirements.txt | 23 ++++++++++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 20a1f3bf2..34fea18a6 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -35,6 +35,13 @@ acme = {path = "../../acme", extras = ["dev", "docs"]} windows-installer = {path = "../../windows-installer"} # Extra dependencies +# As of writing this, cython is a build dependency of pyyaml. Since there +# doesn't appear to be a good way to automatically track down and pin build +# dependencies in Python (see +# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list it +# as a dependency here to ensure a version of cython is pinned for extra +# stability. +cython = "*" # We install mock in our "external-mock" tox environment to test that we didn't # break Certbot's test API which used to always use mock objects from the 3rd # party mock library. We list the mock dependency here so that is pinned, but diff --git a/tools/requirements.txt b/tools/requirements.txt index 498c1e87a..b662744c6 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -18,8 +18,8 @@ backcall==0.2.0 bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.47; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.47; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -36,7 +36,8 @@ configobj==5.0.6; python_version >= "3.6" coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" -decorator==5.0.6 +cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +decorator==5.0.7 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -51,9 +52,9 @@ execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or pyt filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.1.0; python_version >= "3.6" +google-api-python-client==2.2.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==1.28.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-auth==1.29.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" httplib2==0.19.1; python_version >= "3.6" @@ -97,8 +98,8 @@ pickleshare==0.7.5 pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" ply==3.11; python_version >= "3.6" -poetry-core==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -poetry==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry-core==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry==1.1.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" prompt-toolkit==3.0.3 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" @@ -139,14 +140,14 @@ requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0 requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" rfc3986==1.4.0; python_version >= "3.6" rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -s3transfer==0.3.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +s3transfer==0.3.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" snowballstemmer==2.1.0; python_version >= "3.6" soupsieve==2.2.1; python_version >= "3.6" sphinx-rtd-theme==0.5.2; python_version >= "3.6" -sphinx==3.5.3; python_version >= "3.6" +sphinx==3.5.4; python_version >= "3.6" sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" @@ -161,7 +162,7 @@ tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python tqdm==4.60.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" -typed-ast==1.4.2; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" +typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" typing-extensions==3.7.4.3; python_version >= "3.6" uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" @@ -175,7 +176,7 @@ zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and pytho zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -zope.interface==5.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pip==20.2.4 setuptools==54.1.2 wheel==0.35.1 From 9292666b28e0f4159dd4096265e69a58c9930677 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 21 Apr 2021 15:55:05 -0700 Subject: [PATCH 72/99] fix ciphers link (#8799) --- certbot/docs/ciphers.rst | 8 ++++++++ certbot/docs/conf.py | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 43f648898..b4ef5902a 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -1,3 +1,11 @@ +.. + Sphinx complains that this file isn't included in any toctree, however, we + currently link to it in the section about installing Certbot through Docker. + Setting :orphan: below suppresses this warning. See + https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html#special-metadata-fields. + +:orphan: + ============ Ciphersuites ============ diff --git a/certbot/docs/conf.py b/certbot/docs/conf.py index 254bd3edd..5496c42a2 100644 --- a/certbot/docs/conf.py +++ b/certbot/docs/conf.py @@ -98,7 +98,6 @@ language = None exclude_patterns = [ '_build', 'challenges.rst', - 'ciphers.rst' ] # The reST default role (used for this markup: `text`) to use for all From e4f5aced1cfddaba4e6816fbafa345b4b6eb9992 Mon Sep 17 00:00:00 2001 From: alexzorin Date: Fri, 23 Apr 2021 05:38:18 +1000 Subject: [PATCH 73/99] docs: add certbot-dns-azure third-party plugin (#8796) --- certbot/docs/using.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index e38725ca3..cc061b622 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -284,6 +284,7 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server dns-clouddns_ Y N DNS Authentication using CloudDNS API dns-lightsail_ Y N DNS Authentication using Amazon Lightsail DNS API dns-inwx_ Y Y DNS Authentication for INWX through the XML API +dns-azure_ Y N DNS Authentication using Azure DNS ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -298,6 +299,7 @@ dns-inwx_ Y Y DNS Authentication for INWX through the XML API .. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns .. _dns-lightsail: https://github.com/noi/certbot-dns-lightsail .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/ +.. _dns-azure: https://github.com/binkhq/certbot-dns-azure If you're interested, you can also :ref:`write your own plugin `. From 32247b3c89cb44b87f764a21e6deda9168431dec Mon Sep 17 00:00:00 2001 From: ohemorange Date: Thu, 22 Apr 2021 13:37:46 -0700 Subject: [PATCH 74/99] Remove modifications to certbot-auto from the release script (#8797) Fixes #8707. * Remove modifications to certbot-auto from the release script * Update tools/_release.sh * Delete tools/eff-pubkey.pem Co-authored-by: Brad Warren --- tools/_release.sh | 72 +++++--------------------------------------- tools/eff-pubkey.pem | 9 ------ 2 files changed, 7 insertions(+), 74 deletions(-) delete mode 100644 tools/eff-pubkey.pem diff --git a/tools/_release.sh b/tools/_release.sh index e9e0e49f0..5166753d2 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -30,9 +30,6 @@ echo Releasing production version "$version"... nextversion="$2" RELEASE_BRANCH="candidate-$version" -if [ "$RELEASE_OPENSSL_PUBKEY" = "" ] ; then - RELEASE_OPENSSL_PUBKEY="`realpath \`dirname $0\``/eff-pubkey.pem" -fi RELEASE_GPG_KEY=${RELEASE_GPG_KEY:-A2CFB51FA275A7286234E7B24D17C995CD9775F2} # Needed to fix problems with git signatures and pinentry export GPG_TTY=$(tty) @@ -40,14 +37,13 @@ export GPG_TTY=$(tty) # port for a local Python Package Index (used in testing) PORT=${PORT:-1234} -# subpackages to be released (the way developers think about them) -SUBPKGS_IN_AUTO_NO_CERTBOT="acme certbot-apache certbot-nginx" -SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud" - # subpackages to be released (the way the script thinks about them) -SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" -SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" -SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" +SUBPKGS_NO_CERTBOT="acme certbot-apache certbot-nginx certbot-dns-cloudflare certbot-dns-cloudxns \ + certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy \ + certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns \ + certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 \ + certbot-dns-sakuracloud" +SUBPKGS="certbot $SUBPKGS_NO_CERTBOT" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs # - it causes problems when running pytest - the latter tries to @@ -198,60 +194,10 @@ for module in $SUBPKGS ; do # use an empty configuration file rather than the one in the repo root pytest -c <(echo '') $module done - -# pin pip hashes of the things we just built -for pkg in $SUBPKGS_IN_AUTO ; do - echo $pkg==$version \\ - pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python -c 'from sys import stdin; input = stdin.read(); print(" ", input.replace("\n--hash", " \\\n --hash"), end="")' -done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -# there should be one requirement specifier and two hashes for each subpackage -expected_count=$(expr $(echo $SUBPKGS_IN_AUTO | wc -w) \* 3) -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then - echo Unexpected pip hash output - exit 1 -fi -# ensure we have the latest built version of leauto -letsencrypt-auto-source/build.py - -# Now we have to sign the built version of leauto. -SignLEAuto() { - yubico-piv-tool -a verify-pin --sign -s 9c -i letsencrypt-auto-source/letsencrypt-auto -o letsencrypt-auto-source/letsencrypt-auto.sig -} - -# Loop until letsencrypt-auto is signed correctly. -SignLEAuto || true -while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ - letsencrypt-auto-source/letsencrypt-auto.sig \ - letsencrypt-auto-source/letsencrypt-auto ; do - echo "The signature on letsencrypt-auto is not correct." - read -p "Would you like this script to try and sign it again [Y/n]?" response - case $response in - [yY][eE][sS]|[yY]|"") - SignLEAuto || true;; - *) - ;; - esac -done - -# This signature is not quite as strong, but easier for people to verify out of band -while ! gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto; do - echo "Unable to sign letsencrypt-auto using $RELEASE_KEY." - echo "Make sure your OpenPGP card is in your computer if you are using one." - echo "You may need to take the card out and put it back in again." - read -p "Press enter to try signing again." -done -# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, -# but we can use the right name for certbot-auto.asc from day one -mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc - -# copy leauto to the root, overwriting the previous release version -cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto -cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto - -git add certbot-auto letsencrypt-auto letsencrypt-auto-source certbot/docs/cli-help.txt +git add certbot/docs/cli-help.txt while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do echo "Unable to sign the release commit using git." echo "You may have to configure git to use gpg2 by running:" @@ -283,15 +229,11 @@ git commit -m "Add contents to certbot/CHANGELOG.md for next version" echo "New root: $root" echo "Test commands (in the letstest repo):" -echo 'python multitester.py auto_targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py auto_targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' echo "In order to upload packages run the following command:" echo twine upload "$root/dist.$version/*/*" if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 - letsencrypt-auto-source/build.py - git add letsencrypt-auto-source/letsencrypt-auto git commit -m "Bump version to $nextversion" fi diff --git a/tools/eff-pubkey.pem b/tools/eff-pubkey.pem deleted file mode 100644 index fe6c2f5bb..000000000 --- a/tools/eff-pubkey.pem +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- From c06e40dbefadb95c3441af444c552926c5f6c5dd Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 26 Apr 2021 13:50:10 -0700 Subject: [PATCH 75/99] Update certbot-auto modification checks (#8805) * revert changes to letsencrypt-auto-source/le-auto * update modification tests --- .../templates/jobs/standard-tests-jobs.yml | 2 + letsencrypt-auto-source/letsencrypt-auto | 2 +- tests/modification-check.py | 152 +++++------------- tox.ini | 5 +- 4 files changed, 48 insertions(+), 113 deletions(-) diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 62f22b223..c949af44a 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -56,6 +56,8 @@ jobs: apache-compat: IMAGE_NAME: ubuntu-18.04 TOXENV: apache_compat + # le-modification can be moved to the extended test suite once + # https://github.com/certbot/certbot/issues/8742 is resolved. le-modification: IMAGE_NAME: ubuntu-18.04 TOXENV: modification diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 9ddc7e076..c37c45596 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.15.0.dev0" +LE_AUTO_VERSION="1.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates diff --git a/tests/modification-check.py b/tests/modification-check.py index 357b25350..8f3ae1264 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -1,122 +1,58 @@ #!/usr/bin/env python +"""Ensures there have been no changes to important certbot-auto files.""" -from __future__ import print_function - +import hashlib import os -import shutil -import subprocess -import sys -import tempfile -from urllib.request import urlretrieve -def find_repo_path(): +# Relative to the root of the Certbot repo, these files are expected to exist +# and have the SHA-256 hashes contained in this dictionary. These hashes were +# taken from our v1.14.0 tag which was the last release we intended to make +# changes to certbot-auto. +# +# certbot-auto, letsencrypt-auto, and letsencrypt-auto-source/certbot-auto.asc +# can be removed from this dict after coordinating with tech ops to ensure we +# get the behavior we want from https://dl.eff.org. See +# https://github.com/certbot/certbot/issues/8742 for more info. +# +# Deleting letsencrypt-auto-source/letsencrypt-auto and +# letsencrypt-auto-source/letsencrypt-auto.sig can be done once we're +# comfortable breaking any certbot-auto scripts that haven't already updated to +# the last version. See +# https://opensource.eff.org/eff-open-source/pl/65geri7c4tr6iqunc1rpb3mpna for +# more info. +EXPECTED_FILES = { + 'certbot-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + 'letsencrypt-auto': + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto'): + 'b997e3608526650a08e36e682fc3bf0c29903c06fa5ba4cc49308c43832450c2', + os.path.join('letsencrypt-auto-source', 'certbot-auto.asc'): + '0558ba7bd816732b38c092e8fedb6033dad01f263e290ec6b946263aaf6625a8', + os.path.join('letsencrypt-auto-source', 'letsencrypt-auto.sig'): + '61c036aabf75da350b0633da1b2bef0260303921ecda993455ea5e6d3af3b2fe', +} + + +def find_repo_root(): return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -# We do not use filecmp.cmp to take advantage of universal newlines -# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. -# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. -# But it will work correctly on Linux for any version, because every file tested will be LF. -def compare_files(path_1, path_2): - l1 = l2 = True - with open(path_1, 'r') as f1, open(path_2, 'r') as f2: - line = 1 - while l1 and l2: - line += 1 - l1 = f1.readline() - l2 = f2.readline() - if l1 != l2: - print('---') - print(( - 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' - .format(os.path.basename(path_1), os.path.basename(path_2), line))) - print('(1): {0}'.format(repr(l1))) - print('(2): {0}'.format(repr(l2))) - print('---') - return False - return True +def sha256_hash(filename): + hash_object = hashlib.sha256() + with open(filename, 'rb') as f: + hash_object.update(f.read()) + return hash_object.hexdigest() -def validate_scripts_content(repo_path, temp_cwd): - errors = False - - if not compare_files( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(repo_path, 'letsencrypt-auto')): - print('Root certbot-auto and letsencrypt-auto differ.') - errors = True - else: - shutil.copyfile( - os.path.join(repo_path, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')) - shutil.copy(os.path.normpath(os.path.join( - repo_path, - 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) - - # Compare file against current version in the target branch - branch = os.environ.get('TARGET_BRANCH', 'master') - url = ( - 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' - .format(branch)) - urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) - - if compare_files( - os.path.join(temp_cwd, 'certbot-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were unchanged') - else: - # Compare file against the latest released version - latest_version = subprocess.check_output( - [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) - subprocess.check_call( - [sys.executable, 'fetch.py', '--le-auto-script', - 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) - if compare_files( - os.path.join(temp_cwd, 'letsencrypt-auto'), - os.path.join(temp_cwd, 'local-auto')): - print('Root *-auto were updated to the latest version.') - else: - print('Root *-auto have unexpected changes.') - errors = True - - return errors def main(): - repo_path = find_repo_path() - temp_cwd = tempfile.mkdtemp() - errors = False + repo_root = find_repo_root() + for filename, expected_hash in EXPECTED_FILES.items(): + filepath = os.path.join(repo_root, filename) + assert sha256_hash(filepath) == expected_hash, f'unexpected changes to {filepath}' + print('All certbot-auto files have correct hashes.') - try: - errors = validate_scripts_content(repo_path, temp_cwd) - - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'original-lea') - ) - subprocess.check_call([sys.executable, os.path.normpath(os.path.join( - repo_path, 'letsencrypt-auto-source/build.py'))]) - shutil.copyfile( - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), - os.path.join(temp_cwd, 'build-lea') - ) - shutil.copyfile( - os.path.join(temp_cwd, 'original-lea'), - os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) - ) - - if not compare_files( - os.path.join(temp_cwd, 'original-lea'), - os.path.join(temp_cwd, 'build-lea')): - print('Script letsencrypt-auto-source/letsencrypt-auto ' - 'doesn\'t match output of build.py.') - errors = True - else: - print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') - finally: - shutil.rmtree(temp_cwd) - - return errors if __name__ == '__main__': - if main(): - sys.exit(1) + main() diff --git a/tox.ini b/tox.ini index ada7002da..90aecc1b3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3-cover,lint,mypy +envlist = py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -182,12 +182,9 @@ commands = {[base]pip_install} acme certbot certbot-apache certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata -# This is a duplication of the command line in testenv:le_auto to -# allow users to run the modification check by running `tox` [testenv:modification] commands = python {toxinidir}/tests/modification-check.py -passenv = TARGET_BRANCH [testenv:apache_compat] commands = From ba912018f8507e190b5a90611c91201d952efdb9 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 26 Apr 2021 15:18:05 -0700 Subject: [PATCH 76/99] Remove pytest run from release script (#8810) Fixes #8802. Also removed the unused `kgs` cruft while I was here, since it's leftover from the [initial release commit](https://github.com/certbot/certbot/commit/3c08b512c3b546528880a1353f6b8607307533a3) and I'm pretty sure we don't use that anymore. --- tools/_release.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index 5166753d2..ba54536bf 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -177,23 +177,6 @@ cd ~- CERTBOT_DOCS=1 certbot --help all > certbot/docs/cli-help.txt jws --help > acme/docs/jws-help.txt -cd .. -# freeze before installing anything else, so that we know end-user KGS -# make sure "twine upload" doesn't catch "kgs" -if [ -d kgs ] ; then - echo Deleting old kgs... - rm -rf kgs -fi -mkdir kgs -kgs="kgs/$version" -pip freeze | tee $kgs -python ../tools/pip_install.py pytest -cd ~- -for module in $SUBPKGS ; do - echo testing $module - # use an empty configuration file rather than the one in the repo root - pytest -c <(echo '') $module -done deactivate From ac3edc2c1d4ca1098a7d6517f12688e378f965b3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 26 Apr 2021 15:47:49 -0700 Subject: [PATCH 77/99] don't ignore kgs (#8811) --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 34b3fc99e..f4542f412 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ build/ dist*/ /venv*/ -/kgs/ /.tox/ /releases*/ /log* From f339d23e54e88e7568251e9436281506b19ac86f Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 27 Apr 2021 15:27:21 -0700 Subject: [PATCH 78/99] Remove further references to certbot-auto in the repo (#8814) * Move version.py to tests/letstest since it's used by test_sdists.sh * Delete unused components of certbot-auto * Remove test_leauto_upgrades.sh and references to it * Remove test_letsencrypt_auto_certonly_standalone.sh and references to it * Remove outstanding references to certbot-auto * Remove references to letsencrypt-auto * find certbot in the correct directory * delete letsencrypt-auto-source line from .isort.cfg since that directory no longer contains any python code * remove (-auto) from certbot(-auto) * delete line from test * Improve style for version.py --- .../templates/jobs/extended-tests-jobs.yml | 6 - .gitignore | 1 - .isort.cfg | 1 - certbot/certbot/_internal/cli/__init__.py | 3 +- certbot/certbot/_internal/client.py | 2 +- certbot/tests/client_test.py | 1 - letsencrypt-auto-source/build.py | 54 -- .../letsencrypt-auto.template | 786 ------------------ .../pieces/bootstrappers/arch_common.sh | 37 - .../pieces/bootstrappers/deb_common.sh | 67 -- .../pieces/bootstrappers/free_bsd.sh | 15 - .../pieces/bootstrappers/gentoo_common.sh | 31 - .../pieces/bootstrappers/mac.sh | 48 -- .../pieces/bootstrappers/mageia_common.sh | 30 - .../pieces/bootstrappers/rpm_common.sh | 45 - .../pieces/bootstrappers/rpm_common_base.sh | 60 -- .../pieces/bootstrappers/rpm_python3.sh | 23 - .../bootstrappers/rpm_python3_legacy.sh | 78 -- .../pieces/bootstrappers/smartos.sh | 8 - .../pieces/bootstrappers/suse_common.sh | 36 - .../pieces/certbot-requirements.txt | 12 - .../pieces/check_permissions.py | 81 -- letsencrypt-auto-source/pieces/create_venv.py | 27 - .../pieces/dependency-requirements.txt | 264 ------ letsencrypt-auto-source/pieces/fetch.py | 148 ---- .../pieces/letsencrypt-requirements.txt | 10 - letsencrypt-auto-source/pieces/pipstrap.py | 182 ---- letsencrypt-auto-source/tests/signing.key | 27 - tests/letstest/README.md | 3 +- tests/letstest/multitester.py | 4 +- .../letstest/scripts/test_leauto_upgrades.sh | 160 ---- ...st_letsencrypt_auto_certonly_standalone.sh | 29 - tests/letstest/scripts/test_sdists.sh | 2 +- .../letstest/scripts}/version.py | 7 +- tox.ini | 14 - 35 files changed, 9 insertions(+), 2293 deletions(-) delete mode 100755 letsencrypt-auto-source/build.py delete mode 100755 letsencrypt-auto-source/letsencrypt-auto.template delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/mac.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh delete mode 100644 letsencrypt-auto-source/pieces/bootstrappers/smartos.sh delete mode 100755 letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh delete mode 100644 letsencrypt-auto-source/pieces/certbot-requirements.txt delete mode 100644 letsencrypt-auto-source/pieces/check_permissions.py delete mode 100755 letsencrypt-auto-source/pieces/create_venv.py delete mode 100644 letsencrypt-auto-source/pieces/dependency-requirements.txt delete mode 100644 letsencrypt-auto-source/pieces/fetch.py delete mode 100644 letsencrypt-auto-source/pieces/letsencrypt-requirements.txt delete mode 100755 letsencrypt-auto-source/pieces/pipstrap.py delete mode 100644 letsencrypt-auto-source/tests/signing.key delete mode 100755 tests/letstest/scripts/test_leauto_upgrades.sh delete mode 100755 tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh rename {letsencrypt-auto-source => tests/letstest/scripts}/version.py (75%) diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index cffedfcb2..c8cbe7fb3 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -85,12 +85,6 @@ jobs: IMAGE_NAME: macOS-10.15 PYTHON_VERSION: 3.8 TOXENV: test-farm-apache2 - farmtest-leauto-upgrades: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-leauto-upgrades - farmtest-certonly-standalone: - PYTHON_VERSION: 3.7 - TOXENV: test-farm-certonly-standalone farmtest-sdists: PYTHON_VERSION: 3.7 TOXENV: test-farm-sdists diff --git a/.gitignore b/.gitignore index f4542f412..a50dddc06 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ dist*/ /log* letsencrypt.log certbot.log -letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 poetry.lock # coverage diff --git a/.isort.cfg b/.isort.cfg index 11c895f4d..6b17b459b 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,5 @@ [settings] skip_glob=venv* -skip=letsencrypt-auto-source force_sort_within_sections=True force_single_line=True order_by_type=False diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index d835d0f13..7d53ad649 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -243,8 +243,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): " to --server " + constants.STAGING_URI) helpful.add( "testing", "--debug", action="store_true", default=flag_default("debug"), - help="Show tracebacks in case of errors, and allow certbot-auto " - "execution on experimental platforms") + help="Show tracebacks in case of errors") helpful.add( [None, "certonly", "run"], "--debug-challenges", action="store_true", default=flag_default("debug_challenges"), diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 0442febda..b5ee73211 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -58,7 +58,7 @@ def determine_user_agent(config): ua = ("CertbotACMEClient/{0} ({1}; {2}{8}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") if os.environ.get("CERTBOT_DOCS") == "1": - cli_command = "certbot(-auto)" + cli_command = "certbot" os_info = "OS_NAME OS_VERSION" python_version = "major.minor.patchlevel" else: diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 2acaa71b1..1d88ea84c 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -47,7 +47,6 @@ class DetermineUserAgentTest(test_util.ConfigTestCase): doc_value_check = self.assertNotIn real_value_check = self.assertIn - doc_value_check("certbot(-auto)", ua) doc_value_check("OS_NAME OS_VERSION", ua) doc_value_check("major.minor.patchlevel", ua) real_value_check(util.get_os_info_ua(), ua) diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py deleted file mode 100755 index a1e40fe44..000000000 --- a/letsencrypt-auto-source/build.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -"""Stitch together the letsencrypt-auto script. - -Implement a simple templating language in which {{ some/file }} turns into the -contents of the file at ./pieces/some/file except for certain tokens which have -other, special definitions. - -""" -from os.path import abspath, dirname, join -import re - -from version import certbot_version, file_contents - - -DIR = dirname(abspath(__file__)) - - -def build(version=None, requirements=None): - """Return the built contents of the letsencrypt-auto script. - - :arg version: The version to attach to the script. Default: the version of - the certbot package - :arg requirements: The contents of the requirements file to embed. Default: - contents of dependency-requirements.txt, letsencrypt-requirements.txt, - and certbot-requirements.txt - - """ - special_replacements = { - 'LE_AUTO_VERSION': version or certbot_version(DIR) - } - if requirements: - special_replacements['dependency-requirements.txt'] = '' - special_replacements['letsencrypt-requirements.txt'] = '' - special_replacements['certbot-requirements.txt'] = requirements - - def replacer(match): - token = match.group(1) - if token in special_replacements: - return special_replacements[token] - else: - return file_contents(join(DIR, 'pieces', token)) - - return re.sub(r'{{\s*([A-Za-z0-9_./-]+)\s*}}', - replacer, - file_contents(join(DIR, 'letsencrypt-auto.template'))) - - -def main(): - with open(join(DIR, 'letsencrypt-auto'), 'w') as out: - out.write(build()) - - -if __name__ == '__main__': - main() diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template deleted file mode 100755 index 72876bb59..000000000 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ /dev/null @@ -1,786 +0,0 @@ -#!/bin/sh -# -# Download and run the latest release version of the Certbot client. -# -# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING -# -# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE -# "--no-self-upgrade" FLAG -# -# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS -# letsencrypt-auto-source/letsencrypt-auto.template AND -# letsencrypt-auto-source/pieces/bootstrappers/* - -set -e # Work even if somebody does "sh thisscript.sh". - -# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, -# if you want to change where the virtual environment will be installed - -# HOME might not be defined when being run through something like systemd -if [ -z "$HOME" ]; then - HOME=~root -fi -if [ -z "$XDG_DATA_HOME" ]; then - XDG_DATA_HOME=~/.local/share -fi -if [ -z "$VENV_PATH" ]; then - # We export these values so they are preserved properly if this script is - # rerun with sudo/su where $HOME/$XDG_DATA_HOME may have a different value. - export OLD_VENV_PATH="$XDG_DATA_HOME/letsencrypt" - export VENV_PATH="/opt/eff.org/certbot/venv" -fi -VENV_BIN="$VENV_PATH/bin" -BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" -BASENAME=$(basename $0) -USAGE="Usage: $BASENAME [OPTIONS] -A self-updating wrapper script for the Certbot ACME client. When run, updates -to both this script and certbot will be downloaded and installed. After -ensuring you have the latest versions installed, certbot will be invoked with -all arguments you have provided. - -Help for certbot itself cannot be provided until it is installed. - - --debug attempt experimental installation - -h, --help print this help - -n, --non-interactive, --noninteractive run without asking for user input - --no-bootstrap do not install OS dependencies - --no-permissions-check do not warn about file system permissions - --no-self-upgrade do not download updates - --os-packages-only install OS dependencies and exit - --install-only install certbot, upgrade if needed, and exit - -v, --verbose provide more output - -q, --quiet provide only update/error output; - implies --non-interactive - -All arguments are accepted and forwarded to the Certbot client when run." -export CERTBOT_AUTO="$0" - -for arg in "$@" ; do - case "$arg" in - --debug) - DEBUG=1;; - --os-packages-only) - OS_PACKAGES_ONLY=1;; - --install-only) - INSTALL_ONLY=1;; - --no-self-upgrade) - # Do not upgrade this script (also prevents client upgrades, because each - # copy of the script pins a hash of the python client) - NO_SELF_UPGRADE=1;; - --no-permissions-check) - NO_PERMISSIONS_CHECK=1;; - --no-bootstrap) - NO_BOOTSTRAP=1;; - --help) - HELP=1;; - --noninteractive|--non-interactive) - NONINTERACTIVE=1;; - --quiet) - QUIET=1;; - renew) - ASSUME_YES=1;; - --verbose) - VERBOSE=1;; - -[!-]*) - OPTIND=1 - while getopts ":hnvq" short_arg $arg; do - case "$short_arg" in - h) - HELP=1;; - n) - NONINTERACTIVE=1;; - q) - QUIET=1;; - v) - VERBOSE=1;; - esac - done;; - esac -done - -if [ $BASENAME = "letsencrypt-auto" ]; then - # letsencrypt-auto does not respect --help or --yes for backwards compatibility - NONINTERACTIVE=1 - HELP=0 -fi - -# Set ASSUME_YES to 1 if QUIET or NONINTERACTIVE -if [ "$QUIET" = 1 -o "$NONINTERACTIVE" = 1 ]; then - ASSUME_YES=1 -fi - -say() { - if [ "$QUIET" != 1 ]; then - echo "$@" - fi -} - -error() { - echo "$@" -} - -# Support for busybox and others where there is no "command", -# but "which" instead -if command -v command > /dev/null 2>&1 ; then - export EXISTS="command -v" -elif which which > /dev/null 2>&1 ; then - export EXISTS="which" -else - error "Cannot find command nor which... please install one!" - exit 1 -fi - -# Certbot itself needs root access for almost all modes of operation. -# certbot-auto needs root access to bootstrap OS dependencies and install -# Certbot at a protected path so it can be safely run as root. To accomplish -# this, this script will attempt to run itself as root if it doesn't have the -# necessary privileges by using `sudo` or falling back to `su` if it is not -# available. The mechanism used to obtain root access can be set explicitly by -# setting the environment variable LE_AUTO_SUDO to 'sudo', 'su', 'su_sudo', -# 'SuSudo', or '' as used below. - -# Because the parameters in `su -c` has to be a string, -# we need to properly escape it. -SuSudo() { - args="" - # This `while` loop iterates over all parameters given to this function. - # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string - # will be wrapped in a pair of `'`, then appended to `$args` string - # For example, `echo "It's only 1\$\!"` will be escaped to: - # 'echo' 'It'"'"'s only 1$!' - # │ │└┼┘│ - # │ │ │ └── `'s only 1$!'` the literal string - # │ │ └── `\"'\"` is a single quote (as a string) - # │ └── `'It'`, to be concatenated with the strings following it - # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself - while [ $# -ne 0 ]; do - args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " - shift - done - su root -c "$args" -} - -# Sets the environment variable SUDO to be the name of the program or function -# to call to get root access. If this script already has root privleges, SUDO -# is set to an empty string. The value in SUDO should be run with the command -# to called with root privileges as arguments. -SetRootAuthMechanism() { - SUDO="" - if [ -n "${LE_AUTO_SUDO+x}" ]; then - case "$LE_AUTO_SUDO" in - SuSudo|su_sudo|su) - SUDO=SuSudo - ;; - sudo) - SUDO="sudo -E" - ;; - '') - # If we're not running with root, don't check that this script can only - # be modified by system users and groups. - NO_PERMISSIONS_CHECK=1 - ;; - *) - error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." - exit 1 - esac - say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." - else - if test "`id -u`" -ne "0" ; then - if $EXISTS sudo 1>/dev/null 2>&1; then - SUDO="sudo -E" - else - say \"sudo\" is not available, will use \"su\" for installation steps... - SUDO=SuSudo - fi - fi - fi -} - -if [ "$1" = "--cb-auto-has-root" ]; then - shift 1 -else - SetRootAuthMechanism - if [ -n "$SUDO" ]; then - say "Requesting to rerun $0 with root privileges..." - $SUDO "$0" --cb-auto-has-root "$@" - exit 0 - fi -fi - -# Runs this script again with the given arguments. --cb-auto-has-root is added -# to the command line arguments to ensure we don't try to acquire root a -# second time. After the script is rerun, we exit the current script. -RerunWithArgs() { - "$0" --cb-auto-has-root "$@" - exit 0 -} - -BootstrapMessage() { - # Arguments: Platform name - say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" -} - -ExperimentalBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "FATAL: $1 support is very experimental at present..." - error "if you would like to work on improving it, please ensure you have backups" - error "and then run this script again with the --debug flag!" - error "Alternatively, you can install OS dependencies yourself and run this script" - error "again with --no-bootstrap." - exit 1 - fi -} - -DeprecationBootstrap() { - # Arguments: Platform name, bootstrap function name - if [ "$DEBUG" = 1 ]; then - if [ "$2" != "" ]; then - BootstrapMessage $1 - $2 - fi - else - error "WARNING: certbot-auto support for this $1 is DEPRECATED!" - error "Please visit certbot.eff.org to learn how to download a version of" - error "Certbot that is packaged for your system. While an existing version" - error "of certbot-auto may work currently, we have stopped supporting updating" - error "system packages for your system. Please switch to a packaged version" - error "as soon as possible." - exit 1 - fi -} - -MIN_PYTHON_2_VERSION="2.7" -MIN_PYVER2=$(echo "$MIN_PYTHON_2_VERSION" | sed 's/\.//') -MIN_PYTHON_3_VERSION="3.6" -MIN_PYVER3=$(echo "$MIN_PYTHON_3_VERSION" | sed 's/\.//') -# Sets LE_PYTHON to Python version string and PYVER to the first two -# digits of the python version. -# MIN_PYVER and MIN_PYTHON_VERSION are also set by this function, and their -# values depend on if we try to use Python 3 or Python 2. -DeterminePythonVersion() { - # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python - # - # If no Python is found, PYVER is set to 0. - if [ "$USE_PYTHON_3" = 1 ]; then - MIN_PYVER=$MIN_PYVER3 - MIN_PYTHON_VERSION=$MIN_PYTHON_3_VERSION - for LE_PYTHON in "$LE_PYTHON" python3; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - else - MIN_PYVER=$MIN_PYVER2 - MIN_PYTHON_VERSION=$MIN_PYTHON_2_VERSION - for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do - # Break (while keeping the LE_PYTHON value) if found. - $EXISTS "$LE_PYTHON" > /dev/null && break - done - fi - if [ "$?" != "0" ]; then - if [ "$1" != "NOCRASH" ]; then - error "Cannot find any Pythons; please install one!" - exit 1 - else - PYVER=0 - return 0 - fi - fi - - PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - if [ "$1" != "NOCRASH" ]; then - error "You have an ancient version of Python entombed in your operating system..." - error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION." - exit 1 - fi - fi -} - -{{ bootstrappers/deb_common.sh }} -{{ bootstrappers/rpm_common_base.sh }} -{{ bootstrappers/rpm_common.sh }} -{{ bootstrappers/rpm_python3_legacy.sh }} -{{ bootstrappers/rpm_python3.sh }} -{{ bootstrappers/suse_common.sh }} -{{ bootstrappers/arch_common.sh }} -{{ bootstrappers/gentoo_common.sh }} -{{ bootstrappers/free_bsd.sh }} -{{ bootstrappers/mac.sh }} -{{ bootstrappers/smartos.sh }} -{{ bootstrappers/mageia_common.sh }} - -# Set Bootstrap to the function that installs OS dependencies on this system -# and BOOTSTRAP_VERSION to the unique identifier for the current version of -# that function. If Bootstrap is set to a function that doesn't install any -# packages BOOTSTRAP_VERSION is not set. -if [ -f /etc/debian_version ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/mageia-release ]; then - # Mageia has both /etc/mageia-release and /etc/redhat-release - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/redhat-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 - # Run DeterminePythonVersion to decide on the basis of available Python versions - # whether to use 2.x or 3.x on RedHat-like systems. - # Then, revert LE_PYTHON to its previous state. - prev_le_python="$LE_PYTHON" - unset LE_PYTHON - DeterminePythonVersion "NOCRASH" - - RPM_DIST_NAME=`(. /etc/os-release 2> /dev/null && echo $ID) || echo "unknown"` - - if [ "$PYVER" -eq 26 -a $(uname -m) != 'x86_64' ]; then - # 32 bits CentOS 6 and affiliates are not supported anymore by certbot-auto. - DEPRECATED_OS=1 - fi - - # Set RPM_DIST_VERSION to VERSION_ID from /etc/os-release after splitting on - # '.' characters (e.g. "8.0" becomes "8"). If the command exits with an - # error, RPM_DIST_VERSION is set to "unknown". - RPM_DIST_VERSION=$( (. /etc/os-release 2> /dev/null && echo "$VERSION_ID") | cut -d '.' -f1 || echo "unknown") - - # If RPM_DIST_VERSION is an empty string or it contains any nonnumeric - # characters, the value is unexpected so we set RPM_DIST_VERSION to 0. - if [ -z "$RPM_DIST_VERSION" ] || [ -n "$(echo "$RPM_DIST_VERSION" | tr -d '[0-9]')" ]; then - RPM_DIST_VERSION=0 - fi - - # Handle legacy RPM distributions - if [ "$PYVER" -eq 26 ]; then - # Check if an automated bootstrap can be achieved on this system. - if ! Python36SclIsAvailable; then - INTERACTIVE_BOOTSTRAP=1 - fi - - USE_PYTHON_3=1 - - # Try now to enable SCL rh-python36 for systems already bootstrapped - # NB: EnablePython36SCL has been defined along with BootstrapRpmPython3Legacy in certbot-auto - EnablePython36SCL - else - # Starting to Fedora 29, python2 is on a deprecation path. Let's move to python3 then. - # RHEL 8 also uses python3 by default. - if [ "$RPM_DIST_NAME" = "fedora" -a "$RPM_DIST_VERSION" -ge 29 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "rhel" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - elif [ "$RPM_DIST_NAME" = "centos" -a "$RPM_DIST_VERSION" -ge 8 ]; then - RPM_USE_PYTHON_3=1 - else - RPM_USE_PYTHON_3=0 - fi - - if [ "$RPM_USE_PYTHON_3" = 1 ]; then - USE_PYTHON_3=1 - fi - fi - - LE_PYTHON="$prev_le_python" -elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/arch-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/manjaro-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/gentoo-release ]; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq FreeBSD ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif uname | grep -iq Darwin ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -else - DEPRECATED_OS=1 - NO_SELF_UPGRADE=1 -fi - -# We handle this case after determining the normal bootstrap version to allow -# variables like USE_PYTHON_3 to be properly set. As described above, if the -# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not -# be set so we unset it here. -if [ "$NO_BOOTSTRAP" = 1 ]; then - Bootstrap() { - : - } - unset BOOTSTRAP_VERSION -fi - -if [ "$DEPRECATED_OS" = 1 ]; then - Bootstrap() { - error "Skipping bootstrap because certbot-auto is deprecated on this system." - } - unset BOOTSTRAP_VERSION -fi - -# Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used -# to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set -# if it is unknown how OS dependencies were installed on this system. -SetPrevBootstrapVersion() { - if [ -f $BOOTSTRAP_VERSION_PATH ]; then - PREV_BOOTSTRAP_VERSION=$(cat "$BOOTSTRAP_VERSION_PATH") - # The list below only contains bootstrap version strings that existed before - # we started writing them to disk. - # - # DO NOT MODIFY THIS LIST UNLESS YOU KNOW WHAT YOU'RE DOING! - elif grep -Fqx "$BOOTSTRAP_VERSION" << "UNLIKELY_EOF" -BootstrapDebCommon 1 -BootstrapMageiaCommon 1 -BootstrapRpmCommon 1 -BootstrapSuseCommon 1 -BootstrapArchCommon 1 -BootstrapGentooCommon 1 -BootstrapFreeBsd 1 -BootstrapMac 1 -BootstrapSmartOS 1 -UNLIKELY_EOF - then - # If there's no bootstrap version saved to disk, but the currently selected - # bootstrap script is from before we started saving the version number, - # return the currently selected version to prevent us from rebootstrapping - # unnecessarily. - PREV_BOOTSTRAP_VERSION="$BOOTSTRAP_VERSION" - fi -} - -TempDir() { - mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || macOS -} - -# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise, -# returns a non-zero number. -OldVenvExists() { - [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] -} - -# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. -# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated -# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 -# is outdated, and "UP_TO_DATE" if not. -# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. -CompareVersions() { - "$1" - "$2" "$3" << "UNLIKELY_EOF" -import sys -from distutils.version import StrictVersion - -try: - current = StrictVersion(sys.argv[1]) -except ValueError: - sys.stdout.write('UNOFFICIAL') - sys.exit() - -try: - remote = StrictVersion(sys.argv[2]) -except ValueError: - sys.stdout.write('UP_TO_DATE') - sys.exit() - -if current < remote: - sys.stdout.write('OUTDATED') -else: - sys.stdout.write('UP_TO_DATE') -UNLIKELY_EOF -} - -# Create a new virtual environment for Certbot. It will overwrite any existing one. -# Parameters: LE_PYTHON, VENV_PATH, PYVER, VERBOSE -CreateVenv() { - "$1" - "$2" "$3" "$4" << "UNLIKELY_EOF" -{{ create_venv.py }} -UNLIKELY_EOF -} - -# Check that the given PATH_TO_CHECK has secured permissions. -# Parameters: LE_PYTHON, PATH_TO_CHECK -CheckPathPermissions() { - "$1" - "$2" << "UNLIKELY_EOF" -{{ check_permissions.py }} -UNLIKELY_EOF -} - -if [ "$1" = "--le-auto-phase2" ]; then - # Phase 2: Create venv, install LE, and run. - - shift 1 # the --le-auto-phase2 arg - - if [ "$DEPRECATED_OS" = 1 ]; then - # Phase 2 damage control mode for deprecated OSes. - # In this situation, we bypass any bootstrap or certbot venv setup. - error "Your system is not supported by certbot-auto anymore." - - if [ ! -d "$VENV_PATH" ] && OldVenvExists; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - - if [ -f "$VENV_BIN/letsencrypt" -a "$INSTALL_ONLY" != 1 ]; then - error "certbot-auto and its Certbot installation will no longer receive updates." - error "You will not receive any bug fixes including those fixing server compatibility" - error "or security problems." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - "$VENV_BIN/letsencrypt" "$@" - exit 0 - else - error "Certbot cannot be installed." - error "Please visit https://certbot.eff.org/ to check for other alternatives." - exit 1 - fi - fi - - SetPrevBootstrapVersion - - if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then - unset LE_PYTHON - fi - - INSTALLED_VERSION="none" - if [ -d "$VENV_PATH" ] || OldVenvExists; then - # If the selected Bootstrap function isn't a noop and it differs from the - # previously used version - if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then - # Check if we can rebootstrap without manual user intervention: this requires that - # certbot-auto is in non-interactive mode AND selected bootstrap does not claim to - # require a manual user intervention. - if [ "$NONINTERACTIVE" = 1 -a "$INTERACTIVE_BOOTSTRAP" != 1 ]; then - CAN_REBOOTSTRAP=1 - fi - # Check if rebootstrap can be done non-interactively and current shell is non-interactive - # (true if stdin and stdout are not attached to a terminal). - if [ \( "$CAN_REBOOTSTRAP" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then - if [ -d "$VENV_PATH" ]; then - rm -rf "$VENV_PATH" - fi - # In the case the old venv was just a symlink to the new one, - # OldVenvExists is now false because we deleted the venv at VENV_PATH. - if OldVenvExists; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - RerunWithArgs "$@" - # Otherwise bootstrap needs to be done manually by the user. - else - # If it is because bootstrapping is interactive, --non-interactive will be of no use. - if [ "$INTERACTIVE_BOOTSTRAP" = 1 ]; then - error "Skipping upgrade because new OS dependencies may need to be installed." - error "This requires manual user intervention: please run this script again manually." - # If this is because of the environment (eg. non interactive shell without - # --non-interactive flag set), help the user in that direction. - else - error "Skipping upgrade because new OS dependencies may need to be installed." - error - error "To upgrade to a newer version, please run this script again manually so you can" - error "approve changes or with --non-interactive on the command line to automatically" - error "install any required packages." - fi - # Set INSTALLED_VERSION to be the same so we don't update the venv - INSTALLED_VERSION="$LE_AUTO_VERSION" - # Continue to use OLD_VENV_PATH if the new venv doesn't exist - if [ ! -d "$VENV_PATH" ]; then - VENV_BIN="$OLD_VENV_PATH/bin" - fi - fi - elif [ -f "$VENV_BIN/letsencrypt" ]; then - # --version output ran through grep due to python-cryptography DeprecationWarnings - # grep for both certbot and letsencrypt until certbot and shim packages have been released - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) - if [ -z "$INSTALLED_VERSION" ]; then - error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 - "$VENV_BIN/letsencrypt" --version - exit 1 - fi - fi - fi - - if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - say "Creating virtual environment..." - DeterminePythonVersion - CreateVenv "$LE_PYTHON" "$VENV_PATH" "$PYVER" "$VERBOSE" - - if [ -n "$BOOTSTRAP_VERSION" ]; then - echo "$BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - elif [ -n "$PREV_BOOTSTRAP_VERSION" ]; then - echo "$PREV_BOOTSTRAP_VERSION" > "$BOOTSTRAP_VERSION_PATH" - fi - - say "Installing Python packages..." - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # There is no $ interpolation due to quotes on starting heredoc delimiter. - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -{{ dependency-requirements.txt }} -{{ letsencrypt-requirements.txt }} -{{ certbot-requirements.txt }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" -{{ pipstrap.py }} -UNLIKELY_EOF - # ------------------------------------------------------------------------- - # Set PATH so pipstrap upgrades the right (v)env: - PATH="$VENV_BIN:$PATH" "$VENV_BIN/python" "$TEMP_DIR/pipstrap.py" - set +e - if [ "$VERBOSE" = 1 ]; then - "$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" - else - PIP_OUT=`"$VENV_BIN/pip" install --disable-pip-version-check --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` - fi - PIP_STATUS=$? - set -e - if [ "$PIP_STATUS" != 0 ]; then - # Report error. (Otherwise, be quiet.) - error "Had a problem while installing Python packages." - if [ "$VERBOSE" != 1 ]; then - error - error "pip prints the following errors: " - error "=====================================================" - error "$PIP_OUT" - error "=====================================================" - error - error "Certbot has problem setting up the virtual environment." - - if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - error - error "Based on your pip output, the problem can likely be fixed by " - error "increasing the available memory." - else - error - error "We were not be able to guess the right solution from your pip " - error "output." - fi - - error - error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - error "for possible solutions." - error "You may also find some support resources at https://certbot.eff.org/support/ ." - fi - rm -rf "$VENV_PATH" - exit 1 - fi - - if [ -d "$OLD_VENV_PATH" -a ! -L "$OLD_VENV_PATH" ]; then - rm -rf "$OLD_VENV_PATH" - ln -s "$VENV_PATH" "$OLD_VENV_PATH" - fi - - say "Installation succeeded." - fi - - # If you're modifying any of the code after this point in this current `if` block, you - # may need to update the "$DEPRECATED_OS" = 1 case at the beginning of phase 2 as well. - - if [ "$INSTALL_ONLY" = 1 ]; then - say "Certbot is installed." - exit 0 - fi - - "$VENV_BIN/letsencrypt" "$@" - -else - # Phase 1: Upgrade certbot-auto if necessary, then self-invoke. - # - # Each phase checks the version of only the thing it is responsible for - # upgrading. Phase 1 checks the version of the latest release of - # certbot-auto (which is always the same as that of the certbot - # package). Phase 2 checks the version of the locally installed certbot. - export PHASE_1_VERSION="$LE_AUTO_VERSION" - - if [ ! -f "$VENV_BIN/letsencrypt" ]; then - if ! OldVenvExists; then - if [ "$HELP" = 1 ]; then - echo "$USAGE" - exit 0 - fi - # If it looks like we've never bootstrapped before, bootstrap: - Bootstrap - fi - fi - if [ "$OS_PACKAGES_ONLY" = 1 ]; then - say "OS packages installed." - exit 0 - fi - - DeterminePythonVersion "NOCRASH" - # Don't warn about file permissions if the user disabled the check or we - # can't find an up-to-date Python. - if [ "$PYVER" -ge "$MIN_PYVER" -a "$NO_PERMISSIONS_CHECK" != 1 ]; then - # If the script fails for some reason, don't break certbot-auto. - set +e - # Suppress unexpected error output. - CHECK_PERM_OUT=$(CheckPathPermissions "$LE_PYTHON" "$0" 2>/dev/null) - CHECK_PERM_STATUS="$?" - set -e - # Only print output if the script ran successfully and it actually produced - # output. The latter check resolves - # https://github.com/certbot/certbot/issues/7012. - if [ "$CHECK_PERM_STATUS" = 0 -a -n "$CHECK_PERM_OUT" ]; then - error "$CHECK_PERM_OUT" - fi - fi - - if [ "$NO_SELF_UPGRADE" != 1 ]; then - TEMP_DIR=$(TempDir) - trap 'rm -rf "$TEMP_DIR"' EXIT - # --------------------------------------------------------------------------- - cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" -{{ fetch.py }} -UNLIKELY_EOF - # --------------------------------------------------------------------------- - if [ "$PYVER" -lt "$MIN_PYVER" ]; then - error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." - elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - error "WARNING: unable to check for updates." - fi - - # If for any reason REMOTE_VERSION is not set, let's assume certbot-auto is up-to-date, - # and do not go into the self-upgrading process. - if [ -n "$REMOTE_VERSION" ]; then - LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` - - if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then - say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" - elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then - say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." - - # Now we drop into Python so we don't have to install even more - # dependencies (curl, etc.), for better flow control, and for the option of - # future Windows compatibility. - "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - - # Install new copy of certbot-auto. - # TODO: Deal with quotes in pathnames. - say "Replacing certbot-auto..." - # Clone permissions with cp. chmod and chown don't have a --reference - # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: - cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" - # Using mv rather than cp leaves the old file descriptor pointing to the - # original copy so the shell can continue to read it unmolested. mv across - # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the - # cp is unlikely to fail if the rm doesn't. - mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - fi # A newer version is available. - fi - fi # Self-upgrading is allowed. - - RerunWithArgs --le-auto-phase2 "$@" -fi diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh deleted file mode 100755 index 3be78d3f8..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ /dev/null @@ -1,37 +0,0 @@ -# If new packages are installed by BootstrapArchCommon below, this version -# number must be increased. -BOOTSTRAP_ARCH_COMMON_VERSION=1 - -BootstrapArchCommon() { - # Tested with: - # - ArchLinux (x86_64) - # - # "python-virtualenv" is Python3, but "python2-virtualenv" provides - # only "virtualenv2" binary, not "virtualenv". - - deps=" - python2 - python-virtualenv - gcc - augeas - openssl - libffi - ca-certificates - pkg-config - " - - # pacman -T exits with 127 if there are missing dependencies - missing=$(pacman -T $deps) || true - - if [ "$ASSUME_YES" = 1 ]; then - noconfirm="--noconfirm" - fi - - if [ "$missing" ]; then - if [ "$QUIET" = 1 ]; then - pacman -S --needed $missing $noconfirm > /dev/null - else - pacman -S --needed $missing $noconfirm - fi - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh deleted file mode 100644 index 93bdc63b4..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ /dev/null @@ -1,67 +0,0 @@ -# If new packages are installed by BootstrapDebCommon below, this version -# number must be increased. -BOOTSTRAP_DEB_COMMON_VERSION=1 - -BootstrapDebCommon() { - # Current version tested with: - # - # - Ubuntu - # - 14.04 (x64) - # - 15.04 (x64) - # - Debian - # - 7.9 "wheezy" (x64) - # - sid (2015-10-21) (x64) - - # Past versions tested with: - # - # - Debian 8.0 "jessie" (x64) - # - Raspbian 7.8 (armhf) - - # Believed not to work: - # - # - Debian 6.0.10 "squeeze" (x64) - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... - - # virtualenv binary can be found in different packages depending on - # distro version (#346) - - virtualenv= - # virtual env is known to apt and is installable - if apt-cache show virtualenv > /dev/null 2>&1 ; then - if ! LC_ALL=C apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then - virtualenv="virtualenv" - fi - fi - - if apt-cache show python-virtualenv > /dev/null 2>&1; then - virtualenv="$virtualenv python-virtualenv" - fi - - augeas_pkg="libaugeas0 augeas-lenses" - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ - python \ - python-dev \ - $virtualenv \ - gcc \ - $augeas_pkg \ - libssl-dev \ - openssl \ - libffi-dev \ - ca-certificates \ - - - if ! $EXISTS virtualenv > /dev/null ; then - error Failed to install a working \"virtualenv\" command, exiting - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh b/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh deleted file mode 100755 index a67c85619..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/free_bsd.sh +++ /dev/null @@ -1,15 +0,0 @@ -# If new packages are installed by BootstrapFreeBsd below, this version number -# must be increased. -BOOTSTRAP_FREEBSD_VERSION=1 - -BootstrapFreeBsd() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG="--quiet" - fi - - pkg install -Ay $QUIET_FLAG \ - python \ - py27-virtualenv \ - augeas \ - libffi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh deleted file mode 100755 index e2d24b5fb..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/gentoo_common.sh +++ /dev/null @@ -1,31 +0,0 @@ -# If new packages are installed by BootstrapGentooCommon below, this version -# number must be increased. -BOOTSTRAP_GENTOO_COMMON_VERSION=1 - -BootstrapGentooCommon() { - PACKAGES=" - dev-lang/python:2.7 - dev-python/virtualenv - app-admin/augeas - dev-libs/openssl - dev-libs/libffi - app-misc/ca-certificates - virtual/pkgconfig" - - ASK_OPTION="--ask" - if [ "$ASSUME_YES" = 1 ]; then - ASK_OPTION="" - fi - - case "$PACKAGE_MANAGER" in - (paludis) - cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x - ;; - (pkgcore) - pmerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - (portage|*) - emerge --noreplace --oneshot $ASK_OPTION $PACKAGES - ;; - esac -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh deleted file mode 100755 index 9e26d3389..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ /dev/null @@ -1,48 +0,0 @@ -# If new packages are installed by BootstrapMac below, this version number must -# be increased. -BOOTSTRAP_MAC_VERSION=1 - -BootstrapMac() { - if hash brew 2>/dev/null; then - say "Using Homebrew to install dependencies..." - pkgman=brew - pkgcmd="brew install" - elif hash port 2>/dev/null; then - say "Using MacPorts to install dependencies..." - pkgman=port - pkgcmd="port install" - else - say "No Homebrew/MacPorts; installing Homebrew..." - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - pkgman=brew - pkgcmd="brew install" - fi - - $pkgcmd augeas - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ - -o "$(which python)" = "/usr/bin/python" ]; then - # We want to avoid using the system Python because it requires root to use pip. - # python.org, MacPorts or HomeBrew Python installations should all be OK. - say "Installing python..." - $pkgcmd python - fi - - # Workaround for _dlopen not finding augeas on macOS - if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - say "Applying augeas workaround" - mkdir -p /usr/local/lib/ - ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ - fi - - if ! hash pip 2>/dev/null; then - say "pip not installed" - say "Installing pip..." - curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python - fi - - if ! hash virtualenv 2>/dev/null; then - say "virtualenv not installed." - say "Installing with pip..." - pip install virtualenv - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh deleted file mode 100644 index dfa5b47f3..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh +++ /dev/null @@ -1,30 +0,0 @@ -# If new packages are installed by BootstrapMageiaCommon below, this version -# number must be increased. -BOOTSTRAP_MAGEIA_COMMON_VERSION=1 - -BootstrapMageiaCommon() { - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi - - if ! urpmi --force $QUIET_FLAG \ - python \ - libpython-devel \ - python-virtualenv - then - error "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi - - if ! urpmi --force $QUIET_FLAG \ - git \ - gcc \ - python-augeas \ - libopenssl-devel \ - libffi-devel \ - rootcerts - then - error "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh deleted file mode 100755 index 80d55a393..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ /dev/null @@ -1,45 +0,0 @@ -# If new packages are installed by BootstrapRpmCommon below, this version -# number must be increased. -BOOTSTRAP_RPM_COMMON_VERSION=1 - -BootstrapRpmCommon() { - # Tested with: - # - Fedora 20, 21, 22, 23 (x64) - # - Centos 7 (x64: on DigitalOcean droplet) - # - CentOS 7 Minimal install in a Hyper-V VM - # - CentOS 6 - - InitializeRPMCommonBase - - # Most RPM distros use the "python" or "python-" naming convention. Let's try that first. - if $TOOL list python >/dev/null 2>&1; then - python_pkgs="$python - python-devel - python-virtualenv - python-tools - python-pip - " - # Fedora 26 starts to use the prefix python2 for python2 based packages. - # this elseif is theoretically for any Fedora over version 26: - elif $TOOL list python2 >/dev/null 2>&1; then - python_pkgs="$python2 - python2-libs - python2-setuptools - python2-devel - python2-virtualenv - python2-tools - python2-pip - " - # Some distros and older versions of current distros use a "python27" - # instead of the "python" or "python-" naming convention. - else - python_pkgs="$python27 - python27-devel - python27-virtualenv - python27-tools - python27-pip - " - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh deleted file mode 100644 index 2b00b199b..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh +++ /dev/null @@ -1,60 +0,0 @@ -# If new packages are installed by BootstrapRpmCommonBase below, version -# numbers in rpm_common.sh and rpm_python3.sh must be increased. - -# Sets TOOL to the name of the package manager -# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG. -# Note: this function is called both while selecting the bootstrap scripts and -# during the actual bootstrap. Some things like prompting to user can be done in the latter -# case, but not in the former one. -InitializeRPMCommonBase() { - if type dnf 2>/dev/null - then - TOOL=dnf - elif type yum 2>/dev/null - then - TOOL=yum - - else - error "Neither yum nor dnf found. Aborting bootstrap!" - exit 1 - fi - - if [ "$ASSUME_YES" = 1 ]; then - YES_FLAG="-y" - fi - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='--quiet' - fi -} - -BootstrapRpmCommonBase() { - # Arguments: whitespace-delimited python packages to install - - InitializeRPMCommonBase # This call is superfluous in practice - - pkgs=" - gcc - augeas-libs - openssl - openssl-devel - libffi-devel - redhat-rpm-config - ca-certificates - " - - # Add the python packages - pkgs="$pkgs - $1 - " - - if $TOOL list installed "httpd" >/dev/null 2>&1; then - pkgs="$pkgs - mod_ssl - " - fi - - if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then - error "Could not install OS dependencies. Aborting bootstrap!" - exit 1 - fi -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh deleted file mode 100644 index ac0553db5..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh +++ /dev/null @@ -1,23 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_VERSION=1 - -BootstrapRpmPython3() { - # Tested with: - # - Fedora 29 - - InitializeRPMCommonBase - - # Fedora 29 must use python3-virtualenv - if $TOOL list python3-virtualenv >/dev/null 2>&1; then - python_pkgs="python3 - python3-virtualenv - python3-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "$python_pkgs" -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh deleted file mode 100644 index febfc7a83..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3_legacy.sh +++ /dev/null @@ -1,78 +0,0 @@ -# If new packages are installed by BootstrapRpmPython3 below, this version -# number must be increased. -BOOTSTRAP_RPM_PYTHON3_LEGACY_VERSION=1 - -# Checks if rh-python36 can be installed. -Python36SclIsAvailable() { - InitializeRPMCommonBase >/dev/null 2>&1; - - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - return 0 - fi - if "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - return 0 - fi - return 1 -} - -# Try to enable rh-python36 from SCL if it is necessary and possible. -EnablePython36SCL() { - if "$EXISTS" python3.6 > /dev/null 2> /dev/null; then - return 0 - fi - if [ ! -f /opt/rh/rh-python36/enable ]; then - return 0 - fi - set +e - if ! . /opt/rh/rh-python36/enable; then - error 'Unable to enable rh-python36!' - exit 1 - fi - set -e -} - -# This bootstrap concerns old RedHat-based distributions that do not ship by default -# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing -# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. -BootstrapRpmPython3Legacy() { - # Tested with: - # - CentOS 6 - - InitializeRPMCommonBase - - if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then - echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." - if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then - error "Enable the SCL repository and try running Certbot again." - exit 1 - fi - if [ "${ASSUME_YES}" = 1 ]; then - /bin/echo -n "Enabling the SCL repository in 3 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -ne "\e[0K\rEnabling the SCL repository in 2 seconds... (Press Ctrl-C to cancel)" - sleep 1s - /bin/echo -e "\e[0K\rEnabling the SCL repository in 1 second... (Press Ctrl-C to cancel)" - sleep 1s - fi - if ! "${TOOL}" install "${YES_FLAG}" "${QUIET_FLAG}" centos-release-scl; then - error "Could not enable SCL. Aborting bootstrap!" - exit 1 - fi - fi - - # CentOS 6 must use rh-python36 from SCL - if "${TOOL}" list rh-python36 >/dev/null 2>&1; then - python_pkgs="rh-python36-python - rh-python36-python-virtualenv - rh-python36-python-devel - " - else - error "No supported Python package available to install. Aborting bootstrap!" - exit 1 - fi - - BootstrapRpmCommonBase "${python_pkgs}" - - # Enable SCL rh-python36 after bootstrapping. - EnablePython36SCL -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh b/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh deleted file mode 100644 index ac7c0ed4a..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh +++ /dev/null @@ -1,8 +0,0 @@ -# If new packages are installed by BootstrapSmartOS below, this version number -# must be increased. -BOOTSTRAP_SMARTOS_VERSION=1 - -BootstrapSmartOS() { - pkgin update - pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' -} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh deleted file mode 100755 index 7fa28ce50..000000000 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ /dev/null @@ -1,36 +0,0 @@ -# If new packages are installed by BootstrapSuseCommon below, this version -# number must be increased. -BOOTSTRAP_SUSE_COMMON_VERSION=1 - -BootstrapSuseCommon() { - # SLE12 don't have python-virtualenv - - if [ "$ASSUME_YES" = 1 ]; then - zypper_flags="-nq" - install_flags="-l" - fi - - if [ "$QUIET" = 1 ]; then - QUIET_FLAG='-qq' - fi - - if zypper search -x python-virtualenv >/dev/null 2>&1; then - OPENSUSE_VIRTUALENV_PACKAGES="python-virtualenv" - else - # Since Leap 15.0 (and associated Tumbleweed version), python-virtualenv - # is a source package, and python2-virtualenv must be used instead. - # Also currently python2-setuptools is not a dependency of python2-virtualenv, - # while it should be. Installing it explicitly until upstream fix. - OPENSUSE_VIRTUALENV_PACKAGES="python2-virtualenv python2-setuptools" - fi - - zypper $QUIET_FLAG $zypper_flags in $install_flags \ - python \ - python-devel \ - $OPENSUSE_VIRTUALENV_PACKAGES \ - gcc \ - augeas-lenses \ - libopenssl-devel \ - libffi-devel \ - ca-certificates -} diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt deleted file mode 100644 index 90c67663b..000000000 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -certbot==1.14.0 \ - --hash=sha256:67b4d26ceaea6c7f8325d0d45169e7a165a2cabc7122c84bc971ba068ca19cca \ - --hash=sha256:959ea90c6bb8dca38eab9772722cb940972ef6afcd5f15deef08b3c3636841eb -acme==1.14.0 \ - --hash=sha256:4f48c41261202f1a389ec2986b2580b58f53e0d5a1ae2463b34318d78b87fc66 \ - --hash=sha256:61daccfb0343628cbbca551a7fc4c82482113952c21db3fe0c585b7c98fa1c35 -certbot-apache==1.14.0 \ - --hash=sha256:b757038db23db707c44630fecb46e99172bd791f0db5a8e623c0842613c4d3d9 \ - --hash=sha256:887fe4a21af2de1e5c2c9428bacba6eb7c1219257bc70f1a1d8447c8a321adb0 -certbot-nginx==1.14.0 \ - --hash=sha256:8916a815437988d6c192df9f035bb7a176eab20eee0956677b335d0698d243fb \ - --hash=sha256:cc2a8a0de56d9bb6b2efbda6c80c647dad8db2bb90675cac03ade94bd5fc8597 diff --git a/letsencrypt-auto-source/pieces/check_permissions.py b/letsencrypt-auto-source/pieces/check_permissions.py deleted file mode 100644 index ba55e6d97..000000000 --- a/letsencrypt-auto-source/pieces/check_permissions.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Verifies certbot-auto cannot be modified by unprivileged users. - -This script takes the path to certbot-auto as its only command line -argument. It then checks that the file can only be modified by uid/gid -< 1000 and if other users can modify the file, it prints a warning with -a suggestion on how to solve the problem. - -Permissions on symlinks in the absolute path of certbot-auto are ignored -and only the canonical path to certbot-auto is checked. There could be -permissions problems due to the symlinks that are unreported by this -script, however, issues like this were not caused by our documentation -and are ignored for the sake of simplicity. - -All warnings are printed to stdout rather than stderr so all stderr -output from this script can be suppressed to avoid printing messages if -this script fails for some reason. - -""" -from __future__ import print_function - -import os -import stat -import sys - - -FORUM_POST_URL = 'https://community.letsencrypt.org/t/certbot-auto-deployment-best-practices/91979/' - - -def has_safe_permissions(path): - """Returns True if the given path has secure permissions. - - The permissions are considered safe if the file is only writable by - uid/gid < 1000. - - The reason we allow more IDs than 0 is because on some systems such - as Debian, system users/groups other than uid/gid 0 are used for the - path we recommend in our instructions which is /usr/local/bin. 1000 - was chosen because on Debian 0-999 is reserved for system IDs[1] and - on RHEL either 0-499 or 0-999 is reserved depending on the - version[2][3]. Due to these differences across different OSes, this - detection isn't perfect so we only determine permissions are - insecure when we can be reasonably confident there is a problem - regardless of the underlying OS. - - [1] https://www.debian.org/doc/debian-policy/ch-opersys.html#uid-and-gid-classes - [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/ch-managing_users_and_groups - [3] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-managing_users_and_groups - - :param str path: filesystem path to check - :returns: True if the path has secure permissions, otherwise, False - :rtype: bool - - """ - # os.stat follows symlinks before obtaining information about a file. - stat_result = os.stat(path) - if stat_result.st_mode & stat.S_IWOTH: - return False - if stat_result.st_mode & stat.S_IWGRP and stat_result.st_gid >= 1000: - return False - if stat_result.st_mode & stat.S_IWUSR and stat_result.st_uid >= 1000: - return False - return True - - -def main(certbot_auto_path): - current_path = os.path.realpath(certbot_auto_path) - last_path = None - permissions_ok = True - # This loop makes use of the fact that os.path.dirname('/') == '/'. - while current_path != last_path and permissions_ok: - permissions_ok = has_safe_permissions(current_path) - last_path = current_path - current_path = os.path.dirname(current_path) - - if not permissions_ok: - print('{0} has insecure permissions!'.format(certbot_auto_path)) - print('To learn how to fix them, visit {0}'.format(FORUM_POST_URL)) - - -if __name__ == '__main__': - main(sys.argv[1]) diff --git a/letsencrypt-auto-source/pieces/create_venv.py b/letsencrypt-auto-source/pieces/create_venv.py deleted file mode 100755 index a618e228a..000000000 --- a/letsencrypt-auto-source/pieces/create_venv.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -import os -import shutil -import subprocess -import sys - - -def create_venv(venv_path, pyver, verbose): - if os.path.exists(venv_path): - shutil.rmtree(venv_path) - - stdout = sys.stdout if verbose == '1' else open(os.devnull, 'w') - - if int(pyver) <= 27: - # Use virtualenv binary - environ = os.environ.copy() - environ['VIRTUALENV_NO_DOWNLOAD'] = '1' - command = ['virtualenv', '--no-site-packages', '--python', sys.executable, venv_path] - subprocess.check_call(command, stdout=stdout, env=environ) - else: - # Use embedded venv module in Python 3 - command = [sys.executable, '-m', 'venv', venv_path] - subprocess.check_call(command, stdout=stdout) - - -if __name__ == '__main__': - create_venv(*sys.argv[1:]) diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt deleted file mode 100644 index f7a517e06..000000000 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ /dev/null @@ -1,264 +0,0 @@ -# This is the flattened list of packages certbot-auto installs. -# To generate this, do (with docker and package hashin installed): -# ``` -# letsencrypt-auto-source/rebuild_dependencies.py \ -# letsencrypt-auto-source/pieces/dependency-requirements.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r dependency-requirements.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.2.3 \ - --hash=sha256:edd17be986d5c1ba2e307150b8e5f5107aba125f3574dddd02c85d5cdcfd37dc -certifi==2020.4.5.1 \ - --hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \ - --hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 -cffi==1.14.0 \ - --hash=sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff \ - --hash=sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b \ - --hash=sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac \ - --hash=sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0 \ - --hash=sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384 \ - --hash=sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26 \ - --hash=sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 \ - --hash=sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b \ - --hash=sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e \ - --hash=sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd \ - --hash=sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2 \ - --hash=sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66 \ - --hash=sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc \ - --hash=sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8 \ - --hash=sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55 \ - --hash=sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4 \ - --hash=sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5 \ - --hash=sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d \ - --hash=sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78 \ - --hash=sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa \ - --hash=sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793 \ - --hash=sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f \ - --hash=sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a \ - --hash=sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f \ - --hash=sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30 \ - --hash=sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f \ - --hash=sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3 \ - --hash=sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c -chardet==3.0.4 \ - --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \ - --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==2.8 \ - --hash=sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c \ - --hash=sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595 \ - --hash=sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad \ - --hash=sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651 \ - --hash=sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2 \ - --hash=sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff \ - --hash=sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d \ - --hash=sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42 \ - --hash=sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d \ - --hash=sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e \ - --hash=sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912 \ - --hash=sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793 \ - --hash=sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13 \ - --hash=sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7 \ - --hash=sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0 \ - --hash=sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879 \ - --hash=sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f \ - --hash=sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9 \ - --hash=sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2 \ - --hash=sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf \ - --hash=sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8 -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -enum34==1.1.10; python_version < '3.4' \ - --hash=sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53 \ - --hash=sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328 \ - --hash=sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248 -funcsigs==1.0.2 \ - --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ - --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 -idna==2.9 \ - --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \ - --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa -ipaddress==1.0.23 \ - --hash=sha256:6e0f4a39e66cb5bb9a137b00276a2eff74f93b71dcbdad6f10ff7df9d3557fcc \ - --hash=sha256:b7f8e0369580bb4a24d5ba1d7cc29660a4a6987763faf1d8a8046830e020e7e2 -josepy==1.3.0 \ - --hash=sha256:c341ffa403399b18e9eae9012f804843045764d1390f9cb4648980a7569b1619 \ - --hash=sha256:e54882c64be12a2a76533f73d33cba9e331950fda9e2731e843490b774e7a01c -mock==1.3.0 \ - --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6 \ - --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb -parsedatetime==2.5 \ - --hash=sha256:3b835fc54e472c17ef447be37458b400e3fefdf14bb1ffdedb5d2c853acf4ba1 \ - --hash=sha256:d2e9ddb1e463de871d32088a3f3cea3dc8282b1b2800e081bd0ef86900451667 -pbr==5.4.5 \ - --hash=sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c \ - --hash=sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8 -pyOpenSSL==19.1.0 \ - --hash=sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504 \ - --hash=sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507 -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2020.1 \ - --hash=sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed \ - --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048 -requests==2.23.0 \ - --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \ - --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.25.9 \ - --hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \ - --hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115 -zope.component==4.6.1 \ - --hash=sha256:bfbe55d4a93e70a78b10edc3aad4de31bb8860919b7cbd8d66f717f7d7b279ac \ - --hash=sha256:d9c7c27673d787faff8a83797ce34d6ebcae26a370e25bddb465ac2182766aca -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.4 \ - --hash=sha256:69c27debad9bdacd9ce9b735dad382142281ac770c4a432b533d6d65c4614bcf \ - --hash=sha256:d8e97d165fd5a0997b45f5303ae11ea3338becfe68c401dd88ffd2113fe5cae7 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.1.0 \ - --hash=sha256:0103cba5ed09f27d2e3de7e48bb320338592e2fabc5ce1432cf33808eb2dfd8b \ - --hash=sha256:14415d6979356629f1c386c8c4249b4d0082f2ea7f75871ebad2e29584bd16c5 \ - --hash=sha256:1ae4693ccee94c6e0c88a4568fb3b34af8871c60f5ba30cf9f94977ed0e53ddd \ - --hash=sha256:1b87ed2dc05cb835138f6a6e3595593fea3564d712cb2eb2de963a41fd35758c \ - --hash=sha256:269b27f60bcf45438e8683269f8ecd1235fa13e5411de93dae3b9ee4fe7f7bc7 \ - --hash=sha256:27d287e61639d692563d9dab76bafe071fbeb26818dd6a32a0022f3f7ca884b5 \ - --hash=sha256:39106649c3082972106f930766ae23d1464a73b7d30b3698c986f74bf1256a34 \ - --hash=sha256:40e4c42bd27ed3c11b2c983fecfb03356fae1209de10686d03c02c8696a1d90e \ - --hash=sha256:461d4339b3b8f3335d7e2c90ce335eb275488c587b61aca4b305196dde2ff086 \ - --hash=sha256:4f98f70328bc788c86a6a1a8a14b0ea979f81ae6015dd6c72978f1feff70ecda \ - --hash=sha256:558a20a0845d1a5dc6ff87cd0f63d7dac982d7c3be05d2ffb6322a87c17fa286 \ - --hash=sha256:562dccd37acec149458c1791da459f130c6cf8902c94c93b8d47c6337b9fb826 \ - --hash=sha256:5e86c66a6dea8ab6152e83b0facc856dc4d435fe0f872f01d66ce0a2131b7f1d \ - --hash=sha256:60a207efcd8c11d6bbeb7862e33418fba4e4ad79846d88d160d7231fcb42a5ee \ - --hash=sha256:645a7092b77fdbc3f68d3cc98f9d3e71510e419f54019d6e282328c0dd140dcd \ - --hash=sha256:6874367586c020705a44eecdad5d6b587c64b892e34305bb6ed87c9bbe22a5e9 \ - --hash=sha256:74bf0a4f9091131de09286f9a605db449840e313753949fe07c8d0fe7659ad1e \ - --hash=sha256:7b726194f938791a6691c7592c8b9e805fc6d1b9632a833b9c0640828cd49cbc \ - --hash=sha256:8149ded7f90154fdc1a40e0c8975df58041a6f693b8f7edcd9348484e9dc17fe \ - --hash=sha256:8cccf7057c7d19064a9e27660f5aec4e5c4001ffcf653a47531bde19b5aa2a8a \ - --hash=sha256:911714b08b63d155f9c948da2b5534b223a1a4fc50bb67139ab68b277c938578 \ - --hash=sha256:a5f8f85986197d1dd6444763c4a15c991bfed86d835a1f6f7d476f7198d5f56a \ - --hash=sha256:a744132d0abaa854d1aad50ba9bc64e79c6f835b3e92521db4235a1991176813 \ - --hash=sha256:af2c14efc0bb0e91af63d00080ccc067866fb8cbbaca2b0438ab4105f5e0f08d \ - --hash=sha256:b054eb0a8aa712c8e9030065a59b5e6a5cf0746ecdb5f087cca5ec7685690c19 \ - --hash=sha256:b0becb75418f8a130e9d465e718316cd17c7a8acce6fe8fe07adc72762bee425 \ - --hash=sha256:b1d2ed1cbda2ae107283befd9284e650d840f8f7568cb9060b5466d25dc48975 \ - --hash=sha256:ba4261c8ad00b49d48bbb3b5af388bb7576edfc0ca50a49c11dcb77caa1d897e \ - --hash=sha256:d1fe9d7d09bb07228650903d6a9dc48ea649e3b8c69b1d263419cc722b3938e8 \ - --hash=sha256:d7804f6a71fc2dda888ef2de266727ec2f3915373d5a785ed4ddc603bbc91e08 \ - --hash=sha256:da2844fba024dd58eaa712561da47dcd1e7ad544a257482392472eae1c86d5e5 \ - --hash=sha256:dcefc97d1daf8d55199420e9162ab584ed0893a109f45e438b9794ced44c9fd0 \ - --hash=sha256:dd98c436a1fc56f48c70882cc243df89ad036210d871c7427dc164b31500dc11 \ - --hash=sha256:e74671e43ed4569fbd7989e5eecc7d06dc134b571872ab1d5a88f4a123814e9f \ - --hash=sha256:eb9b92f456ff3ec746cd4935b73c1117538d6124b8617bc0fe6fda0b3816e345 \ - --hash=sha256:ebb4e637a1fb861c34e48a00d03cffa9234f42bef923aec44e5625ffb9a8e8f9 \ - --hash=sha256:ef739fe89e7f43fb6494a43b1878a36273e5924869ba1d866f752c5812ae8d58 \ - --hash=sha256:f40db0e02a8157d2b90857c24d89b6310f9b6c3642369852cdc3b5ac49b92afc \ - --hash=sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6 \ - --hash=sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py deleted file mode 100644 index 1515fe353..000000000 --- a/letsencrypt-auto-source/pieces/fetch.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Do downloading and JSON parsing without additional dependencies. :: - - # Print latest released version of LE to stdout: - python fetch.py --latest-version - - # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm - # in, and make sure its signature verifies: - python fetch.py --le-auto-script v1.2.3 - -On failure, return non-zero. - -""" - -from __future__ import print_function, unicode_literals - -from distutils.version import LooseVersion -from json import loads -from os import devnull, environ -from os.path import dirname, join -import re -import ssl -from subprocess import check_call, CalledProcessError -from sys import argv, exit -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler - from urllib2 import HTTPError, URLError -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler - from urllib.error import HTTPError, URLError - -PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq -OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 -xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp -9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij -n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH -cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ -CQIDAQAB ------END PUBLIC KEY----- -""") - -class ExpectedError(Exception): - """A novice-readable exception that also carries the original exception for - debugging""" - - -class HttpsGetter(object): - def __init__(self): - """Build an HTTPS opener.""" - # Based on pip 1.4.1's URLOpener - # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set. - if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'): - self._opener = build_opener(HTTPSHandler(context=cert_none_context())) - else: - self._opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in self._opener.handlers: - if isinstance(handler, HTTPHandler): - self._opener.handlers.remove(handler) - - def get(self, url): - """Return the document contents pointed to by an HTTPS URL. - - If something goes wrong (404, timeout, etc.), raise ExpectedError. - - """ - try: - # socket module docs say default timeout is None: that is, no - # timeout - return self._opener.open(url, timeout=30).read() - except (HTTPError, IOError) as exc: - raise ExpectedError("Couldn't download %s." % url, exc) - - -def write(contents, dir, filename): - """Write something to a file in a certain directory.""" - with open(join(dir, filename), 'wb') as file: - file.write(contents) - - -def latest_stable_version(get): - """Return the latest stable release of letsencrypt.""" - metadata = loads(get( - environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8')) - # metadata['info']['version'] actually returns the latest of any kind of - # release release, contrary to https://wiki.python.org/moin/PyPIJSON. - # The regex is a sufficient regex for picking out prereleases for most - # packages, LE included. - return str(max(LooseVersion(r) for r - in metadata['releases'].keys() - if re.match('^[0-9.]+$', r))) - - -def verified_new_le_auto(get, tag, temp_dir): - """Return the path to a verified, up-to-date letsencrypt-auto script. - - If the download's signature does not verify or something else goes wrong - with the verification process, raise ExpectedError. - - """ - le_auto_dir = environ.get( - 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/certbot/certbot/%s/' - 'letsencrypt-auto-source/') % tag - write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') - write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') - write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem') - try: - with open(devnull, 'w') as dev_null: - check_call(['openssl', 'dgst', '-sha256', '-verify', - join(temp_dir, 'public_key.pem'), - '-signature', - join(temp_dir, 'letsencrypt-auto.sig'), - join(temp_dir, 'letsencrypt-auto')], - stdout=dev_null, - stderr=dev_null) - except CalledProcessError as exc: - raise ExpectedError("Couldn't verify signature of downloaded " - "certbot-auto.", exc) - - -def cert_none_context(): - """Create a SSLContext object to not check hostname.""" - # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this. - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.verify_mode = ssl.CERT_NONE - return context - - -def main(): - get = HttpsGetter().get - flag = argv[1] - try: - if flag == '--latest-version': - print(latest_stable_version(get)) - elif flag == '--le-auto-script': - tag = argv[2] - verified_new_le_auto(get, tag, dirname(argv[0])) - except ExpectedError as exc: - print(exc.args[0], exc.args[1]) - return 1 - else: - return 0 - - -if __name__ == '__main__': - exit(main()) diff --git a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt deleted file mode 100644 index 8e745c9cd..000000000 --- a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Contains the requirements for the letsencrypt package. -# -# Since the letsencrypt package depends on certbot and using pip with hashes -# requires that all installed packages have hashes listed, this allows -# dependency-requirements.txt to be used without requiring a hash for a -# (potentially unreleased) Certbot package. - -letsencrypt==0.7.0 \ - --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ - --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py deleted file mode 100755 index 7610c2686..000000000 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python -"""A small script that can act as a trust root for installing pip >=8 -Embed this in your project, and your VCS checkout is all you have to trust. In -a post-peep era, this lets you claw your way to a hash-checking version of pip, -with which you can install the rest of your dependencies safely. All it assumes -is Python 2.6 or better and *some* version of pip already installed. If -anything goes wrong, it will exit with a non-zero status code. -""" -# This is here so embedded copies are MIT-compliant: -# Copyright (c) 2016 Erik Rose -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -from __future__ import print_function -from distutils.version import StrictVersion -from hashlib import sha256 -from os import environ -from os.path import join -from shutil import rmtree -try: - from subprocess import check_output -except ImportError: - from subprocess import CalledProcessError, PIPE, Popen - - def check_output(*popenargs, **kwargs): - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be ' - 'overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return output -import sys -from tempfile import mkdtemp -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse # 3.4 - - -__version__ = 1, 5, 1 -PIP_VERSION = '9.0.1' -DEFAULT_INDEX_BASE = 'https://pypi.python.org' - - -# wheel has a conditional dependency on argparse: -maybe_argparse = ( - [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/' - 'argparse-1.4.0.tar.gz', - '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] - if sys.version_info < (2, 7, 0) else []) - - -# Be careful when updating the pinned versions here, in particular for pip. -# Indeed starting from 10.0, pip will build dependencies in isolation if the -# related projects are compliant with PEP 517. This is not something we want -# as of now, so the isolation build will need to be disabled wherever -# pipstrap is used (see https://github.com/certbot/certbot/issues/8256). -PACKAGES = maybe_argparse + [ - # Pip has no dependencies, as it vendors everything: - ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/' - 'pip-{0}.tar.gz'.format(PIP_VERSION), - '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'), - # This version of setuptools has only optional dependencies: - ('37/1b/b25507861991beeade31473868463dad0e58b1978c209de27384ae541b0b/' - 'setuptools-40.6.3.zip', - '3b474dad69c49f0d2d86696b68105f3a6f195f7ab655af12ef9a9c326d2b08f8'), - ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/' - 'wheel-0.29.0.tar.gz', - '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') -] - - -class HashError(Exception): - def __str__(self): - url, path, actual, expected = self.args - return ('{url} did not match the expected hash {expected}. Instead, ' - 'it was {actual}. The file (left at {path}) may have been ' - 'tampered with.'.format(**locals())) - - -def hashed_download(url, temp, digest): - """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``, - and return its path.""" - # Based on pip 1.4.1's URLOpener but with cert verification removed. Python - # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert - # authenticity has only privacy (not arbitrary code execution) - # implications, since we're checking hashes. - def opener(using_https=True): - opener = build_opener(HTTPSHandler()) - if using_https: - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) - return opener - - def read_chunks(response, chunk_size): - while True: - chunk = response.read(chunk_size) - if not chunk: - break - yield chunk - - parsed_url = urlparse(url) - response = opener(using_https=parsed_url.scheme == 'https').open(url) - path = join(temp, parsed_url.path.split('/')[-1]) - actual_hash = sha256() - with open(path, 'wb') as file: - for chunk in read_chunks(response, 4096): - file.write(chunk) - actual_hash.update(chunk) - - actual_digest = actual_hash.hexdigest() - if actual_digest != digest: - raise HashError(url, path, actual_digest, digest) - return path - - -def get_index_base(): - """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the - end if it's there; that is likely to give us the right dir. - """ - env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') - if env_var: - SIMPLE = '/simple' - if env_var.endswith(SIMPLE): - return env_var[:-len(SIMPLE)] - else: - return env_var - else: - return DEFAULT_INDEX_BASE - - -def main(): - python = sys.executable or 'python' - pip_version = StrictVersion(check_output([python, '-m', 'pip', '--version']) - .decode('utf-8').split()[1]) - has_pip_cache = pip_version >= StrictVersion('6.0') - index_base = get_index_base() - temp = mkdtemp(prefix='pipstrap-') - try: - downloads = [hashed_download(index_base + '/packages/' + path, - temp, - digest) - for path, digest in PACKAGES] - # Calling pip as a module is the preferred way to avoid problems about pip self-upgrade. - command = [python, '-m', 'pip', 'install', '--no-index', '--no-deps', '-U'] - # Disable cache since it is not used and it otherwise sometimes throws permission warnings: - command.extend(['--no-cache-dir'] if has_pip_cache else []) - command.extend(downloads) - check_output(command) - except HashError as exc: - print(exc) - except Exception: - rmtree(temp) - raise - else: - rmtree(temp) - return 0 - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/letsencrypt-auto-source/tests/signing.key b/letsencrypt-auto-source/tests/signing.key deleted file mode 100644 index b9964d00c..000000000 --- a/letsencrypt-auto-source/tests/signing.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAsMoSzLYQ7E1sdSOkwelgtzKIh2qi3bpXuYtcfFC0XrvWig07 -1NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7GhFW0VdbxL6JdGzS2ShNWkX9hE9z+ -j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTTuUtJmmGcuk3a9Aq/sCT6DdfmTSdP -5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVglLsIVPBuy9IcgHidUQ96hJnoPsDCW -sHwX62495QKEarauyKQrJzFes0EY95orDM47Z5o/NDiQB11m91yNB0MmPYY9QSbn -OA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68iQIDAQABAoIBAQCJE3W2Mqk2f+XL -geKa1BjAkzcXQJCduYGRhUQlw/HGzoBPtGki56Tf53MeHTAkIGfIq3CAr1zRhiNv -8SQzvrLQIx/buvhxhcQJdzqsfwgNcqXT3/OliF34P3LMx8GUfPy/6xq2Qdv4fvwA -nLJH8wyDTKP6RxtdvUY7GSZ+Ln2QQv/3Nco7tax4GHNGom8iSgeH/YKTDnvitdqh -a0fr930QzU39TfOftLmasdmKUOIg8G2wr4Sy6Kn060+OUoQr1fZF5mnLvvQeILCK -uav91JkIeMLggzk+t88IJUFWdOoxv5hWTnNzHyt+/GYfovyRz2fKQMwzdh1F8iM5 -+867rEb9AoGBANn1ncemJBedDshStdCBUH0+2ExPrawveaXOZKnx8/VGFXNi0hAf -KzkntMWd5g5kB077FtKO9CYTBvK4pZBWIFLcJEqAz88JeXME6dfUbRucDr72ko+l -rcLHXj7F0IDVzj/9CphMGAhC9J/4YW9SPcSbMw6dQ6xOk73f1Vowve0DAoGBAM+k -/F+hVqCS3f22Bg9KuDtx+zCydaZxC842DgIkV1SO2iFhNHjnpQ5EIR0WrSYeV2n+ -rD7kVs5OH1HvnGScHaQKtAVqZClSwF14jzE+Aj8XDwxiHLSOhJgKlzfVX7h1ymMh -7fsslDl6xNGQ+40gubhkCLT5qABFKy1mrZ8b+3yDAoGAGLGUI6d2FVrM7vM3+Bx+ -gwIYvWSVl5l1XcypaPupmRNMoNsEU6FEY2BVQcJm6yB4F4GpD0f0709ejSdQUq7/ -UIPydKJtaNZ49QgMelBt4B/pJ8eFyVKLAjNWQSRmQAJ5MJS5m5Gbc2wqjOk2GMen -idvPiAtXPHFWmb9/S42UJwMCgYEAjymAe2qgcGtyNNfIC8kHhqzKdEPGi/ALJKzu -MZnewEURrcv4QpfrnA9rCUQ2Mz7eJA1bsqz6EJmaTIK4wEFGynA6uDUnQ7pzOL7D -cz7+i4MZc/89LVvJnY5Hvk4WBfboiDq/etq8g3jatGaSmTYD9la6DhTHORB3eYD+ -meHQHYMCgYEA18y9hnx2k4vNeBei4YXF4pAvKdwKLQD+CcP9ljb3VT+kXktjRA1C -aWj3HhMwvcxtttfkQzEnwwGRAkTEtNewJ8KFxhmc9nYElZTNZ+SuHD5Dkv8xqoj8 -NvG8rU1eiEyPwE2wQxpM5JLqbo7IWtR0dmptjKoF1gRxn6Wh4TwEiHA= ------END RSA PRIVATE KEY----- diff --git a/tests/letstest/README.md b/tests/letstest/README.md index 76db57153..aea4e2e13 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -59,8 +59,7 @@ The tests take quite a while to run. Example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed to them at runtime via environment variables. test_apache2.sh is a useful reference. -Note that the 
test_letsencrypt_auto_*
scripts pull code from PyPI using the letsencrypt-auto script, -__not__ the local python code. test_apache2 runs the dev venv and does local tests. +test_apache2 runs the dev venv and does local tests. See: - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 8c0528990..360bb979e 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -22,7 +22,7 @@ Usage: >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ --query 'KeyMaterial' --output text > MyKeyPair.pem then: ->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_leauto_upgrades.sh +>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html @@ -55,7 +55,7 @@ parser.add_argument('key_file', parser.add_argument('aws_profile', help='profile for AWS (i.e. as in ~/.aws/certificates)') parser.add_argument('test_script', - default='test_letsencrypt_auto_certonly_standalone.sh', + default='test_sdists.sh', help='path of bash script in to deploy and run') parser.add_argument('--repo', default='https://github.com/letsencrypt/letsencrypt.git', diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh deleted file mode 100755 index 3964364e1..000000000 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -xe -set -o pipefail - -# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME -# are dynamically set at execution - -cd letsencrypt - -if ! command -v git ; then - if [ "$OS_TYPE" = "ubuntu" ] ; then - sudo apt-get update - fi - if ! ( sudo apt-get install -y git || sudo yum install -y git-all || sudo yum install -y git || sudo dnf install -y git ) ; then - echo git installation failed! - exit 1 - fi -fi -# If we're on a RHEL 6 based system, we can be confident Python is already -# installed because the package manager is written in Python. -if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # 0.20.0 is the latest version of letsencrypt-auto that doesn't install - # Python 3 on RHEL 6. - INITIAL_VERSION="0.20.0" - RUN_RHEL6_TESTS=1 -else - # 0.39.0 is the oldest version of letsencrypt-auto that works on CentOS 8. - INITIAL_VERSION="0.39.0" -fi - -git checkout -f "v$INITIAL_VERSION" letsencrypt-auto -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo initial installation appeared to fail - exit 1 -fi - -if command -v python; then - PYTHON_NAME="python" -else - PYTHON_NAME="python3" -fi - -# Now that python and openssl have been installed, we can set up a fake server -# to provide a new version of letsencrypt-auto. First, we start the server and -# directory to be served. -MY_TEMP_DIR=$(mktemp -d) -PORT_FILE="$MY_TEMP_DIR/port" -LOG_FILE="$MY_TEMP_DIR/log" -SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py) -cd "$MY_TEMP_DIR" -# We set PYTHONUNBUFFERED to disable buffering of output to LOG_FILE -PYTHONUNBUFFERED=1 "$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE 2> "$LOG_FILE" & -SERVER_PID=$! -trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT -cd ~- - -# Then, we set up the files to be served. -FAKE_VERSION_NUM="99.99.99" -echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json" -LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM" -NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -mkdir "$LE_AUTO_SOURCE_DIR" -cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto" -SIGNING_KEY="letsencrypt-auto-source/tests/signing.key" -openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH" - -# Next, we wait for the server to start and get the port number. -sleep 5s -SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE") - -# Finally, we set the necessary certbot-auto environment variables. -export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/" -export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json" -export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg -tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G -hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT -uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl -LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47 -Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68 -iQIDAQAB ------END PUBLIC KEY----- -" - -if [ "$RUN_RHEL6_TESTS" = 1 ]; then - if command -v python3; then - echo "Didn't expect Python 3 to be installed!" - exit 1 - fi - cp letsencrypt-auto cb-auto - if ! ./cb-auto -v --debug --version 2>&1 | grep "$INITIAL_VERSION" ; then - echo "Certbot shouldn't have updated to a new version!" - exit 1 - fi - # Create a 2nd venv at the old path to ensure we properly handle the (unlikely) case of two separate virtual environments below. - HOME=${HOME:-~root} - XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} - OLD_VENV_PATH="$XDG_DATA_HOME/letsencrypt" - export VENV_PATH="$OLD_VENV_PATH" - if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | tail -n1 | grep "^certbot $INITIAL_VERSION$" ; then - echo second installation appeared to fail - exit 1 - fi - unset VENV_PATH -fi - -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi - -# Since certbot-auto is deprecated, we expect it to leave existing Certbot -# installations unmodified so we check for the same version that was initially -# installed below. -EXPECTED_VERSION="$INITIAL_VERSION" - -if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi - -if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then - echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ - exit 1 -fi - -# Now let's test if letsencrypt-auto still tries to upgrade to a new version. -# Regardless of the OS, versions of the script with development version numbers -# ending in .dev0 will not upgrade. See -# https://github.com/certbot/certbot/blob/bdfb9f19c4086a60ef010d2431768850c26d838a/certbot-auto#L1947-L1948. -# In order to test the process of different OSes setting NO_SELF_UPGRADE as -# part of the script's deprecation, we make use of the fact that -# letsencrypt-auto should still attempt to fetch the version number from PyPI -# even if it has a development version number unless NO_SELF_UPGRADE is set in -# which case all of that logic should be skipped. -# -# First we make a copy of the current server logs. -PREVIOUS_LOG_FILE="$MY_TEMP_DIR/previous-log" -cp "$LOG_FILE" "$PREVIOUS_LOG_FILE" - -# Next we run letsencrypt-auto and make sure there were no problems checking -# for updates, the Certbot install still works, the version number is what -# we expect, and it prints a message about not receiving updates. -if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then - echo "Had problems checking for updates!" - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then - echo unexpected certbot version found - exit 1 -fi -if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive updates" ; then - echo script did not print warning about not receiving updates! - exit 1 -fi - -# Finally, we check if our local server received more requests. -if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then - echo our local server received unexpected requests - exit 1 -fi diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh deleted file mode 100755 index 9573ab690..000000000 --- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -x -set -eo pipefail - -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME are dynamically set at execution - -# with curl, instance metadata available from EC2 metadata service: -#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) - -cd letsencrypt -LE_AUTO_DIR="/usr/local/bin" -LE_AUTO_PATH="$LE_AUTO_DIR/letsencrypt-auto" -sudo cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_PATH" -sudo chown root "$LE_AUTO_PATH" -sudo chmod 0755 "$LE_AUTO_PATH" -export PATH="$LE_AUTO_DIR:$PATH" - -# Since certbot-auto is deprecated, we expect certbot-auto to error and -# refuse to install Certbot. -set +o pipefail -if ! letsencrypt-auto --debug --version | grep "Certbot cannot be installed."; then - echo "letsencrypt-auto didn't report being uninstallable." - exit 1 -fi -if [ ${PIPESTATUS[0]} != 1 ]; then - echo "letsencrypt-auto didn't exit with status 1 as expected" - exit 1 -fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index aa385ad2f..b0d53fb67 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -41,7 +41,7 @@ for pkg_dir in acme certbot $PLUGINS; do cd - done -VERSION=$(python letsencrypt-auto-source/version.py) +VERSION=$(python tests/letstest/scripts/version.py) # test sdists cd $TEMP_DIR for pkg in acme certbot $PLUGINS; do diff --git a/letsencrypt-auto-source/version.py b/tests/letstest/scripts/version.py similarity index 75% rename from letsencrypt-auto-source/version.py rename to tests/letstest/scripts/version.py index d70ffefac..dce2eaec2 100755 --- a/letsencrypt-auto-source/version.py +++ b/tests/letstest/scripts/version.py @@ -1,8 +1,7 @@ #!/usr/bin/env python """Get the current Certbot version number. -Provides simple utilities for determining the Certbot version number and -building letsencrypt-auto. +Provides a simple utility for determining the Certbot version number """ from __future__ import print_function @@ -10,10 +9,10 @@ from os.path import abspath, dirname, join import re -def certbot_version(build_script_dir): +def certbot_version(caller_dir): """Return the version number stamped in certbot/__init__.py.""" return re.search('''^__version__ = ['"](.+)['"].*''', - file_contents(join(dirname(build_script_dir), + file_contents(join(dirname(dirname(dirname(caller_dir))), 'certbot', 'certbot', '__init__.py')), diff --git a/tox.ini b/tox.ini index 90aecc1b3..f7db07b52 100644 --- a/tox.ini +++ b/tox.ini @@ -290,20 +290,6 @@ deps = {[testenv:test-farm-tests-base]deps} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} -[testenv:test-farm-leauto-upgrades] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - -[testenv:test-farm-certonly-standalone] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} - [testenv:test-farm-sdists] changedir = {[testenv:test-farm-tests-base]changedir} commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} From 2cf177586497724f0eebbbe85cfcb9432f6963cf Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Thu, 29 Apr 2021 02:45:08 +0200 Subject: [PATCH 79/99] Update assertTrue/False to Python 3 precise asserts (#8792) * Update assertTrue/False to Python 3 precise asserts * Fix test failures * Fix test failures * More replacements * Update to Python 3 asserts in acme-module * Fix Windows test failure * Fix failures * Fix test failure * More replacements * Don't include the semgrep rules * Fix test failure --- acme/tests/challenges_test.py | 4 +- acme/tests/client_test.py | 18 ++-- acme/tests/crypto_util_test.py | 6 +- acme/tests/errors_test.py | 4 +- acme/tests/jws_test.py | 4 +- acme/tests/messages_test.py | 20 ++-- certbot/tests/account_test.py | 2 +- certbot/tests/auth_handler_test.py | 16 +-- certbot/tests/cert_manager_test.py | 47 +++++---- certbot/tests/cli_test.py | 128 +++++++++++------------ certbot/tests/client_test.py | 28 ++--- certbot/tests/compat/filesystem_test.py | 40 +++---- certbot/tests/compat/misc_test.py | 2 +- certbot/tests/crypto_util_test.py | 14 +-- certbot/tests/display/completer_test.py | 8 +- certbot/tests/display/ops_test.py | 26 +++-- certbot/tests/display/util_test.py | 21 ++-- certbot/tests/eff_test.py | 18 ++-- certbot/tests/error_handler_test.py | 2 +- certbot/tests/helpful_test.py | 12 +-- certbot/tests/hook_test.py | 20 ++-- certbot/tests/lock_test.py | 14 +-- certbot/tests/log_test.py | 27 +++-- certbot/tests/main_test.py | 115 ++++++++++---------- certbot/tests/ocsp_test.py | 48 ++++----- certbot/tests/plugins/common_test.py | 6 +- certbot/tests/plugins/disco_test.py | 80 +++++++------- certbot/tests/plugins/dns_common_test.py | 12 +-- certbot/tests/plugins/manual_test.py | 9 +- certbot/tests/plugins/null_test.py | 2 +- certbot/tests/plugins/selection_test.py | 12 +-- certbot/tests/plugins/standalone_test.py | 11 +- certbot/tests/plugins/storage_test.py | 10 +- certbot/tests/plugins/util_test.py | 8 +- certbot/tests/plugins/webroot_test.py | 8 +- certbot/tests/renewal_test.py | 6 +- certbot/tests/renewupdater_test.py | 10 +- certbot/tests/reporter_test.py | 20 ++-- certbot/tests/reverter_test.py | 6 +- certbot/tests/storage_test.py | 34 +++--- certbot/tests/util_test.py | 18 ++-- 41 files changed, 444 insertions(+), 452 deletions(-) diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index cc604b0de..71bf61976 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -292,7 +292,7 @@ class TLSALPN01ResponseTest(unittest.TestCase): def test_gen_verify_cert_gen_key(self): cert, key = self.response.gen_cert(self.domain) - self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertIsInstance(key, OpenSSL.crypto.PKey) self.assertTrue(self.response.verify_cert(self.domain, cert)) def test_verify_bad_cert(self): @@ -431,7 +431,7 @@ class DNSTest(unittest.TestCase): mock_gen.return_value = mock.sentinel.validation response = self.msg.gen_response(KEY) from acme.challenges import DNSResponse - self.assertTrue(isinstance(response, DNSResponse)) + self.assertIsInstance(response, DNSResponse) self.assertEqual(response.validation, mock.sentinel.validation) def test_validation_domain_name(self): diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index 89e66c6d6..35cc0ba25 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -604,8 +604,8 @@ class ClientTest(ClientTestBase): # make sure that max_attempts is per-authorization, rather # than global max_attempts=max(len(authzrs[0].retries), len(authzrs[1].retries))) - self.assertTrue(cert[0] is csr) - self.assertTrue(cert[1] is updated_authzrs) + self.assertIs(cert[0], csr) + self.assertIs(cert[1], updated_authzrs) self.assertEqual(updated_authzrs[0].uri, 'a...') self.assertEqual(updated_authzrs[1].uri, 'b.') self.assertEqual(updated_authzrs[0].times, [ @@ -641,7 +641,7 @@ class ClientTest(ClientTestBase): authzr = self.client.deactivate_authorization(self.authzr) self.assertEqual(authzb, authzr.body) self.assertEqual(self.client.net.post.call_count, 1) - self.assertTrue(self.authzr.uri in self.net.post.call_args_list[0][0]) + self.assertIn(self.authzr.uri, self.net.post.call_args_list[0][0]) def test_check_cert(self): self.response.headers['Location'] = self.certr.uri @@ -700,7 +700,7 @@ class ClientTest(ClientTestBase): def test_revocation_payload(self): obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn) - self.assertTrue('reason' in obj.to_partial_json().keys()) + self.assertIn('reason', obj.to_partial_json().keys()) self.assertEqual(self.rsn, obj.to_partial_json()['reason']) def test_revoke_bad_status_raises_error(self): @@ -877,9 +877,9 @@ class ClientV2Test(ClientTestBase): self.response.headers['Location'] = self.regr.uri self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.update_registration(self.regr)) - self.assertNotEqual(self.client.net.account, None) + self.assertIsNotNone(self.client.net.account) self.assertEqual(self.client.net.post.call_count, 2) - self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0]) + self.assertIn(DIRECTORY_V2.newAccount, self.net.post.call_args_list[0][0]) self.response.json.return_value = self.regr.body.update( contact=()).to_json() @@ -943,7 +943,7 @@ class ClientNetworkTest(unittest.TestCase): self.response.links = {} def test_init(self): - self.assertTrue(self.net.verify_ssl is self.verify_ssl) + self.assertIs(self.net.verify_ssl, self.verify_ssl) def test_wrap_in_jws(self): # pylint: disable=protected-access @@ -1185,7 +1185,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): def send_request(*args, **kwargs): # pylint: disable=unused-argument,missing-docstring - self.assertFalse("new_nonce_url" in kwargs) + self.assertNotIn("new_nonce_url", kwargs) method = args[0] uri = args[1] if method == 'HEAD' and uri != "new_nonce_uri": @@ -1330,7 +1330,7 @@ class ClientNetworkSourceAddressBindingTest(unittest.TestCase): from acme.client import ClientNetwork net = ClientNetwork(key=None, alg=None, source_address=self.source_address) for adapter in net.session.adapters.values(): - self.assertTrue(self.source_address in adapter.source_address) + self.assertIn(self.source_address, adapter.source_address) def test_behavior_assumption(self): """This is a test that guardrails the HTTPAdapter behavior so that if the default for diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 8c1f905d2..cc81a2f1f 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -191,7 +191,7 @@ class RandomSnTest(unittest.TestCase): for _ in range(self.cert_count): cert = gen_ss_cert(self.key, ['dummy'], force_san=True) self.serial_num.append(cert.get_serial_number()) - self.assertTrue(len(set(self.serial_num)) > 1) + self.assertGreater(len(set(self.serial_num)), 1) class MakeCSRTest(unittest.TestCase): """Test for standalone functions.""" @@ -206,8 +206,8 @@ class MakeCSRTest(unittest.TestCase): def test_make_csr(self): csr_pem = self._call_with_key(["a.example", "b.example"]) - self.assertTrue(b'--BEGIN CERTIFICATE REQUEST--' in csr_pem) - self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) + self.assertIn(b'--BEGIN CERTIFICATE REQUEST--', csr_pem) + self.assertIn(b'--END CERTIFICATE REQUEST--', csr_pem) csr = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, csr_pem) # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 11c57059c..f325b284e 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -24,8 +24,8 @@ class MissingNonceTest(unittest.TestCase): self.error = MissingNonce(self.response) def test_str(self): - self.assertTrue("FOO" in str(self.error)) - self.assertTrue("{}" in str(self.error)) + self.assertIn("FOO", str(self.error)) + self.assertIn("{}", str(self.error)) class PollErrorTest(unittest.TestCase): diff --git a/acme/tests/jws_test.py b/acme/tests/jws_test.py index 2e6ad72dd..0787fb340 100644 --- a/acme/tests/jws_test.py +++ b/acme/tests/jws_test.py @@ -48,7 +48,7 @@ class JWSTest(unittest.TestCase): self.assertEqual(jws.signature.combined.nonce, self.nonce) self.assertEqual(jws.signature.combined.url, self.url) self.assertEqual(jws.signature.combined.kid, self.kid) - self.assertEqual(jws.signature.combined.jwk, None) + self.assertIsNone(jws.signature.combined.jwk) # TODO: check that nonce is in protected header self.assertEqual(jws, JWS.from_json(jws.to_json())) @@ -58,7 +58,7 @@ class JWSTest(unittest.TestCase): jws = JWS.sign(payload=b'foo', key=self.privkey, alg=jose.RS256, nonce=self.nonce, url=self.url) - self.assertEqual(jws.signature.combined.kid, None) + self.assertIsNone(jws.signature.combined.kid) self.assertEqual(jws.signature.combined.jwk, self.pubkey) diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index 99a3a9ce4..3f0f29215 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -41,13 +41,13 @@ class ErrorTest(unittest.TestCase): def test_description(self): self.assertEqual('The request message was malformed', self.error.description) - self.assertTrue(self.error_custom.description is None) + self.assertIsNone(self.error_custom.description) def test_code(self): from acme.messages import Error self.assertEqual('malformed', self.error.code) - self.assertEqual(None, self.error_custom.code) - self.assertEqual(None, Error().code) + self.assertIsNone(self.error_custom.code) + self.assertIsNone(Error().code) def test_is_acme_error(self): from acme.messages import is_acme_error, Error @@ -260,10 +260,10 @@ class RegistrationTest(unittest.TestCase): self.assertEqual(empty_new_reg.contact, ()) self.assertEqual(new_reg_with_contact.contact, ()) - self.assertTrue('contact' not in empty_new_reg.to_partial_json()) - self.assertTrue('contact' not in empty_new_reg.fields_to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.to_partial_json()) - self.assertTrue('contact' in new_reg_with_contact.fields_to_partial_json()) + self.assertNotIn('contact', empty_new_reg.to_partial_json()) + self.assertNotIn('contact', empty_new_reg.fields_to_partial_json()) + self.assertIn('contact', new_reg_with_contact.to_partial_json()) + self.assertIn('contact', new_reg_with_contact.fields_to_partial_json()) class UpdateRegistrationTest(unittest.TestCase): @@ -406,7 +406,7 @@ class AuthorizationResourceTest(unittest.TestCase): authzr = AuthorizationResource( uri=mock.sentinel.uri, body=mock.sentinel.body) - self.assertTrue(isinstance(authzr, jose.JSONDeSerializable)) + self.assertIsInstance(authzr, jose.JSONDeSerializable) class CertificateRequestTest(unittest.TestCase): @@ -417,7 +417,7 @@ class CertificateRequestTest(unittest.TestCase): self.req = CertificateRequest(csr=CSR) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.req, jose.JSONDeSerializable)) + self.assertIsInstance(self.req, jose.JSONDeSerializable) from acme.messages import CertificateRequest self.assertEqual( self.req, CertificateRequest.from_json(self.req.to_json())) @@ -433,7 +433,7 @@ class CertificateResourceTest(unittest.TestCase): cert_chain_uri=mock.sentinel.cert_chain_uri) def test_json_de_serializable(self): - self.assertTrue(isinstance(self.certr, jose.JSONDeSerializable)) + self.assertIsInstance(self.certr, jose.JSONDeSerializable) from acme.messages import CertificateResource self.assertEqual( self.certr, CertificateResource.from_json(self.certr.to_json())) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7c8f52273..e034c5f32 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -150,7 +150,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase): path = os.path.join(self.config.accounts_dir, self.acc.id, "regr.json") with open(path, "r") as f: regr = json.load(f) - self.assertTrue("new_authzr_uri" in regr) + self.assertIn("new_authzr_uri", regr) def test_update_regr(self): self.storage.update_regr(self.acc, self.mock_client) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 8eb5d7702..1f798c2d8 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -106,9 +106,9 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(mock_time.sleep.call_count, 2) # Retry-After header is 30 seconds, but at the time sleep is invoked, several # instructions are executed, and next pool is in less than 30 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] <= 30) + self.assertLessEqual(mock_time.sleep.call_args_list[1][0][0], 30) # However, assert that we did not took the default value of 3 seconds. - self.assertTrue(mock_time.sleep.call_args_list[1][0][0] > 3) + self.assertGreater(mock_time.sleep.call_args_list[1][0][0], 3) self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall @@ -139,7 +139,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual(self.mock_auth.cleanup.call_count, 1) # Test if list first element is http-01, use typ because it is an achall for achall in self.mock_auth.cleanup.call_args[0][0]: - self.assertTrue(achall.typ in ["http-01", "dns-01"]) + self.assertIn(achall.typ, ["http-01", "dns-01"]) # Length of authorizations list self.assertEqual(len(authzr), 1) @@ -225,7 +225,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with self.assertRaises(errors.AuthorizationError) as error: # We retry only once, so retries will be exhausted before STATUS_VALID is returned. self.handler.handle_authorizations(mock_order, False, 1) - self.assertTrue('All authorizations were not finalized by the CA.' in str(error.exception)) + self.assertIn('All authorizations were not finalized by the CA.', str(error.exception)) def test_no_domains(self): mock_order = mock.MagicMock(authorizations=[]) @@ -305,7 +305,7 @@ class HandleAuthorizationsTest(unittest.TestCase): with test_util.patch_get_utility(): with self.assertRaises(errors.AuthorizationError) as error: self.handler.handle_authorizations(mock_order, False) - self.assertTrue('Some challenges have failed.' in str(error.exception)) + self.assertIn('Some challenges have failed.', str(error.exception)) self.assertEqual(self.mock_auth.cleanup.call_count, 1) self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "http-01") @@ -341,7 +341,7 @@ class HandleAuthorizationsTest(unittest.TestCase): self.handler.handle_authorizations(mock_order, True) # Despite best_effort=True, process will fail because no authzr is valid. - self.assertTrue('All challenges have failed.' in str(error.exception)) + self.assertIn('All challenges have failed.', str(error.exception)) def test_validated_challenge_not_rerun(self): # With a pending challenge that is not supported by the plugin, we @@ -486,7 +486,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): } # Prevent future regressions if the error type changes - self.assertTrue(kwargs["error"].description is not None) + self.assertIsNotNone(kwargs["error"].description) http_01 = messages.ChallengeBody(**kwargs) @@ -511,7 +511,7 @@ class ReportFailedAuthzrsTest(unittest.TestCase): auth_handler._report_failed_authzrs([self.authzr1], 'key') call_list = mock_zope().add_message.call_args_list self.assertEqual(len(call_list), 1) - self.assertTrue("Domain: example.com\nType: tls\nDetail: detail" in call_list[0][0][0]) + self.assertIn("Domain: example.com\nType: tls\nDetail: detail", call_list[0][0][0]) @test_util.patch_get_utility() def test_different_errors_and_domains(self, mock_zope): diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index 918459256..3e8fb0de7 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -211,7 +211,7 @@ class CertificatesTest(BaseCertManagerTest): def test_certificates_quiet(self, mock_utility, mock_logger): self.config.quiet = True self._certificates(self.config) - self.assertFalse(mock_utility.notification.called) + self.assertIs(mock_utility.notification.called, False) self.assertTrue(mock_logger.warning.called) #pylint: disable=no-member @mock.patch('certbot.crypto_util.verify_renewable_cert') @@ -224,7 +224,7 @@ class CertificatesTest(BaseCertManagerTest): mock_verifier.return_value = None mock_report.return_value = "" self._certificates(self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_report.called) self.assertTrue(mock_utility.called) self.assertTrue(mock_renewable_cert.called) @@ -242,7 +242,7 @@ class CertificatesTest(BaseCertManagerTest): filesystem.makedirs(empty_config.renewal_configs_dir) self._certificates(empty_config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) @@ -269,31 +269,34 @@ class CertificatesTest(BaseCertManagerTest): get_report = lambda: cert_manager._report_human_readable(mock_config, parsed_certs) out = get_report() - self.assertTrue("INVALID: EXPIRED" in out) + self.assertIn("INVALID: EXPIRED", out) cert.target_expiry += datetime.timedelta(hours=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIs('1 hour' in out or '2 hour(s)' in out, True) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=1) # pylint: disable=protected-access out = get_report() - self.assertTrue('1 day' in out) - self.assertFalse('under' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('1 day', out) + self.assertNotIn('under', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.target_expiry += datetime.timedelta(days=2) # pylint: disable=protected-access out = get_report() - self.assertTrue('3 days' in out) - self.assertTrue('VALID' in out and 'INVALID' not in out) + self.assertIn('3 days', out) + self.assertIn('VALID', out) + self.assertNotIn('INVALID', out) cert.is_test_cert = True mock_revoked.return_value = True out = get_report() - self.assertTrue('INVALID: TEST_CERT, REVOKED' in out) + self.assertIn('INVALID: TEST_CERT, REVOKED', out) cert = mock.MagicMock(lineagename="indescribable") cert.target_expiry = expiry @@ -353,7 +356,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "other.com.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @mock.patch('certbot.util.make_or_verify_dir') @@ -361,7 +364,7 @@ class LineageForCertnameTest(BaseCertManagerTest): def test_no_renewal_file(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.side_effect = errors.CertStorageError() from certbot._internal import cert_manager - self.assertEqual(cert_manager.lineage_for_certname(self.config, "example.com"), None) + self.assertIsNone(cert_manager.lineage_for_certname(self.config, "example.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -388,7 +391,7 @@ class DomainsForCertnameTest(BaseCertManagerTest): def test_no_match(self, mock_renewal_conf_file, mock_make_or_verify_dir): mock_renewal_conf_file.return_value = "somefile.conf" from certbot._internal import cert_manager - self.assertEqual(cert_manager.domains_for_certname(self.config, "other.com"), None) + self.assertIsNone(cert_manager.domains_for_certname(self.config, "other.com")) self.assertTrue(mock_make_or_verify_dir.called) @@ -450,7 +453,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -463,7 +466,7 @@ class RenameLineageTest(BaseCertManagerTest): self._call(self.config) from certbot._internal import cert_manager updated_lineage = cert_manager.lineage_for_certname(self.config, self.config.new_certname) - self.assertTrue(updated_lineage is not None) + self.assertIsNotNone(updated_lineage) self.assertEqual(updated_lineage.lineagename, self.config.new_certname) @test_util.patch_get_utility() @@ -503,12 +506,12 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): result = find_duplicative_certs( self.config, ['example.com', 'www.example.com']) self.assertTrue(result[0].configfile.filename.endswith('example.org.conf')) - self.assertEqual(result[1], None) + self.assertIsNone(result[1]) # Superset result = find_duplicative_certs( self.config, ['example.com', 'www.example.com', 'something.new']) - self.assertEqual(result[0], None) + self.assertIsNone(result[0]) self.assertTrue(result[1].configfile.filename.endswith('example.org.conf')) # Partial overlap doesn't count @@ -629,8 +632,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=False), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().menu.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().menu.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') @@ -671,8 +673,7 @@ class GetCertnameTest(unittest.TestCase): self.assertEqual( cert_manager.get_certnames( self.config, "verb", allow_multiple=True), ['example.com']) - self.assertTrue( - prompt in self.mock_get_utility().checklist.call_args[0][0]) + self.assertIn(prompt, self.mock_get_utility().checklist.call_args[0][0]) @mock.patch('certbot._internal.storage.renewal_conf_files') @mock.patch('certbot._internal.storage.lineagename_for_filename') diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 0a0d2634a..8cab7a5b1 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -150,79 +150,79 @@ class ParseTest(unittest.TestCase): def test_help(self): self._help_output(['--help']) # assert SystemExit is raised here out = self._help_output(['--help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) - self.assertTrue("--renew-hook" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) + self.assertNotIn("--renew-hook", out) out = self._help_output(['-h', 'nginx']) if "nginx" in PLUGINS: # may be false while building distributions without plugins - self.assertTrue("--nginx-ctl" in out) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--checkpoints" not in out) + self.assertIn("--nginx-ctl", out) + self.assertNotIn("--webroot-path", out) + self.assertNotIn("--checkpoints", out) out = self._help_output(['-h']) if "nginx" in PLUGINS: - self.assertTrue("Use the Nginx plugin" in out) + self.assertIn("Use the Nginx plugin", out) else: - self.assertTrue("(the certbot nginx plugin is not" in out) + self.assertIn("(the certbot nginx plugin is not", out) out = self._help_output(['--help', 'plugins']) - self.assertTrue("--webroot-path" not in out) - self.assertTrue("--prepare" in out) - self.assertTrue('"plugins" subcommand' in out) + self.assertNotIn("--webroot-path", out) + self.assertIn("--prepare", out) + self.assertIn('"plugins" subcommand', out) # test multiple topics out = self._help_output(['-h', 'renew']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'automation']) - self.assertTrue("--keep" in out) + self.assertIn("--keep", out) out = self._help_output(['-h', 'revoke']) - self.assertTrue("--keep" not in out) + self.assertNotIn("--keep", out) out = self._help_output(['--help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['--help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) - self.assertTrue("--reason" in out) - self.assertTrue("--delete-after-revoke" in out) - self.assertTrue("--no-delete-after-revoke" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) + self.assertIn("--reason", out) + self.assertIn("--delete-after-revoke", out) + self.assertIn("--no-delete-after-revoke", out) out = self._help_output(['-h', 'register']) - self.assertTrue("--cert-path" not in out) - self.assertTrue("--key-path" not in out) + self.assertNotIn("--cert-path", out) + self.assertNotIn("--key-path", out) out = self._help_output(['-h']) - self.assertTrue(cli.SHORT_USAGE in out) - self.assertTrue(cli.COMMAND_OVERVIEW[:100] in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn(cli.SHORT_USAGE, out) + self.assertIn(cli.COMMAND_OVERVIEW[:100], out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) def test_help_no_dashes(self): self._help_output(['help']) # assert SystemExit is raised here out = self._help_output(['help', 'all']) - self.assertTrue("--configurator" in out) - self.assertTrue("how a certificate is deployed" in out) - self.assertTrue("--webroot-path" in out) - self.assertTrue("--text" not in out) - self.assertTrue("%s" not in out) - self.assertTrue("{0}" not in out) + self.assertIn("--configurator", out) + self.assertIn("how a certificate is deployed", out) + self.assertIn("--webroot-path", out) + self.assertNotIn("--text", out) + self.assertNotIn("%s", out) + self.assertNotIn("{0}", out) out = self._help_output(['help', 'install']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) out = self._help_output(['help', 'revoke']) - self.assertTrue("--cert-path" in out) - self.assertTrue("--key-path" in out) + self.assertIn("--cert-path", out) + self.assertIn("--key-path", out) def test_parse_domains(self): short_args = ['-d', 'example.com'] @@ -270,8 +270,8 @@ class ParseTest(unittest.TestCase): def test_must_staple_flag(self): short_args = ['--must-staple'] namespace = self.parse(short_args) - self.assertTrue(namespace.must_staple) - self.assertTrue(namespace.staple) + self.assertIs(namespace.must_staple, True) + self.assertIs(namespace.staple, True) def _check_server_conflict_message(self, parser_args, conflicting_args): try: @@ -280,31 +280,31 @@ class ParseTest(unittest.TestCase): "The following flags didn't conflict with " '--server: {0}'.format(', '.join(conflicting_args))) except errors.Error as error: - self.assertTrue('--server' in str(error)) + self.assertIn('--server', str(error)) for arg in conflicting_args: - self.assertTrue(arg in str(error)) + self.assertIn(arg, str(error)) def test_staging_flag(self): short_args = ['--staging'] namespace = self.parse(short_args) - self.assertTrue(namespace.staging) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) short_args += '--server example.com'.split() self._check_server_conflict_message(short_args, '--staging') def _assert_dry_run_flag_worked(self, namespace, existing_account): - self.assertTrue(namespace.dry_run) - self.assertTrue(namespace.break_my_certs) - self.assertTrue(namespace.staging) + self.assertIs(namespace.dry_run, True) + self.assertIs(namespace.break_my_certs, True) + self.assertIs(namespace.staging, True) self.assertEqual(namespace.server, constants.STAGING_URI) if existing_account: - self.assertTrue(namespace.tos) - self.assertTrue(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, True) + self.assertIs(namespace.register_unsafely_without_email, True) else: - self.assertFalse(namespace.tos) - self.assertFalse(namespace.register_unsafely_without_email) + self.assertIs(namespace.tos, False) + self.assertIs(namespace.register_unsafely_without_email, False) def test_dry_run_flag(self): config_dir = tempfile.mkdtemp() @@ -350,8 +350,8 @@ class ParseTest(unittest.TestCase): key_size_value = cli.flag_default(key_size_option) self.parse('--rsa-key-size {0}'.format(key_size_value).split()) - self.assertTrue(cli.option_was_set(key_size_option, key_size_value)) - self.assertTrue(cli.option_was_set('no_verify_ssl', True)) + self.assertIs(cli.option_was_set(key_size_option, key_size_value), True) + self.assertIs(cli.option_was_set('no_verify_ssl', True), True) config_dir_option = 'config_dir' self.assertFalse(cli.option_was_set( @@ -425,7 +425,7 @@ class ParseTest(unittest.TestCase): value = "foo" namespace = self.parse( ["--renew-hook", value, "--disable-hook-validation"]) - self.assertEqual(namespace.deploy_hook, None) + self.assertIsNone(namespace.deploy_hook) self.assertEqual(namespace.renew_hook, value) def test_max_log_backups_error(self): @@ -456,19 +456,19 @@ class ParseTest(unittest.TestCase): self.assertFalse(self.parse(["--no-directory-hooks"]).directory_hooks) def test_no_directory_hooks_unset(self): - self.assertTrue(self.parse([]).directory_hooks) + self.assertIs(self.parse([]).directory_hooks, True) def test_delete_after_revoke(self): namespace = self.parse(["--delete-after-revoke"]) - self.assertTrue(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, True) def test_delete_after_revoke_default(self): namespace = self.parse([]) - self.assertEqual(namespace.delete_after_revoke, None) + self.assertIsNone(namespace.delete_after_revoke) def test_no_delete_after_revoke(self): namespace = self.parse(["--no-delete-after-revoke"]) - self.assertFalse(namespace.delete_after_revoke) + self.assertIs(namespace.delete_after_revoke, False) def test_allow_subset_with_wildcard(self): self.assertRaises(errors.Error, self.parse, @@ -477,7 +477,7 @@ class ParseTest(unittest.TestCase): def test_route53_no_revert(self): for help_flag in ['-h', '--help']: for topic in ['all', 'plugins', 'dns-route53']: - self.assertFalse('certbot-route53:auth' in self._help_output([help_flag, topic])) + self.assertNotIn('certbot-route53:auth', self._help_output([help_flag, topic])) class DefaultTest(unittest.TestCase): @@ -490,8 +490,8 @@ class DefaultTest(unittest.TestCase): self.default2 = cli._Default() def test_boolean(self): - self.assertFalse(self.default1) - self.assertFalse(self.default2) + self.assertIs(bool(self.default1), False) + self.assertIs(bool(self.default2), False) def test_equality(self): self.assertEqual(self.default1, self.default2) @@ -514,7 +514,7 @@ class SetByCliTest(unittest.TestCase): def test_webroot_map(self): args = '-w /var/www/html -d example.com'.split() verb = 'renew' - self.assertTrue(_call_set_by_cli('webroot_map', args, verb)) + self.assertIs(_call_set_by_cli('webroot_map', args, verb), True) def _call_set_by_cli(var, args, verb): diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 1d88ea84c..a89636a63 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -93,11 +93,11 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.prepare_subscription") as mock_prepare: mock_client().new_account_and_tos.side_effect = errors.Error self.assertRaises(errors.Error, self._call) - self.assertFalse(mock_prepare.called) + self.assertIs(mock_prepare.called, False) mock_client().new_account_and_tos.side_effect = None self._call() - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_it(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -117,7 +117,7 @@ class RegisterTest(test_util.ConfigTestCase): mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self._call() self.assertEqual(mock_get_email.call_count, 1) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) def test_email_invalid_noninteractive(self): from acme import messages @@ -144,7 +144,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = False self._call() mock_logger.debug.assert_called_once_with(mock.ANY) - self.assertTrue(mock_prepare.called) + self.assertIs(mock_prepare.called, True) @mock.patch("certbot._internal.client.display_ops.get_email") def test_dry_run_no_staging_account(self, mock_get_email): @@ -155,7 +155,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.dry_run = True self._call() # check Certbot did not ask the user to provide an email - self.assertFalse(mock_get_email.called) + self.assertIs(mock_get_email.called, False) # check Certbot created an account with no email. Contact should return empty self.assertFalse(mock_client().new_account_and_tos.call_args[0][0].contact) @@ -172,7 +172,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = "J2OAqW4MHXsrHVa_PVg0Y-L_R4SYw0_aL1le6mfblbE" self._call() - self.assertTrue(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, True) def test_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -184,7 +184,7 @@ class RegisterTest(test_util.ConfigTestCase): self.config.eab_hmac_key = None self._call() - self.assertFalse(mock_eab_from_data.called) + self.assertIs(mock_eab_from_data.called, False) def test_external_account_required_without_eab_arguments(self): with mock.patch("certbot._internal.client.acme_client.BackwardsCompatibleClientV2") as mock_client: @@ -209,7 +209,7 @@ class RegisterTest(test_util.ConfigTestCase): with mock.patch("certbot._internal.eff.handle_subscription") as mock_handle: mock_client().new_account_and_tos.side_effect = [mx_err, mock.MagicMock()] self.assertRaises(messages.Error, self._call) - self.assertFalse(mock_handle.called) + self.assertIs(mock_handle.called, False) class ClientTestCommon(test_util.ConfigTestCase): @@ -246,7 +246,7 @@ class ClientTest(ClientTestCommon): def test_init_acme_verify_ssl(self): net = self.acme_client.call_args[0][0] - self.assertTrue(net.verify_ssl) + self.assertIs(net.verify_ssl, True) def _mock_obtain_certificate(self): self.client.auth_handler = mock.MagicMock() @@ -608,7 +608,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_header(self, mock_log): self.config.hsts = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'Strict-Transport-Security') @@ -616,7 +616,7 @@ class EnhanceConfigTest(ClientTestCommon): def test_already_exists_redirect(self, mock_log): self.config.redirect = True self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) + self.assertIs(mock_log.warning.called, True) self.assertEqual(mock_log.warning.call_args[0][1], 'redirect') @@ -624,13 +624,13 @@ class EnhanceConfigTest(ClientTestCommon): def test_config_set_no_warning_redirect(self, mock_log): self.config.redirect = False self._test_with_already_existing() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) @mock.patch("certbot._internal.client.logger") def test_no_warn_redirect(self, mock_log): self.config.redirect = None self._test_with_all_supported() - self.assertFalse(mock_log.warning.called) + self.assertIs(mock_log.warning.called, False) def test_no_ask_hsts(self): self.config.hsts = True @@ -683,7 +683,7 @@ class EnhanceConfigTest(ClientTestCommon): def _test_error_with_rollback(self): self._test_error() - self.assertTrue(self.client.installer.restart.called) + self.assertIs(self.client.installer.restart.called, True) def _test_error(self): self.config.redirect = True diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index 47da2b415..9aab49c34 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -157,17 +157,17 @@ class UmaskTest(TempDirTestCase): try: dir1 = os.path.join(self.tempdir, 'probe1') filesystem.mkdir(dir1) - self.assertTrue(filesystem.check_mode(dir1, 0o755)) + self.assertIs(filesystem.check_mode(dir1, 0o755), True) filesystem.umask(0o077) dir2 = os.path.join(self.tempdir, 'dir2') filesystem.mkdir(dir2) - self.assertTrue(filesystem.check_mode(dir2, 0o700)) + self.assertIs(filesystem.check_mode(dir2, 0o700), True) dir3 = os.path.join(self.tempdir, 'dir3') filesystem.mkdir(dir3, mode=0o777) - self.assertTrue(filesystem.check_mode(dir3, 0o700)) + self.assertIs(filesystem.check_mode(dir3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -177,17 +177,17 @@ class UmaskTest(TempDirTestCase): try: file1 = os.path.join(self.tempdir, 'probe1') UmaskTest._create_file(file1) - self.assertTrue(filesystem.check_mode(file1, 0o755)) + self.assertIs(filesystem.check_mode(file1, 0o755), True) filesystem.umask(0o077) file2 = os.path.join(self.tempdir, 'probe2') UmaskTest._create_file(file2) - self.assertTrue(filesystem.check_mode(file2, 0o700)) + self.assertIs(filesystem.check_mode(file2, 0o700), True) file3 = os.path.join(self.tempdir, 'probe3') UmaskTest._create_file(file3) - self.assertTrue(filesystem.check_mode(file3, 0o700)) + self.assertIs(filesystem.check_mode(file3, 0o700), True) finally: filesystem.umask(previous_umask) @@ -400,7 +400,7 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): util.safe_open(path1, 'w').close() util.safe_open(path2, 'w').close() - self.assertTrue(filesystem.has_same_ownership(path1, path2)) + self.assertIs(filesystem.has_same_ownership(path1, path2), True) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_copy_ownership_and_mode_windows(self): @@ -408,8 +408,8 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): dst = _create_probe(self.tempdir, name='dst') filesystem.chmod(src, 0o700) - self.assertTrue(filesystem.check_mode(src, 0o700)) - self.assertTrue(filesystem.check_mode(dst, 0o744)) + self.assertIs(filesystem.check_mode(src, 0o700), True) + self.assertIs(filesystem.check_mode(dst, 0o744), True) # Checking an actual change of owner is tricky during a unit test, since we do not know # if any user exists beside the current one. So we mock _copy_win_ownership. It's behavior @@ -418,7 +418,7 @@ class CopyOwnershipAndModeTest(test_util.TempDirTestCase): filesystem.copy_ownership_and_mode(src, dst) mock_copy_owner.assert_called_once_with(src, dst) - self.assertTrue(filesystem.check_mode(dst, 0o700)) + self.assertIs(filesystem.check_mode(dst, 0o700), True) class CheckPermissionsTest(test_util.TempDirTestCase): @@ -428,14 +428,14 @@ class CheckPermissionsTest(test_util.TempDirTestCase): self.probe_path = _create_probe(self.tempdir) def test_check_mode(self): - self.assertTrue(filesystem.check_mode(self.probe_path, 0o744)) + self.assertIs(filesystem.check_mode(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.check_mode(self.probe_path, 0o744)) @unittest.skipIf(POSIX_MODE, reason='Test specific to Windows security') def test_check_owner_windows(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) system = win32security.ConvertStringSidToSid(SYSTEM_SID) security = win32security.SECURITY_ATTRIBUTES().SECURITY_DESCRIPTOR @@ -447,7 +447,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): @unittest.skipUnless(POSIX_MODE, reason='Test specific to Linux security') def test_check_owner_linux(self): - self.assertTrue(filesystem.check_owner(self.probe_path)) + self.assertIs(filesystem.check_owner(self.probe_path), True) import os as std_os # pylint: disable=os-module-forbidden # See related inline comment in certbot.compat.filesystem.check_owner method @@ -459,7 +459,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): self.assertFalse(filesystem.check_owner(self.probe_path)) def test_check_permissions(self): - self.assertTrue(filesystem.check_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.check_permissions(self.probe_path, 0o744), True) with mock.patch('certbot.compat.filesystem.check_mode') as mock_mode: mock_mode.return_value = False @@ -471,7 +471,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_check_min_permissions(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_min_permissions(self.probe_path, 0o744)) + self.assertIs(filesystem.has_min_permissions(self.probe_path, 0o744), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_min_permissions(self.probe_path, 0o744)) @@ -481,7 +481,7 @@ class CheckPermissionsTest(test_util.TempDirTestCase): def test_is_world_reachable(self): filesystem.chmod(self.probe_path, 0o744) - self.assertTrue(filesystem.has_world_permissions(self.probe_path)) + self.assertIs(filesystem.has_world_permissions(self.probe_path), True) filesystem.chmod(self.probe_path, 0o700) self.assertFalse(filesystem.has_world_permissions(self.probe_path)) @@ -500,7 +500,7 @@ class OsReplaceTest(test_util.TempDirTestCase): filesystem.replace(src, dst) self.assertFalse(os.path.exists(src)) - self.assertTrue(os.path.exists(dst)) + self.assertIs(os.path.exists(dst), True) class RealpathTest(test_util.TempDirTestCase): @@ -542,7 +542,7 @@ class RealpathTest(test_util.TempDirTestCase): with self.assertRaises(RuntimeError) as error: filesystem.realpath(link1_path) - self.assertTrue('link1 is a loop!' in str(error.exception)) + self.assertIn('link1 is a loop!', str(error.exception)) class IsExecutableTest(test_util.TempDirTestCase): @@ -578,7 +578,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("/path/to/exe")) + self.assertIs(filesystem.is_executable("/path/to/exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") @@ -586,7 +586,7 @@ class IsExecutableTest(test_util.TempDirTestCase): with _fix_windows_runtime(): mock_access.return_value = True mock_isfile.return_value = True - self.assertTrue(filesystem.is_executable("exe")) + self.assertIs(filesystem.is_executable("exe"), True) @mock.patch("certbot.compat.filesystem.os.path.isfile") @mock.patch("certbot.compat.filesystem.os.access") diff --git a/certbot/tests/compat/misc_test.py b/certbot/tests/compat/misc_test.py index 642f395ba..e87498cbe 100644 --- a/certbot/tests/compat/misc_test.py +++ b/certbot/tests/compat/misc_test.py @@ -45,4 +45,4 @@ class ExecuteTest(unittest.TestCase): mock_logger.info.assert_any_call(mock.ANY, mock.ANY, mock.ANY, stdout) if stderr or returncode: - self.assertTrue(mock_logger.error.called) + self.assertIs(mock_logger.error.called, True) diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index a2b89ee17..432fdbe53 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -60,7 +60,7 @@ class InitSaveKeyTest(test_util.TempDirTestCase): mock_make.return_value = b'key_pem' key = self._call(1024, self.workdir) self.assertEqual(key.pem, b'key_pem') - self.assertTrue('key-certbot.pem' in key.file) + self.assertIn('key-certbot.pem', key.file) self.assertTrue(os.path.exists(os.path.join(self.workdir, key.file))) @mock.patch('certbot.crypto_util.make_key') @@ -89,7 +89,7 @@ class InitSaveCSRTest(test_util.TempDirTestCase): mock.Mock(pem='dummy_key'), 'example.com', self.tempdir) self.assertEqual(csr.data, b'csr_pem') - self.assertTrue('csr-certbot.pem' in csr.file) + self.assertIn('csr-certbot.pem', csr.file) class ValidCSRTest(unittest.TestCase): @@ -251,7 +251,7 @@ class VerifyRenewableCertTest(VerifyCertSetup): return verify_renewable_cert(renewable_cert) def test_verify_renewable_cert(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) @mock.patch('certbot.crypto_util.verify_renewable_cert_sig', side_effect=errors.Error("")) def test_verify_renewable_cert_failure(self, unused_verify_renewable_cert_sign): @@ -266,14 +266,14 @@ class VerifyRenewableCertSigTest(VerifyCertSetup): return verify_renewable_cert_sig(renewable_cert) def test_cert_sig_match(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_sig_match_ec(self): renewable_cert = mock.MagicMock() renewable_cert.cert_path = P256_CERT_PATH renewable_cert.chain_path = P256_CERT_PATH renewable_cert.key_path = P256_KEY - self.assertEqual(None, self._call(renewable_cert)) + self.assertIsNone(self._call(renewable_cert)) def test_cert_sig_mismatch(self): self.bad_renewable_cert.cert_path = test_util.vector_path('cert_512_bad.pem') @@ -288,7 +288,7 @@ class VerifyFullchainTest(VerifyCertSetup): return verify_fullchain(renewable_cert) def test_fullchain_matches(self): - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_fullchain_mismatch(self): self.assertRaises(errors.Error, self._call, self.bad_renewable_cert) @@ -308,7 +308,7 @@ class VerifyCertMatchesPrivKeyTest(VerifyCertSetup): def test_cert_priv_key_match(self): self.renewable_cert.cert = SS_CERT_PATH self.renewable_cert.privkey = RSA2048_KEY_PATH - self.assertEqual(None, self._call(self.renewable_cert)) + self.assertIsNone(self._call(self.renewable_cert)) def test_cert_priv_key_mismatch(self): self.bad_renewable_cert.privkey = RSA256_KEY_PATH diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 087617310..75b11d1d7 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -50,12 +50,12 @@ class CompleterTest(test_util.TempDirTestCase): for i in range(num_paths): completion = my_completer.complete(self.tempdir, i) - self.assertTrue(completion in self.paths) + self.assertIn(completion, self.paths) self.paths.remove(completion) - self.assertFalse(self.paths) + self.assertEqual(len(self.paths), 0) completion = my_completer.complete(self.tempdir, num_paths) - self.assertEqual(completion, None) + self.assertIsNone(completion) @unittest.skipIf('readline' not in sys.modules, reason='Not relevant if readline is not available.') @@ -98,7 +98,7 @@ class CompleterTest(test_util.TempDirTestCase): with completer.Completer(): pass - self.assertTrue(mock_readline.parse_and_bind.called) + self.assertIs(mock_readline.parse_and_bind.called, True) def enable_tab_completion(unused_command): diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 1e50ba03b..aeb3ea525 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -61,9 +61,9 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True self._call() - self.assertTrue(invalid_txt not in mock_input.call_args[0][0]) + self.assertNotIn(invalid_txt, mock_input.call_args[0][0]) self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_flag(self, mock_get_utility): @@ -73,8 +73,7 @@ class GetEmailTest(unittest.TestCase): mock_safe_email.side_effect = [False, True] self._call(optional=False) for call in mock_input.call_args_list: - self.assertTrue( - "--register-unsafely-without-email" not in call[0][0]) + self.assertNotIn("--register-unsafely-without-email", call[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_optional_invalid_unsafe(self, mock_get_utility): @@ -84,7 +83,7 @@ class GetEmailTest(unittest.TestCase): with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.side_effect = [False, True] self._call(invalid=True) - self.assertTrue(invalid_txt in mock_input.call_args[0][0]) + self.assertIn(invalid_txt, mock_input.call_args[0][0]) class ChooseAccountTest(test_util.TempDirTestCase): @@ -128,7 +127,7 @@ class ChooseAccountTest(test_util.TempDirTestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_cancel(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertTrue(self._call([self.acc1, self.acc2]) is None) + self.assertIsNone(self._call([self.acc1, self.acc2])) class GenHttpsNamesTest(unittest.TestCase): @@ -210,8 +209,7 @@ class ChooseNamesTest(unittest.TestCase): actual_doms = self._call(self.mock_install) self.assertEqual(mock_util().input.call_count, 1) self.assertEqual(actual_doms, [domain]) - self.assertTrue( - "configuration files" in mock_util().input.call_args[0][0]) + self.assertIn("configuration files", mock_util().input.call_args[0][0]) def test_sort_names_trivial(self): from certbot.display.ops import _sort_names @@ -353,7 +351,7 @@ class SuccessInstallationTest(unittest.TestCase): arg = mock_util().notification.call_args_list[0][0][0] for name in names: - self.assertTrue(name in arg) + self.assertIn(name, arg) class SuccessRenewalTest(unittest.TestCase): @@ -374,7 +372,7 @@ class SuccessRenewalTest(unittest.TestCase): arg = mock_util().notification.call_args_list[0][0][0] for name in names: - self.assertTrue(name in arg) + self.assertIn(name, arg) class SuccessRevocationTest(unittest.TestCase): """Test the success revocation message.""" @@ -478,8 +476,8 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[2]]) result = self._call(items, None) self.assertEqual(result, [items[2]]) - self.assertTrue(mock_util().checklist.called) - self.assertEqual(mock_util().checklist.call_args[0][0], None) + self.assertIs(mock_util().checklist.called, True) + self.assertIsNone(mock_util().checklist.call_args[0][0]) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_choose_names_success_question(self, mock_util): @@ -488,7 +486,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.OK, [items[1]]) result = self._call(items, question) self.assertEqual(result, [items[1]]) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) @test_util.patch_get_utility("certbot.display.ops.z_util") @@ -498,7 +496,7 @@ class ChooseValuesTest(unittest.TestCase): mock_util().checklist.return_value = (display_util.CANCEL, []) result = self._call(items, question) self.assertEqual(result, []) - self.assertTrue(mock_util().checklist.called) + self.assertIs(mock_util().checklist.called, True) self.assertEqual(mock_util().checklist.call_args[0][0], question) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 5f1fac8c0..ca7ecf908 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -74,7 +74,7 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message", False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_pause(self): @@ -82,25 +82,25 @@ class FileOutputDisplayTest(unittest.TestCase): with mock.patch(input_with_timeout, return_value="enter"): self.displayer.notification("message", force_interactive=True) - self.assertTrue("message" in self.mock_stdout.write.call_args[0][0]) + self.assertIn("message", self.mock_stdout.write.call_args[0][0]) def test_notification_noninteractive(self): self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) def test_notification_noninteractive2(self): # The main purpose of this test is to make sure we only call # logger.warning once which _force_noninteractive checks internally self._force_noninteractive(self.displayer.notification, "message") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) self.assertTrue(self.displayer.skipped_interaction) self._force_noninteractive(self.displayer.notification, "message2") string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message2" in string) + self.assertIn("message2", string) def test_notification_decoration(self): from certbot.compat import os @@ -110,7 +110,8 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.notification("message2", pause=False) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("- - - " in string and ("message2" + os.linesep) in string) + self.assertIn("- - - ", string) + self.assertIn("message2" + os.linesep, string) @mock.patch("certbot.display.util." "FileDisplay._get_valid_int_ans") @@ -265,7 +266,7 @@ class FileOutputDisplayTest(unittest.TestCase): result = func(*args, **kwargs) if skipped_interaction: - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) else: self.assertEqual(mock_logger.warning.call_count, 1) @@ -331,7 +332,7 @@ class FileOutputDisplayTest(unittest.TestCase): # force_interactive to prevent workflow regressions. for name in interfaces.IDisplay.names(): arg_spec = inspect.getfullargspec(getattr(self.displayer, name)) - self.assertTrue("force_interactive" in arg_spec.args) + self.assertIn("force_interactive", arg_spec.args) class NoninteractiveDisplayTest(unittest.TestCase): @@ -345,7 +346,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): self.displayer.notification("message", 10) string = self.mock_stdout.write.call_args[0][0] - self.assertTrue("message" in string) + self.assertIn("message", string) mock_logger.debug.assert_called_with("Notifying user: %s", "message") def test_notification_decoration(self): @@ -401,7 +402,7 @@ class NoninteractiveDisplayTest(unittest.TestCase): method = getattr(self.displayer, name) # asserts method accepts arbitrary keyword arguments result = inspect.getfullargspec(method).varkw - self.assertFalse(result is None) + self.assertIsNotNone(result) class SeparateListInputTest(unittest.TestCase): diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index 5da1e6fb6..0527d87d9 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -50,7 +50,7 @@ class PrepareSubscriptionTest(SubscriptionTest): self._call() actual = mock_notify.call_args[0][0] expected_part = "because you didn't provide an e-mail address" - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertIsNone(self.account.meta.register_to_eff) @test_util.patch_get_utility() @@ -92,7 +92,7 @@ class PrepareSubscriptionTest(SubscriptionTest): call_args, call_kwargs = mock_get_utility().yesno.call_args actual = call_args[0] expected_part = 'Electronic Frontier Foundation' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) self.assertFalse(call_kwargs.get('default', True)) @@ -105,7 +105,7 @@ class HandleSubscriptionTest(SubscriptionTest): @mock.patch('certbot._internal.eff.subscribe') def test_no_subscribe(self, mock_subscribe): self._call() - self.assertFalse(mock_subscribe.called) + self.assertIs(mock_subscribe.called, False) @mock.patch('certbot._internal.eff.subscribe') def test_subscribe(self, mock_subscribe): @@ -140,7 +140,7 @@ class SubscribeTest(unittest.TestCase): self.assertEqual(call_args[0], constants.EFF_SUBSCRIBE_URI) data = call_kwargs.get('data') - self.assertFalse(data is None) + self.assertIsNotNone(data) self.assertEqual(data.get('email'), self.email) def test_bad_status(self): @@ -148,7 +148,7 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() expected_part = 'because your e-mail address appears to be invalid.' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_not_ok(self): self.response.ok = False @@ -156,21 +156,21 @@ class SubscribeTest(unittest.TestCase): self._call() actual = self._get_reported_message() unexpected_part = 'because' - self.assertFalse(unexpected_part in actual) + self.assertNotIn(unexpected_part, actual) def test_response_not_json(self): self.response.json.side_effect = ValueError() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def test_response_json_missing_status_element(self): self.json.clear() self._call() actual = self._get_reported_message() expected_part = 'problem' - self.assertTrue(expected_part in actual) + self.assertIn(expected_part, actual) def _get_reported_message(self): self.assertTrue(self.mock_notify.called) @@ -179,7 +179,7 @@ class SubscribeTest(unittest.TestCase): @test_util.patch_get_utility() def test_subscribe(self, mock_get_utility): self._call() - self.assertFalse(mock_get_utility.called) + self.assertIs(mock_get_utility.called, False) if __name__ == '__main__': diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index e04fe0742..ee4c2215d 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -120,7 +120,7 @@ class ErrorHandlerTest(unittest.TestCase): sys.exit(0) except SystemExit: pass - self.assertFalse(self.init_func.called) + self.assertIs(self.init_func.called, False) def test_regular_exit(self): func = mock.MagicMock() diff --git a/certbot/tests/helpful_test.py b/certbot/tests/helpful_test.py index 1a5c2bea6..0abe277bf 100644 --- a/certbot/tests/helpful_test.py +++ b/certbot/tests/helpful_test.py @@ -18,10 +18,10 @@ class TestScanningFlags(unittest.TestCase): arg_parser = HelpfulArgumentParser(['run'], {}) detected_flag = arg_parser.prescan_for_flag('--help', ['all', 'certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) detected_flag = arg_parser.prescan_for_flag('-h', ['all, certonly']) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_unvalid_topic(self): arg_parser = HelpfulArgumentParser(['--help', 'all'], {}) @@ -30,7 +30,7 @@ class TestScanningFlags(unittest.TestCase): self.assertIs(detected_flag, True) detected_flag = arg_parser.prescan_for_flag('-h', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) def test_prescan_valid_topic(self): arg_parser = HelpfulArgumentParser(['-h', 'all'], {}) @@ -39,7 +39,7 @@ class TestScanningFlags(unittest.TestCase): self.assertEqual(detected_flag, 'all') detected_flag = arg_parser.prescan_for_flag('--help', arg_parser.help_topics) - self.assertFalse(detected_flag) + self.assertIs(detected_flag, False) class TestDetermineVerbs(unittest.TestCase): '''Tests for determine_verb methods of HelpfulArgumentParser''' @@ -90,7 +90,7 @@ class TestAdd(unittest.TestCase): metavar="EAB_KID", help="Key Identifier for External Account Binding") parsed_args = arg_parser.parser.parse_args(["--eab-kid", None]) - self.assertIs(parsed_args.eab_kid, None) + self.assertIsNone(parsed_args.eab_kid) self.assertTrue(hasattr(parsed_args, 'eab_kid')) @@ -115,7 +115,7 @@ class TestAddGroup(unittest.TestCase): self.assertTrue(arg_parser.groups["run"]) arg_parser.add_group("certonly", description="description of certonly") with self.assertRaises(KeyError): - self.assertFalse(arg_parser.groups["certonly"]) + self.assertIs(arg_parser.groups["certonly"], False) class TestParseArgsErrors(unittest.TestCase): diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 76d460ae9..bcef2e398 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -59,7 +59,7 @@ class ValidateHookTest(test_util.TempDirTestCase): @mock.patch("certbot._internal.hooks._prog") def test_unset(self, mock_prog): self._call(None, "foo") - self.assertFalse(mock_prog.called) + self.assertIs(mock_prog.called, False) class HookTest(test_util.ConfigTestCase): @@ -132,8 +132,8 @@ class PreHookTest(HookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_renew_disabled_dir_hooks(self): self.config.directory_hooks = False @@ -158,7 +158,7 @@ class PreHookTest(HookTest): def _test_no_executions_common(self): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute(self.config) - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.info.called) @@ -344,7 +344,7 @@ class DeployHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertTrue(mock_logger.warning.called) @mock.patch("certbot._internal.hooks.logger") @@ -352,8 +352,8 @@ class DeployHookTest(RenewalHookTest): self.config.deploy_hook = None mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_success(self): domains = ["example.org", "example.net"] @@ -392,7 +392,7 @@ class RenewHookTest(RenewalHookTest): self.config.dry_run = True mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) + self.assertIs(mock_execute.called, False) self.assertEqual(mock_logger.warning.call_count, 2) def test_no_hooks(self): @@ -402,8 +402,8 @@ class RenewHookTest(RenewalHookTest): with mock.patch("certbot._internal.hooks.logger") as mock_logger: mock_execute = self._call_with_mock_execute( self.config, ["example.org"], "/foo/bar") - self.assertFalse(mock_execute.called) - self.assertFalse(mock_logger.info.called) + self.assertIs(mock_execute.called, False) + self.assertIs(mock_logger.info.called, False) def test_overlap(self): self.config.renew_hook = self.dir_hook diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index ac6b539a4..b45eb8f7a 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -69,7 +69,7 @@ class LockFileTest(test_util.TempDirTestCase): try: locked_repr = repr(lock_file) self._test_repr_common(lock_file, locked_repr) - self.assertTrue('acquired' in locked_repr) + self.assertIn('acquired', locked_repr) finally: lock_file.release() @@ -78,11 +78,11 @@ class LockFileTest(test_util.TempDirTestCase): lock_file.release() released_repr = repr(lock_file) self._test_repr_common(lock_file, released_repr) - self.assertTrue('released' in released_repr) + self.assertIn('released', released_repr) def _test_repr_common(self, lock_file, lock_repr): - self.assertTrue(lock_file.__class__.__name__ in lock_repr) - self.assertTrue(self.lock_path in lock_repr) + self.assertIn(lock_file.__class__.__name__, lock_repr) + self.assertIn(self.lock_path, lock_repr) @test_util.skip_on_windows( 'Race conditions on lock are specific to the non-blocking file access approach on Linux.') @@ -102,7 +102,7 @@ class LockFileTest(test_util.TempDirTestCase): with mock.patch('certbot._internal.lock.filesystem.os.stat') as mock_stat: mock_stat.side_effect = delete_and_stat self._call(self.lock_path) - self.assertFalse(should_delete) + self.assertEqual(len(should_delete), 0) def test_removed(self): lock_file = self._call(self.lock_path) @@ -120,7 +120,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except IOError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('IOError not raised') @@ -136,7 +136,7 @@ class LockFileTest(test_util.TempDirTestCase): try: self._call(self.lock_path) except OSError as err: - self.assertTrue(msg in str(err)) + self.assertIn(msg, str(err)) else: # pragma: no cover self.fail('OSError not raised') diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 37660f51e..12daf0000 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -51,9 +51,8 @@ class PreArgParseSetupTest(unittest.TestCase): memory_handler = handler target = memory_handler.target # type: ignore else: - self.assertTrue(isinstance(handler, logging.StreamHandler)) - self.assertTrue( - isinstance(target, logging.StreamHandler)) + self.assertIsInstance(handler, logging.StreamHandler) + self.assertIsInstance(target, logging.StreamHandler) mock_register.assert_called_once_with(logging.shutdown) mock_sys.excepthook(1, 2, 3) @@ -146,7 +145,7 @@ class SetupLogFileHandlerTest(test_util.ConfigTestCase): try: self._call(self.config, 'test.log', '%(message)s') except errors.Error as err: - self.assertTrue('--logs-dir' in str(err)) + self.assertIn('--logs-dir', str(err)) else: # pragma: no cover self.fail('Error not raised.') @@ -337,7 +336,7 @@ class PostArgParseExceptHookTest(unittest.TestCase): mock_logger, output = self._test_common(get_acme_error, debug=False) self._assert_exception_logged(mock_logger.debug, messages.Error) self._assert_quiet_output(mock_logger, output) - self.assertFalse(messages.ERROR_PREFIX in output) + self.assertNotIn(messages.ERROR_PREFIX, output) def test_other_error(self): exc_type = ValueError @@ -379,20 +378,20 @@ class PostArgParseExceptHookTest(unittest.TestCase): def _assert_exception_logged(self, log_func, exc_type): self.assertTrue(log_func.called) call_kwargs = log_func.call_args[1] - self.assertTrue('exc_info' in call_kwargs) + self.assertIn('exc_info', call_kwargs) actual_exc_info = call_kwargs['exc_info'] expected_exc_info = (exc_type, mock.ANY, mock.ANY) self.assertEqual(actual_exc_info, expected_exc_info) def _assert_logfile_output(self, output): - self.assertTrue('Please see the logfile' in output) - self.assertTrue(self.log_path in output) + self.assertIn('Please see the logfile', output) + self.assertIn(self.log_path, output) def _assert_quiet_output(self, mock_logger, output): - self.assertFalse(mock_logger.exception.called) + self.assertIs(mock_logger.exception.called, False) self.assertTrue(mock_logger.debug.called) - self.assertTrue(self.error_msg in output) + self.assertIn(self.error_msg, output) class ExitWithLogPathTest(test_util.TempDirTestCase): @@ -407,13 +406,13 @@ class ExitWithLogPathTest(test_util.TempDirTestCase): open(log_file, 'w').close() err_str = self._test_common(log_file) - self.assertTrue('logfiles' not in err_str) - self.assertTrue(log_file in err_str) + self.assertNotIn('logfiles', err_str) + self.assertIn(log_file, err_str) def test_log_dir(self): err_str = self._test_common(self.tempdir) - self.assertTrue('logfiles' in err_str) - self.assertTrue(self.tempdir in err_str) + self.assertIn('logfiles', err_str) + self.assertIn(self.tempdir, err_str) # pylint: disable=inconsistent-return-statements def _test_common(self, *args, **kwargs): diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7908cf804..5689c0281 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -84,17 +84,17 @@ class TestHandleCerts(unittest.TestCase): mock_set.return_value = False with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "certname" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) mock_set.side_effect = lambda var: var != "key_type" with self.assertRaises(errors.Error) as raised: main._handle_unexpected_key_type_migration(config, cert) - self.assertTrue("Please provide both --cert-name and --key-type" in str(raised.exception)) + self.assertIn("Please provide both --cert-name and --key-type", str(raised.exception)) class RunTest(test_util.ConfigTestCase): @@ -196,7 +196,7 @@ class CertonlyTest(unittest.TestCase): self._call('certonly --webroot -d example.com'.split()) def _assert_no_pause(self, message, pause=True): # pylint: disable=unused-argument - self.assertFalse(pause) + self.assertIs(pause, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.cert_manager.domains_for_certname') @@ -260,8 +260,7 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname=None) mock_choose_names.return_value = "domainname" # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - ("domainname", None)) + self.assertEqual(main._find_domains_or_certname(mock_config, None), ("domainname", None)) @mock.patch('certbot.display.ops.choose_names') def test_no_results(self, mock_choose_names): @@ -275,8 +274,10 @@ class FindDomainsOrCertnameTest(unittest.TestCase): mock_config = mock.Mock(domains=None, certname="one.com") mock_domains.return_value = ["one.com", "two.com"] # pylint: disable=protected-access - self.assertEqual(main._find_domains_or_certname(mock_config, None), - (["one.com", "two.com"], "one.com")) + self.assertEqual( + main._find_domains_or_certname(mock_config, None), + (["one.com", "two.com"], "one.com") + ) class RevokeTest(test_util.TempDirTestCase): @@ -402,7 +403,7 @@ class RevokeTest(test_util.TempDirTestCase): mock_get_utility().yesno.return_value = False mock_delete_if_appropriate.return_value = False self._call() - self.assertFalse(mock_delete.called) + self.assertIs(mock_delete.called, False) class DeleteIfAppropriateTest(test_util.ConfigTestCase): """Tests for certbot._internal.main._delete_if_appropriate """ @@ -536,13 +537,13 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.config.account = self.accs[1].id self.assertEqual((self.accs[1], None), self._call()) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) def test_single_account(self): self.account_storage.save(self.accs[0], self.mock_client) self.assertEqual((self.accs[0], None), self._call()) self.assertEqual(self.accs[0].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.choose_account') def test_multiple_accounts(self, mock_choose_accounts): @@ -553,7 +554,7 @@ class DetermineAccountTest(test_util.ConfigTestCase): self.assertEqual( set(mock_choose_accounts.call_args[0][0]), set(self.accs)) self.assertEqual(self.accs[1].id, self.config.account) - self.assertTrue(self.config.email is None) + self.assertIsNone(self.config.email) @mock.patch('certbot._internal.client.display_ops.get_email') @mock.patch('certbot._internal.main.display_util.notify') @@ -651,7 +652,7 @@ class MainTest(test_util.ConfigTestCase): pass finally: output = toy_out.getvalue() or toy_err.getvalue() - self.assertTrue("certbot" in output, "Output is {0}".format(output)) + self.assertIn("certbot", output, "Output is {0}".format(output)) def _cli_missing_flag(self, args, message): "Ensure that a particular error raises a missing cli flag error containing message" @@ -661,8 +662,8 @@ class MainTest(test_util.ConfigTestCase): main.main(self.standard_args + args[:]) # NOTE: parser can alter its args! except errors.MissingCommandlineFlag as exc_: exc = exc_ - self.assertTrue(message in str(exc)) - self.assertTrue(exc is not None) + self.assertIn(message, str(exc)) + self.assertIsNotNone(exc) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_noninteractive(self, _): @@ -690,11 +691,11 @@ class MainTest(test_util.ConfigTestCase): self._call_no_clientmock(args) os_ver = util.get_os_info_ua() ua = acme_net.call_args[1]["user_agent"] - self.assertTrue(os_ver in ua) + self.assertIn(os_ver, ua) import platform plat = platform.platform() if "linux" in plat.lower(): - self.assertTrue(util.get_os_info_ua() in ua) + self.assertIn(util.get_os_info_ua(), ua) with mock.patch('certbot._internal.main.client.acme_client.ClientNetwork') as acme_net: ua = "bandersnatch" @@ -803,8 +804,8 @@ class MainTest(test_util.ConfigTestCase): # Sending nginx a non-existent conf dir will simulate misconfiguration # (we can only do that if certbot-nginx is actually present) ret, _, _, _ = self._call(args) - self.assertTrue("The nginx plugin is not working" in ret) - self.assertTrue("MisconfigurationError" in ret) + self.assertIn("The nginx plugin is not working", ret) + self.assertIn("MisconfigurationError", ret) self._cli_missing_flag(["--standalone"], "With the standalone plugin, you probably") @@ -813,7 +814,7 @@ class MainTest(test_util.ConfigTestCase): mock_gsc.return_value = mock.MagicMock() self._call(["certonly", "--manual", "-d", "foo.bar"]) unused_config, auth, unused_installer = mock_init.call_args[0] - self.assertTrue(isinstance(auth, manual.Authenticator)) + self.assertIsInstance(auth, manual.Authenticator) with mock.patch('certbot._internal.main.certonly') as mock_certonly: self._call(["auth", "--standalone"]) @@ -951,7 +952,7 @@ class MainTest(test_util.ConfigTestCase): self._call(['-a', 'bad_auth', 'certonly']) assert False, "Exception should have been raised" except errors.PluginSelectionError as e: - self.assertTrue('The requested bad_auth plugin does not appear' in str(e)) + self.assertIn('The requested bad_auth plugin does not appear', str(e)) def test_check_config_sanity_domain(self): # FQDN @@ -1010,8 +1011,7 @@ class MainTest(test_util.ConfigTestCase): self._certonly_new_request_common(mock_client, ['--dry-run']) self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) - self.assertTrue( - 'dry run' in mock_get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', mock_get_utility().add_message.call_args[0][0]) # Asserts we don't suggest donating after a successful dry run self.assertEqual(mock_get_utility().add_message.call_count, 1) @@ -1032,12 +1032,11 @@ class MainTest(test_util.ConfigTestCase): self.assertEqual( mock_client.obtain_and_enroll_certificate.call_count, 1) cert_msg = mock_get_utility().add_message.call_args_list[0][0][0] - self.assertTrue(cert_path in cert_msg) - self.assertTrue(date in cert_msg) - self.assertTrue(key_path in cert_msg) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn(cert_path, cert_msg) + self.assertIn(date, cert_msg) + self.assertIn(key_path, cert_msg) + self.assertIn('donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) @mock.patch('certbot._internal.eff.handle_subscription') def test_certonly_new_request_failure(self, mock_subscription): @@ -1045,7 +1044,7 @@ class MainTest(test_util.ConfigTestCase): mock_client.obtain_and_enroll_certificate.return_value = False self.assertRaises(errors.Error, self._certonly_new_request_common, mock_client) - self.assertFalse(mock_subscription.called) + self.assertIs(mock_subscription.called, False) def _test_renewal_common(self, due_for_renewal, extra_args, log_out=None, args=None, should_renew=True, error_expected=False, @@ -1120,7 +1119,7 @@ class MainTest(test_util.ConfigTestCase): finally: if log_out: with open(os.path.join(self.config.logs_dir, "letsencrypt.log")) as lf: - self.assertTrue(log_out in lf.read()) + self.assertIn(log_out, lf.read()) return mock_lineage, mock_get_utility, stdout @@ -1131,8 +1130,8 @@ class MainTest(test_util.ConfigTestCase): lineage.update_all_links_to.assert_called_once_with( lineage.latest_common_version()) cert_msg = get_utility().add_message.call_args_list[0][0][0] - self.assertTrue('fullchain.pem' in cert_msg) - self.assertTrue('donate' in get_utility().add_message.call_args[0][0]) + self.assertIn('fullchain.pem', cert_msg) + self.assertIn('donate', get_utility().add_message.call_args[0][0]) @mock.patch('certbot._internal.log.logging.handlers.RotatingFileHandler.doRollover') @mock.patch('certbot.crypto_util.notAfter') @@ -1141,7 +1140,7 @@ class MainTest(test_util.ConfigTestCase): _, get_utility, _ = self._test_renewal_common(False, ['--dry-run', '--keep'], log_out="simulating renewal") self.assertEqual(get_utility().add_message.call_count, 1) - self.assertTrue('dry run' in get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', get_utility().add_message.call_args[0][0]) self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'], log_out="Auto-renewal forced") @@ -1200,8 +1199,8 @@ class MainTest(test_util.ConfigTestCase): expiry = datetime.datetime.now() + datetime.timedelta(days=90) _, _, stdout = self._test_renewal_common(False, extra_args=None, should_renew=False, args=['renew'], expiry_date=expiry) - self.assertTrue('No renewals were attempted.' in stdout.getvalue()) - self.assertTrue('The following certificates are not due for renewal yet:' in stdout.getvalue()) + self.assertIn('No renewals were attempted.', stdout.getvalue()) + self.assertIn('The following certificates are not due for renewal yet:', stdout.getvalue()) @mock.patch('certbot._internal.log.post_arg_parse_setup') def test_quiet_renew(self, _): @@ -1209,7 +1208,7 @@ class MainTest(test_util.ConfigTestCase): args = ["renew", "--dry-run"] _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True) out = stdout.getvalue() - self.assertTrue("renew" in out) + self.assertIn("renew", out) args = ["renew", "--dry-run", "-q"] _, _, stdout = self._test_renewal_common(True, [], args=args, @@ -1275,7 +1274,7 @@ class MainTest(test_util.ConfigTestCase): if assert_oc_called: self.assertTrue(mock_renew_cert.called) else: - self.assertFalse(mock_renew_cert.called) + self.assertIs(mock_renew_cert.called, False) def test_renew_no_renewalparams(self): self._test_renew_common(assert_oc_called=False, error_expected=True) @@ -1354,7 +1353,7 @@ class MainTest(test_util.ConfigTestCase): args=['renew', '--post-hook', '{0} -c "print(\'hello world\');"' .format(sys.executable)]) - self.assertTrue('No hooks were run.' in stdout.getvalue()) + self.assertIn('No hooks were run.', stdout.getvalue()) @test_util.patch_get_utility() @mock.patch('certbot._internal.main._find_lineage_for_domains_and_certname') @@ -1365,8 +1364,8 @@ class MainTest(test_util.ConfigTestCase): mock_renewal.return_value = ('reinstall', mock.MagicMock()) mock_init.return_value = mock_client = mock.MagicMock() self._call(['-d', 'foo.bar', '-a', 'standalone', 'certonly']) - self.assertFalse(mock_client.obtain_certificate.called) - self.assertFalse(mock_client.obtain_and_enroll_certificate.called) + self.assertIs(mock_client.obtain_certificate.called, False) + self.assertIs(mock_client.obtain_and_enroll_certificate.called, False) self.assertEqual(mock_get_utility().add_message.call_count, 0) mock_report_new_cert.assert_not_called() #self.assertTrue('donate' not in mock_get_utility().add_message.call_args[0][0]) @@ -1398,7 +1397,7 @@ class MainTest(test_util.ConfigTestCase): self._call(args) if '--dry-run' in args: - self.assertFalse(mock_client.save_certificate.called) + self.assertIs(mock_client.save_certificate.called, False) else: mock_client.save_certificate.assert_called_once_with( certr, chain, cert_path, chain_path, full_path) @@ -1409,17 +1408,15 @@ class MainTest(test_util.ConfigTestCase): def test_certonly_csr(self, mock_subscription): mock_get_utility = self._test_certonly_csr_common() cert_msg = mock_get_utility().add_message.call_args_list[0][0][0] - self.assertTrue('fullchain.pem' in cert_msg) - self.assertFalse('Your key file has been saved at' in cert_msg) - self.assertTrue( - 'donate' in mock_get_utility().add_message.call_args[0][0]) - self.assertTrue(mock_subscription.called) + self.assertIn('fullchain.pem', cert_msg) + self.assertNotIn('Your key file has been saved at', cert_msg) + self.assertIn('donate', mock_get_utility().add_message.call_args[0][0]) + self.assertIs(mock_subscription.called, True) def test_certonly_csr_dry_run(self): mock_get_utility = self._test_certonly_csr_common(['--dry-run']) self.assertEqual(mock_get_utility().add_message.call_count, 1) - self.assertTrue( - 'dry run' in mock_get_utility().add_message.call_args[0][0]) + self.assertIn('dry run', mock_get_utility().add_message.call_args[0][0]) @mock.patch('certbot._internal.main._delete_if_appropriate') @mock.patch('certbot._internal.main.client.acme_client') @@ -1474,7 +1471,7 @@ class MainTest(test_util.ConfigTestCase): mocked_account.AccountFileStorage.return_value = mocked_storage mocked_storage.find_all.return_value = ["an account"] x = self._call_no_clientmock(["register", "--email", "user@example.org"]) - self.assertTrue("There is an existing account" in x[0]) + self.assertIn("There is an existing account", x[0]) @mock.patch('certbot._internal.plugins.selection.choose_configurator_plugins') @mock.patch('certbot._internal.updater._run_updaters') @@ -1487,7 +1484,7 @@ class MainTest(test_util.ConfigTestCase): updater.run_generic_updaters(self.config, None, None) # Make sure we're returning None, and hence not trying to run the # without installer - self.assertFalse(mock_run.called) + self.assertIs(mock_run.called, False) class UnregisterTest(unittest.TestCase): @@ -1531,7 +1528,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) - self.assertTrue(res is None) + self.assertIsNone(res) mock_notify.assert_called_once_with("Account deactivated.") def test_unregister_no_account(self): @@ -1548,7 +1545,7 @@ class UnregisterTest(unittest.TestCase): res = main.unregister(config, unused_plugins) m = "Could not find existing account to deactivate." self.assertEqual(res, m) - self.assertFalse(cb_client.acme.deactivate_registration.called) + self.assertIs(cb_client.acme.deactivate_registration.called, False) class MakeOrVerifyNeededDirs(test_util.ConfigTestCase): @@ -1612,7 +1609,7 @@ class EnhanceTest(test_util.ConfigTestCase): self._call(['enhance', '--redirect']) self.assertTrue(mock_pick.called) # Check that the message includes "enhancements" - self.assertTrue("enhancements" in mock_pick.call_args[0][3]) + self.assertIn("enhancements", mock_pick.call_args[0][3]) @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1626,7 +1623,7 @@ class EnhanceTest(test_util.ConfigTestCase): with mock.patch('certbot._internal.main.plug_sel.logger.warning') as mock_log: mock_client = self._call(['enhance', '-a', 'webroot', '--redirect']) self.assertTrue(mock_log.called) - self.assertTrue("make sense" in mock_log.call_args[0][0]) + self.assertIn("make sense", mock_log.call_args[0][0]) self.assertTrue(mock_client.enhance_config.called) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @@ -1644,8 +1641,8 @@ class EnhanceTest(test_util.ConfigTestCase): all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( any(getattr(mock_client.config, e) for e in not_req_enh)) - self.assertTrue( - "example.com" in mock_client.enhance_config.call_args[0][0]) + self.assertIn( + "example.com", mock_client.enhance_config.call_args[0][0]) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') @@ -1658,7 +1655,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_client = self._call(['enhance', '--redirect', '--hsts', '--non-interactive']) self.assertTrue(mock_client.enhance_config.called) - self.assertFalse(mock_choose.called) + self.assertIs(mock_choose.called, False) @mock.patch('certbot._internal.main.display_ops.choose_values') @mock.patch('certbot._internal.main.plug_sel.record_chosen_plugins') @@ -1681,7 +1678,7 @@ class EnhanceTest(test_util.ConfigTestCase): mock_pick.return_value = (None, None) mock_pick.side_effect = errors.PluginSelectionError() mock_client = self._call(['enhance', '--hsts']) - self.assertFalse(mock_client.enhance_config.called) + self.assertIs(mock_client.enhance_config.called, False) @mock.patch('certbot._internal.cert_manager.lineage_for_certname') @mock.patch('certbot._internal.main.display_ops.choose_values') diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index fe89fff9f..72a39e2d1 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -68,14 +68,14 @@ class OCSPTestOpenSSL(unittest.TestCase): mock_communicate.communicate.return_value = (None, out.partition("\n")[2]) checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(checker.host_args("x"), ["Host", "x"]) - self.assertEqual(checker.broken, False) + self.assertIs(checker.broken, False) mock_exists.return_value = False mock_popen.call_count = 0 checker = ocsp.RevocationChecker(enforce_openssl_binary_usage=True) self.assertEqual(mock_popen.call_count, 0) self.assertEqual(mock_log.call_count, 1) - self.assertEqual(checker.broken, True) + self.assertIs(checker.broken, True) @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp.crypto_util.notAfter') @@ -89,24 +89,24 @@ class OCSPTestOpenSSL(unittest.TestCase): self.checker.broken = True mock_determine.return_value = ("", "") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.checker.broken = False mock_run.return_value = tuple(openssl_happy[1:]) - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 0) mock_determine.return_value = ("http://x.co", "x.co") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) mock_run.side_effect = errors.SubprocessError("Unable to load certificate launcher") - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_run.call_count, 2) # cert expired mock_na.return_value = now mock_determine.return_value = ("", "") count_before = mock_determine.call_count - self.assertEqual(self.checker.ocsp_revoked(cert_obj), False) + self.assertIs(self.checker.ocsp_revoked(cert_obj), False) self.assertEqual(mock_determine.call_count, count_before) def test_determine_ocsp_server(self): @@ -122,22 +122,22 @@ class OCSPTestOpenSSL(unittest.TestCase): # pylint: disable=protected-access mock_run.return_value = openssl_confused from certbot import ocsp - self.assertEqual(ocsp._translate_ocsp_query(*openssl_happy), False) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_confused), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_happy), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_confused), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) mock_log.debug.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_unknown), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_unknown), False) self.assertEqual(mock_log.debug.call_count, 1) self.assertEqual(mock_log.warning.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp), False) self.assertEqual(mock_log.debug.call_count, 2) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_broken), False) + self.assertIs(ocsp._translate_ocsp_query(*openssl_broken), False) self.assertEqual(mock_log.warning.call_count, 1) mock_log.info.call_count = 0 - self.assertEqual(ocsp._translate_ocsp_query(*openssl_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_revoked), True) self.assertEqual(mock_log.info.call_count, 0) - self.assertEqual(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) + self.assertIs(ocsp._translate_ocsp_query(*openssl_expired_ocsp_revoked), True) self.assertEqual(mock_log.info.call_count, 1) @@ -236,17 +236,17 @@ class OSCPTestCryptography(unittest.TestCase): with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, http_status_code=400): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response in invalid with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.UNAUTHORIZED): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response is valid, but certificate status is unknown with _ocsp_mock(ocsp_lib.OCSPCertStatus.UNKNOWN, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # The OCSP response says that the certificate is revoked, but certificate # does not contain the OCSP extension. @@ -255,32 +255,32 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OCSP response uses an unsupported signature. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=UnsupportedAlgorithm('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # OSCP signature response is invalid. with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=InvalidSignature('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Assertion error on OCSP response validity with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL, check_signature_side_effect=AssertionError('foo')): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # No responder cert in OCSP response with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL) as mocks: mocks['mock_response'].return_value.certificates = [] revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) # Responder cert is not signed by certificate issuer with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, @@ -289,7 +289,7 @@ class OSCPTestCryptography(unittest.TestCase): mocks['mock_response'].return_value.certificates[0] = mock.Mock( issuer='fake', subject=cert.subject) revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): # This mock is necessary to avoid the first call contained in _determine_ocsp_server @@ -300,7 +300,7 @@ class OSCPTestCryptography(unittest.TestCase): side_effect=x509.ExtensionNotFound( 'Not found', x509.AuthorityInformationAccessOID.OCSP)): revoked = self.checker.ocsp_revoked(self.cert_obj) - self.assertFalse(revoked) + self.assertIs(revoked, False) @contextlib.contextmanager diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 7de3134fa..6eb5dbfce 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -174,7 +174,7 @@ class InstallerTest(test_util.ConfigTestCase): def test_current_file_hash_in_all_hashes(self): from certbot._internal.constants import ALL_SSL_DHPARAMS_HASHES - self.assertTrue(self._current_ssl_dhparams_hash() in ALL_SSL_DHPARAMS_HASHES, + self.assertIn(self._current_ssl_dhparams_hash(), ALL_SSL_DHPARAMS_HASHES, "Constants.ALL_SSL_DHPARAMS_HASHES must be appended" " with the sha256 hash of self.config.ssl_dhparams when it is updated.") @@ -330,7 +330,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): mod_ssl_conf.write("a new line for the wrong hash\n") with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) self.assertTrue(os.path.isfile(self.dest_path)) self.assertEqual(crypto_util.sha256sum(self.source_path), self._current_file_hash()) @@ -352,7 +352,7 @@ class InstallVersionControlledFileTest(test_util.TempDirTestCase): # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 1d3fec722..83dfb41ca 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -75,7 +75,7 @@ class PluginEntryPointTest(unittest.TestCase): name, PluginEntryPoint.entry_point_to_plugin_name(entry_point, with_prefix=True)) def test_description(self): - self.assertTrue("temporary webserver" in self.plugin_ep.description) + self.assertIn("temporary webserver", self.plugin_ep.description) def test_description_with_name(self): self.plugin_ep.plugin_cls = mock.MagicMock(description="Desc") @@ -101,31 +101,31 @@ class PluginEntryPointTest(unittest.TestCase): interfaces.IInstaller, interfaces.IAuthenticator))) def test__init__(self): - self.assertFalse(self.plugin_ep.initialized) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) - self.assertTrue(self.plugin_ep.problem is None) - self.assertTrue(self.plugin_ep.entry_point is EP_SA) + self.assertIs(self.plugin_ep.initialized, False) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) + self.assertIsNone(self.plugin_ep.problem) + self.assertIs(self.plugin_ep.entry_point, EP_SA) self.assertEqual("sa", self.plugin_ep.name) - self.assertTrue(self.plugin_ep.plugin_cls is standalone.Authenticator) + self.assertIs(self.plugin_ep.plugin_cls, standalone.Authenticator) def test_init(self): config = mock.MagicMock() plugin = self.plugin_ep.init(config=config) - self.assertTrue(self.plugin_ep.initialized) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.initialized, True) + self.assertIs(plugin.config, config) # memoize! - self.assertTrue(self.plugin_ep.init() is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(), plugin) + self.assertIs(plugin.config, config) # try to give different config - self.assertTrue(self.plugin_ep.init(123) is plugin) - self.assertTrue(plugin.config is config) + self.assertIs(self.plugin_ep.init(123), plugin) + self.assertIs(plugin.config, config) - self.assertFalse(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.prepared, False) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_verify(self): iface1 = mock.MagicMock(__name__="iface1") @@ -155,7 +155,7 @@ class PluginEntryPointTest(unittest.TestCase): self.plugin_ep.init(config=config) self.plugin_ep.prepare() self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) + self.assertIs(self.plugin_ep.misconfigured, False) # output doesn't matter that much, just test if it runs str(self.plugin_ep) @@ -165,12 +165,11 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.MisconfigurationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.prepare(), + errors.MisconfigurationError) self.assertTrue(self.plugin_ep.prepared) self.assertTrue(self.plugin_ep.misconfigured) - self.assertTrue(isinstance(self.plugin_ep.problem, - errors.MisconfigurationError)) + self.assertIsInstance(self.plugin_ep.problem, errors.MisconfigurationError) self.assertTrue(self.plugin_ep.available) def test_prepare_no_installation(self): @@ -178,21 +177,20 @@ class PluginEntryPointTest(unittest.TestCase): plugin.prepare.side_effect = errors.NoInstallationError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), - errors.NoInstallationError)) - self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIsInstance(self.plugin_ep.prepare(), errors.NoInstallationError) + self.assertIs(self.plugin_ep.prepared, True) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_prepare_generic_plugin_error(self): plugin = mock.MagicMock() plugin.prepare.side_effect = errors.PluginError # pylint: disable=protected-access self.plugin_ep._initialized = plugin - self.assertTrue(isinstance(self.plugin_ep.prepare(), errors.PluginError)) + self.assertIsInstance(self.plugin_ep.prepare(), errors.PluginError) self.assertTrue(self.plugin_ep.prepared) - self.assertFalse(self.plugin_ep.misconfigured) - self.assertFalse(self.plugin_ep.available) + self.assertIs(self.plugin_ep.misconfigured, False) + self.assertIs(self.plugin_ep.available, False) def test_repr(self): self.assertEqual("PluginEntryPoint#sa", repr(self.plugin_ep)) @@ -226,14 +224,14 @@ class PluginsRegistryTest(unittest.TestCase): standalone.Authenticator, webroot.Authenticator, null.Installer, null.Installer] plugins = PluginsRegistry.find_all() - self.assertTrue(plugins["sa"].plugin_cls is standalone.Authenticator) - self.assertTrue(plugins["sa"].entry_point is EP_SA) - self.assertTrue(plugins["wr"].plugin_cls is webroot.Authenticator) - self.assertTrue(plugins["wr"].entry_point is EP_WR) - self.assertTrue(plugins["ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["ep1"].entry_point is self.ep1) - self.assertTrue(plugins["p1:ep1"].plugin_cls is null.Installer) - self.assertTrue(plugins["p1:ep1"].entry_point is self.ep1) + self.assertIs(plugins["sa"].plugin_cls, standalone.Authenticator) + self.assertIs(plugins["sa"].entry_point, EP_SA) + self.assertIs(plugins["wr"].plugin_cls, webroot.Authenticator) + self.assertIs(plugins["wr"].entry_point, EP_WR) + self.assertIs(plugins["ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["ep1"].entry_point, self.ep1) + self.assertIs(plugins["p1:ep1"].plugin_cls, null.Installer) + self.assertIs(plugins["p1:ep1"].entry_point, self.ep1) def test_getitem(self): self.assertEqual(self.plugin_ep, self.reg["mock"]) @@ -296,10 +294,10 @@ class PluginsRegistryTest(unittest.TestCase): self.assertEqual({}, self.reg.available()._plugins) def test_find_init(self): - self.assertTrue(self.reg.find_init(mock.Mock()) is None) + self.assertIsNone(self.reg.find_init(mock.Mock())) self.plugin_ep.initialized = True - self.assertTrue( - self.reg.find_init(self.plugin_ep.init()) is self.plugin_ep) + self.assertIs( + self.reg.find_init(self.plugin_ep.init()), self.plugin_ep) def test_repr(self): self.plugin_ep.__repr__ = lambda _: "PluginEntryPoint#mock" diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index 377244c4c..12fac00d0 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -211,20 +211,20 @@ class CredentialsConfigurationRequireTest(test_util.TempDirTestCase): class DomainNameGuessTest(unittest.TestCase): def test_simple_case(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("example.com") ) def test_sub_domain(self): - self.assertTrue( - 'example.com' in + self.assertIn( + 'example.com', dns_common.base_domain_name_guesses("foo.bar.baz.example.com") ) def test_second_level_domain(self): - self.assertTrue( - 'example.co.uk' in + self.assertIn( + 'example.co.uk', dns_common.base_domain_name_guesses("foo.bar.baz.example.co.uk") ) diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index 08ab20456..c97e08fa6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -52,7 +52,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.assertRaises(errors.HookCommandNotFound, self.auth.prepare) def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref('example.org'), @@ -96,9 +96,8 @@ class AuthenticatorTest(test_util.TempDirTestCase): [achall.response(achall.account_key) for achall in self.achalls]) for i, (args, kwargs) in enumerate(mock_get_utility().notification.call_args_list): achall = self.achalls[i] - self.assertTrue( - achall.validation(achall.account_key) in args[0]) - self.assertFalse(kwargs['wrap']) + self.assertIn(achall.validation(achall.account_key), args[0]) + self.assertIs(kwargs['wrap'], False) def test_cleanup(self): self.config.manual_auth_hook = ('{0} -c "import sys; sys.stdout.write(\'foo\')"' @@ -119,7 +118,7 @@ class AuthenticatorTest(test_util.TempDirTestCase): os.environ['CERTBOT_TOKEN'], achall.chall.encode('token')) else: - self.assertFalse('CERTBOT_TOKEN' in os.environ) + self.assertNotIn('CERTBOT_TOKEN', os.environ) if __name__ == '__main__': diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index dad5b270a..dfdd0a7de 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -15,7 +15,7 @@ class InstallerTest(unittest.TestCase): self.installer = Installer(config=mock.MagicMock(), name="null") def test_it(self): - self.assertTrue(isinstance(self.installer.more_info(), str)) + self.assertIsInstance(self.installer.more_info(), str) self.assertEqual([], self.installer.get_all_names()) self.assertEqual([], self.installer.supported_enhancements()) diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index 7fcd213a0..60917626a 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -70,7 +70,7 @@ class PickPluginTest(unittest.TestCase): self.assertEqual(1, self.reg.visible().ifaces.call_count) def test_no_candidate(self): - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_single(self): plugin_ep = mock.MagicMock() @@ -88,7 +88,7 @@ class PickPluginTest(unittest.TestCase): self.reg.visible().ifaces().verify().available.return_value = { "bar": plugin_ep} - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) def test_multiple(self): plugin_ep = mock.MagicMock() @@ -111,7 +111,7 @@ class PickPluginTest(unittest.TestCase): with mock.patch("certbot._internal.plugins.selection.choose_plugin") as mock_choose: mock_choose.return_value = None - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) class ChoosePluginTest(unittest.TestCase): @@ -153,7 +153,7 @@ class ChoosePluginTest(unittest.TestCase): @test_util.patch_get_utility("certbot._internal.plugins.selection.z_util") def test_no_choice(self, mock_util): mock_util().menu.return_value = (display_util.CANCEL, 0) - self.assertTrue(self._call() is None) + self.assertIsNone(self._call()) class GetUnpreparedInstallerTest(test_util.ConfigTestCase): @@ -180,7 +180,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_no_installer_defined(self): self.config.configurator = None - self.assertEqual(self._call(), None) + self.assertIsNone(self._call()) def test_no_available_installers(self): self.config.configurator = "apache" @@ -190,7 +190,7 @@ class GetUnpreparedInstallerTest(test_util.ConfigTestCase): def test_get_plugin(self): self.config.configurator = "apache" installer = self._call() - self.assertTrue(installer is self.mock_apache_plugin) + self.assertIs(installer, self.mock_apache_plugin) def test_multiple_installers_returned(self): self.config.configurator = "apache" diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 5b3fffd9d..6f2ae91ba 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -29,9 +29,8 @@ class ServerManagerTest(unittest.TestCase): self.mgr = ServerManager(self.certs, self.http_01_resources) def test_init(self): - self.assertTrue(self.mgr.certs is self.certs) - self.assertTrue( - self.mgr.http_01_resources is self.http_01_resources) + self.assertIs(self.mgr.certs, self.certs) + self.assertIs(self.mgr.http_01_resources, self.http_01_resources) def _test_run_stop(self, challenge_type): server = self.mgr.run(port=0, challenge_type=challenge_type) @@ -48,7 +47,7 @@ class ServerManagerTest(unittest.TestCase): port = server.getsocknames()[0][1] server2 = self.mgr.run(port=port, challenge_type=challenges.HTTP01) self.assertEqual(self.mgr.running(), {port: server}) - self.assertTrue(server is server2) + self.assertIs(server, server2) self.mgr.stop(port) self.assertEqual(self.mgr.running(), {}) @@ -89,7 +88,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.servers = mock.MagicMock() def test_more_info(self): - self.assertTrue(isinstance(self.auth.more_info(), str)) + self.assertIsInstance(self.auth.more_info(), str) def test_get_chall_pref(self): self.assertEqual(self.auth.get_chall_pref(domain=None), @@ -126,7 +125,7 @@ class AuthenticatorTest(unittest.TestCase): def _assert_correct_yesno_call(self, mock_yesno): yesno_args, yesno_kwargs = mock_yesno.call_args - self.assertTrue("in use" in yesno_args[0]) + self.assertIn("in use", yesno_args[0]) self.assertFalse(yesno_kwargs.get("default", True)) def test_perform_eacces(self): diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index 36159e44f..d01845510 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -49,7 +49,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(KeyError, nocontent.storage.fetch, "value") self.assertTrue(mock_log.called) - self.assertTrue("no values loaded" in mock_log.call_args[0][0]) + self.assertIn("no values loaded", mock_log.call_args[0][0]) def test_load_errors_corrupted(self): with open(os.path.join(self.config.config_dir, @@ -61,7 +61,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.assertRaises(errors.PluginError, corrupted.storage.fetch, "value") - self.assertTrue("is corrupted" in mock_log.call_args[0][0]) + self.assertIn("is corrupted", mock_log.call_args[0][0]) def test_save_errors_cant_serialize(self): with mock.patch("certbot.plugins.storage.logger.error") as mock_log: @@ -71,7 +71,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.plugin.storage._data = self.plugin_cls # pylint: disable=protected-access self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not serialize" in mock_log.call_args[0][0]) + self.assertIn("Could not serialize", mock_log.call_args[0][0]) def test_save_errors_unable_to_write_file(self): mock_open = mock.mock_open() @@ -83,7 +83,7 @@ class PluginStorageTest(test_util.ConfigTestCase): self.plugin.storage._storagepath = "/tmp/whatever" self.assertRaises(errors.PluginStorageError, self.plugin.storage.save) - self.assertTrue("Could not write" in mock_log.call_args[0][0]) + self.assertIn("Could not write", mock_log.call_args[0][0]) def test_save_uninitialized(self): with mock.patch("certbot.reverter.util"): @@ -114,7 +114,7 @@ class PluginStorageTest(test_util.ConfigTestCase): ".pluginstorage.json"), 'r') as fh: psdata = fh.read() psjson = json.loads(psdata) - self.assertTrue("mockplugin" in psjson.keys()) + self.assertIn("mockplugin", psjson.keys()) self.assertEqual(len(psjson), 1) self.assertEqual(psjson["mockplugin"]["testkey"], "testvalue") diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index 9387b4ae7..1b4fcd652 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -30,7 +30,7 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', all_path): with mock.patch('certbot.util.exe_exists') as mock_exists: mock_exists.return_value = True - self.assertEqual(path_surgery("eg"), True) + self.assertIs(path_surgery("eg"), True) self.assertEqual(mock_debug.call_count, 0) self.assertEqual(os.environ["PATH"], all_path["PATH"]) if os.name != 'nt': @@ -39,9 +39,9 @@ class PathSurgeryTest(unittest.TestCase): with mock.patch.dict('os.environ', no_path): path_surgery("thingy") self.assertEqual(mock_debug.call_count, 2 if os.name != 'nt' else 1) - self.assertTrue("Failed to find" in mock_debug.call_args[0][0]) - self.assertTrue("/usr/local/bin" in os.environ["PATH"]) - self.assertTrue("/tmp" in os.environ["PATH"]) + self.assertIn("Failed to find", mock_debug.call_args[0][0]) + self.assertIn("/usr/local/bin", os.environ["PATH"]) + self.assertIn("/tmp", os.environ["PATH"]) if __name__ == "__main__": diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index 5792924a9..f158486b6 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -58,8 +58,8 @@ class AuthenticatorTest(unittest.TestCase): def test_more_info(self): more_info = self.auth.more_info() - self.assertTrue(isinstance(more_info, str)) - self.assertTrue(self.path in more_info) + self.assertIsInstance(more_info, str) + self.assertIn(self.path, more_info) def test_add_parser_arguments(self): add = mock.MagicMock() @@ -79,7 +79,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.perform([self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) @@ -96,7 +96,7 @@ class AuthenticatorTest(unittest.TestCase): self.assertRaises(errors.PluginError, self.auth.perform, [self.achall]) self.assertTrue(mock_display.menu.called) for call in mock_display.menu.call_args_list: - self.assertTrue(self.achall.domain in call[0][0]) + self.assertIn(self.achall.domain, call[0][0]) self.assertTrue(all( webroot in call[0][1] for webroot in self.config.webroot_map.values())) diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index 7c9c53fb4..edee8df6c 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -115,7 +115,7 @@ class RenewalTest(test_util.ConfigTestCase): renewal_candidate = renewal._reconstitute(lineage_config, rc_path) # This means that manual_public_ip_logging_ok was not modified in the config based on its # value in the renewal conf file - self.assertTrue(isinstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock)) + self.assertIsInstance(lineage_config.manual_public_ip_logging_ok, mock.MagicMock) class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): @@ -129,7 +129,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_allow_subset_of_names_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'allow_subset_of_names': 'True'}) - self.assertTrue(self.config.allow_subset_of_names is True) + self.assertIs(self.config.allow_subset_of_names, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_allow_subset_of_names_failure(self, mock_set_by_cli): @@ -164,7 +164,7 @@ class RestoreRequiredConfigElementsTest(test_util.ConfigTestCase): def test_must_staple_success(self, mock_set_by_cli): mock_set_by_cli.return_value = False self._call(self.config, {'must_staple': 'True'}) - self.assertTrue(self.config.must_staple is True) + self.assertIs(self.config.must_staple, True) @mock.patch('certbot._internal.renewal.cli.set_by_cli') def test_must_staple_failure(self, mock_set_by_cli): diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index 5137c81c3..55faff2a4 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -42,7 +42,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_generic_updater.generic_updates.reset_mock() updater.run_generic_updaters(self.config, mock.MagicMock(), None) self.assertEqual(mock_generic_updater.generic_updates.call_count, 1) - self.assertFalse(mock_generic_updater.restart.called) + self.assertIs(mock_generic_updater.restart.called, False) def test_renew_deployer(self): lineage = mock.MagicMock() @@ -83,13 +83,13 @@ class RenewUpdaterTest(test_util.ConfigTestCase): self.config.disable_renew_updates = True mock_geti.return_value = self.mockinstaller updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_deployer_not_called(self): self.config.disable_renew_updates = True updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) @mock.patch('certbot._internal.plugins.selection.get_unprepared_installer') def test_enhancement_no_updater(self, mock_geti): @@ -105,7 +105,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): mock_geti.return_value = self.mockinstaller with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_generic_updaters(self.config, mock.MagicMock(), None) - self.assertFalse(self.mockinstaller.update_autohsts.called) + self.assertIs(self.mockinstaller.update_autohsts.called, False) def test_enhancement_no_deployer(self): FAKEINDEX = [ @@ -120,7 +120,7 @@ class RenewUpdaterTest(test_util.ConfigTestCase): with mock.patch("certbot.plugins.enhancements._INDEX", FAKEINDEX): updater.run_renewal_deployer(self.config, mock.MagicMock(), self.mockinstaller) - self.assertFalse(self.mockinstaller.deploy_autohsts.called) + self.assertIs(self.mockinstaller.deploy_autohsts.called, False) if __name__ == '__main__': diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 7a37f782e..c4af40591 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -26,8 +26,8 @@ class ReporterTest(unittest.TestCase): self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY) self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("Line 1\n" in output) - self.assertTrue("Line 2" in output) + self.assertIn("Line 1\n", output) + self.assertIn("Line 2", output) def test_tty_print_empty(self): sys.stdout.isatty = lambda: True # type: ignore @@ -60,10 +60,10 @@ class ReporterTest(unittest.TestCase): self._add_messages() self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" in output) - self.assertTrue("Low" in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertIn("Med", output) + self.assertIn("Low", output) def _unsuccessful_exit_common(self): self._add_messages() @@ -72,10 +72,10 @@ class ReporterTest(unittest.TestCase): except ValueError: self.reporter.print_messages() output = sys.stdout.getvalue() # type: ignore - self.assertTrue("IMPORTANT NOTES:" in output) - self.assertTrue("High" in output) - self.assertTrue("Med" not in output) - self.assertTrue("Low" not in output) + self.assertIn("IMPORTANT NOTES:", output) + self.assertIn("High", output) + self.assertNotIn("Med", output) + self.assertNotIn("Low", output) def _add_messages(self): self.reporter.add_message("High", self.reporter.HIGH_PRIORITY) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 45479b16d..e8d85d4d1 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -48,7 +48,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): no_change = os.path.join(self.reverter.config.backup_dir, path, "CHANGES_SINCE") with open(no_change, "r") as f: x = f.read() - self.assertTrue("No changes" in x) + self.assertIn("No changes", x) def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt @@ -328,8 +328,8 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): # One dir left... check title all_dirs = os.listdir(self.config.backup_dir) self.assertEqual(len(all_dirs), 1) - self.assertTrue( - "First Checkpoint" in get_save_notes( + self.assertIn( + "First Checkpoint", get_save_notes( os.path.join(self.config.backup_dir, all_dirs[0]))) # Final rollback self.reverter.rollback_checkpoints(1) diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index d7ef24b43..aa5910f1e 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -195,11 +195,11 @@ class RenewableCertTests(BaseRenewableCertTest): from certbot._internal import storage self._write_out_ex_kinds() - self.assertTrue("version" not in self.config_file) + self.assertNotIn("version", self.config_file) with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) - self.assertFalse(mock_logger.warning.called) + self.assertIs(mock_logger.warning.called, False) def test_renewal_newer_version(self): from certbot._internal import storage @@ -211,7 +211,7 @@ class RenewableCertTests(BaseRenewableCertTest): with mock.patch("certbot._internal.storage.logger") as mock_logger: storage.RenewableCert(self.config_file.filename, self.config) self.assertTrue(mock_logger.info.called) - self.assertTrue("version" in mock_logger.info.call_args[0][0]) + self.assertIn("version", mock_logger.info.call_args[0][0]) def test_consistent(self): # pylint: disable=protected-access @@ -282,7 +282,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version("cert"), 10) def test_no_current_version(self): - self.assertEqual(self.test_rc.current_version("cert"), None) + self.assertIsNone(self.test_rc.current_version("cert")) def test_latest_and_next_versions(self): for ver in range(1, 6): @@ -314,12 +314,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.test_rc.latest_common_version = mock.Mock() mock_has_pending.return_value = False - self.assertEqual(self.test_rc.ensure_deployed(), True) + self.assertIs(self.test_rc.ensure_deployed(), True) self.assertEqual(mock_update.call_count, 0) self.assertEqual(mock_logger.warning.call_count, 0) mock_has_pending.return_value = True - self.assertEqual(self.test_rc.ensure_deployed(), False) + self.assertIs(self.test_rc.ensure_deployed(), False) self.assertEqual(mock_update.call_count, 1) self.assertEqual(mock_logger.warning.call_count, 1) @@ -586,7 +586,7 @@ class RenewableCertTests(BaseRenewableCertTest): self._write_out_kind(kind, 1) self.test_rc.update_all_links_to(1) self.test_rc.save_successor(1, b"newcert", None, b"new chain", self.config) - self.assertFalse(mock_ownership.called) + self.assertIs(mock_ownership.called, False) self.test_rc.save_successor(2, b"newcert", b"new_privkey", b"new chain", self.config) self.assertTrue(mock_ownership.called) @@ -767,7 +767,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_server(self): self.test_rc.configuration["renewalparams"] = {} - self.assertEqual(self.test_rc.server, None) + self.assertIsNone(self.test_rc.server) rp = self.test_rc.configuration["renewalparams"] rp["server"] = "https://acme.example/dir" self.assertEqual(self.test_rc.server, "https://acme.example/dir") @@ -775,15 +775,15 @@ class RenewableCertTests(BaseRenewableCertTest): def test_is_test_cert(self): self.test_rc.configuration["renewalparams"] = {} rp = self.test_rc.configuration["renewalparams"] - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-staging-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://staging.someotherca.com/directory" - self.assertEqual(self.test_rc.is_test_cert, True) + self.assertIs(self.test_rc.is_test_cert, True) rp["server"] = "https://acme-v01.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) rp["server"] = "https://acme-v02.api.letsencrypt.org/directory" - self.assertEqual(self.test_rc.is_test_cert, False) + self.assertIs(self.test_rc.is_test_cert, False) def test_missing_cert(self): from certbot._internal import storage @@ -817,13 +817,13 @@ class RenewableCertTests(BaseRenewableCertTest): with open(temp2, "r") as f: content = f.read() # useful value was updated - self.assertTrue("useful = new_value" in content) + self.assertIn("useful = new_value", content) # associated comment was preserved - self.assertTrue("A useful value" in content) + self.assertIn("A useful value", content) # useless value was deleted - self.assertTrue("useless" not in content) + self.assertNotIn("useless", content) # check version was stored - self.assertTrue("version = {0}".format(certbot.__version__) in content) + self.assertIn("version = {0}".format(certbot.__version__), content) # ensure permissions are copied self.assertEqual(stat.S_IMODE(os.lstat(temp).st_mode), stat.S_IMODE(os.lstat(temp2).st_mode)) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index e2f0218b8..a78b614cf 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -260,7 +260,7 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): def test_basic(self): f, path = self._call("wow") - self.assertTrue(isinstance(f, file_type)) + self.assertIsInstance(f, file_type) self.assertEqual(os.path.join(self.tempdir, "wow.conf"), path) f.close() @@ -269,9 +269,9 @@ class UniqueLineageNameTest(test_util.TempDirTestCase): for _ in range(10): items.append(self._call("wow")) f, name = items[-1] - self.assertTrue(isinstance(f, file_type)) - self.assertTrue(isinstance(name, str)) - self.assertTrue("wow-0009.conf" in name) + self.assertIsInstance(f, file_type) + self.assertIsInstance(name, str) + self.assertIn("wow-0009.conf", name) for f, _ in items: f.close() @@ -349,7 +349,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_warning_with_arg(self): @@ -357,7 +357,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.logger.warning") as mock_warn: self.parser.parse_args(["--old-option", "42"]) self.assertEqual(mock_warn.call_count, 1) - self.assertTrue("is deprecated" in mock_warn.call_args[0][0]) + self.assertIn("is deprecated", mock_warn.call_args[0][0]) self.assertEqual("--old-option", mock_warn.call_args[0][1]) def test_help(self): @@ -368,7 +368,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): self.parser.parse_args(["-h"]) except SystemExit: pass - self.assertTrue("--old-option" not in stdout.getvalue()) + self.assertNotIn("--old-option", stdout.getvalue()) def test_set_constant(self): """Test when ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set. @@ -519,7 +519,7 @@ class OsInfoTest(unittest.TestCase): m_distro.like.return_value = "first debian third" id_likes = cbutil.get_systemd_os_like() self.assertEqual(len(id_likes), 3) - self.assertTrue("debian" in id_likes) + self.assertIn("debian", id_likes) @mock.patch("certbot.util.distro") @unittest.skipUnless(sys.platform.startswith("linux"), "requires Linux") @@ -610,7 +610,7 @@ class AtexitRegisterTest(unittest.TestCase): def test_not_called(self): self._test_common(initial_pid=-1) - self.assertFalse(self.func.called) + self.assertIs(self.func.called, False) def _test_common(self, initial_pid): with mock.patch('certbot.util._INITIAL_PID', initial_pid): From d3d9a05826af027b31de5cb60948dafbf6597873 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 3 May 2021 12:38:54 -0700 Subject: [PATCH 80/99] fix client email address (#8817) client-dev@letsencrypt.org is no longer used by the Certbot team so this PR updates the email address in our packages to our current mailing list. --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-ci/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/setup.py | 2 +- windows-installer/setup.py | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 443aab6e8..2e2eb6a6d 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -37,7 +37,7 @@ setup( description='ACME protocol implementation in Python', url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 76ebbddfd..efc9ea38a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -24,7 +24,7 @@ setup( description="Apache plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py index ad7672e17..9d52b6268 100644 --- a/certbot-ci/setup.py +++ b/certbot-ci/setup.py @@ -38,7 +38,7 @@ setup( description="Certbot continuous integration framework", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index f1bf28596..51d555fbb 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -24,7 +24,7 @@ setup( description="Compatibility tests for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index fcc7c3036..b9c0244fb 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -36,7 +36,7 @@ setup( description="Cloudflare DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 6452d325d..b3185a32d 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -36,7 +36,7 @@ setup( description="CloudXNS DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 33f16314e..656800f72 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -36,7 +36,7 @@ setup( description="DigitalOcean DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 14fcd4a69..09713b928 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -47,7 +47,7 @@ setup( description="DNSimple DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index dd02d96b5..18bff0761 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -36,7 +36,7 @@ setup( description="DNS Made Easy DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 00ff0834a..8d8656fdc 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -35,7 +35,7 @@ setup( description="Gehirn Infrastructure Service DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 4933ffb0b..b7e984f2b 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -39,7 +39,7 @@ setup( description="Google Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 55dad1d60..372e2d5dd 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -35,7 +35,7 @@ setup( description="Linode DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 46c71c46b..5136ed274 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -36,7 +36,7 @@ setup( description="LuaDNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 6d3218221..e9e1a53b0 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -36,7 +36,7 @@ setup( description="NS1 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index e8e44db30..c6c27bdd6 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -36,7 +36,7 @@ setup( description="OVH DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 6b4648d91..39315c866 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -36,7 +36,7 @@ setup( description="RFC 2136 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 30c45dc84..afff18efa 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -36,7 +36,7 @@ setup( description="Route53 DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index c328429c9..bbd85944f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -35,7 +35,7 @@ setup( description="Sakura Cloud DNS Authenticator plugin for Certbot", url='https://github.com/certbot/certbot', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 11df5eddf..740218449 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -20,7 +20,7 @@ setup( description="Nginx plugin for Certbot", url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/certbot/setup.py b/certbot/setup.py index 6913d8384..e0078bd6e 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -100,7 +100,7 @@ setup( long_description=readme, url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ diff --git a/windows-installer/setup.py b/windows-installer/setup.py index d5b4d2bde..cddc9ea18 100644 --- a/windows-installer/setup.py +++ b/windows-installer/setup.py @@ -9,7 +9,7 @@ setup( description='Environment to build the Certbot Windows installer', url='https://github.com/letsencrypt/letsencrypt', author="Certbot Project", - author_email='client-dev@letsencrypt.org', + author_email='certbot-dev@eff.org', license='Apache License 2.0', python_requires='>=3.6', classifiers=[ From dd0e590de3095097832051756d13850bf1561d8e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 3 May 2021 17:42:30 -0700 Subject: [PATCH 81/99] Make a test farm tests package (#8821) Fixes https://github.com/certbot/certbot/issues/8781. This PR makes our test farm tests into a normal package so it and its dependencies can be tracked and installed like our other packages. Other noteworthy changes in this PR: * Rather than continuing to place logs in your CWD, they're placed in a temporary directory that is printed to the terminal. * `tests/letstest/auto_targets.yaml` was deleted rather than renamed because the file is no longer used. * make a letstest package * remove deleted deps * fix letstest install * add __init__.py * call main * Explicitly mention activating venv * rerename file * fix version.py path * clarify "this" * Use >= instead of caret requirement --- .gitignore | 6 -- {tests/letstest => letstest}/README.md | 22 +++---- letstest/letstest/__init__.py | 0 {tests => letstest}/letstest/multitester.py | 8 +-- .../scripts/bootstrap_os_packages.sh | 0 .../scripts/test_apache2.sh | 4 +- .../scripts/test_openssl_version.py | 0 .../scripts/test_sdists.sh | 4 +- .../letstest => letstest}/scripts/version.py | 4 +- letstest/setup.py | 45 ++++++++++++++ .../targets}/apache2_targets.yaml | 0 .../targets}/targets.yaml | 0 tests/letstest/auto_targets.yaml | 59 ------------------- tests/letstest/requirements.txt | 22 ------- tools/_release.sh | 4 +- tools/pinning/pyproject.toml | 9 +++ tools/requirements.txt | 40 +++++++------ tools/venv.py | 1 + tox.ini | 16 +++-- 19 files changed, 108 insertions(+), 136 deletions(-) rename {tests/letstest => letstest}/README.md (74%) create mode 100644 letstest/letstest/__init__.py rename {tests => letstest}/letstest/multitester.py (98%) rename {tests/letstest => letstest}/scripts/bootstrap_os_packages.sh (100%) rename {tests/letstest => letstest}/scripts/test_apache2.sh (96%) rename {tests/letstest => letstest}/scripts/test_openssl_version.py (100%) rename {tests/letstest => letstest}/scripts/test_sdists.sh (92%) rename {tests/letstest => letstest}/scripts/version.py (85%) create mode 100644 letstest/setup.py rename {tests/letstest => letstest/targets}/apache2_targets.yaml (100%) rename {tests/letstest => letstest/targets}/targets.yaml (100%) delete mode 100644 tests/letstest/auto_targets.yaml delete mode 100644 tests/letstest/requirements.txt diff --git a/.gitignore b/.gitignore index a50dddc06..285e68a42 100644 --- a/.gitignore +++ b/.gitignore @@ -30,12 +30,6 @@ tags # auth --cert-path --chain-path /*.pem -# letstest -tests/letstest/letest-*/ -tests/letstest/*.pem -tests/letstest/venv/ -tests/letstest/venv3/ - .venv # pytest cache diff --git a/tests/letstest/README.md b/letstest/README.md similarity index 74% rename from tests/letstest/README.md rename to letstest/README.md index aea4e2e13..c569d1e8f 100644 --- a/tests/letstest/README.md +++ b/letstest/README.md @@ -14,17 +14,13 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're -on a Debian based system, make sure you also have the python3-venv package -installed. If you have Python 3 installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: -``` -python3 -m venv venv3 -. ./venv3/bin/activate -pip install --requirement requirements.txt -``` -You can then configure AWS credentials and create a key by running: +This package is installed in the Certbot development environment that is +created by following the instructions at +https://certbot.eff.org/docs/contributing.html#running-a-local-copy-of-the-client. + +After activating that virtual environment, you can then configure AWS +credentials and create a key by running: ``` >aws configure --profile [interactive: enter secrets for IAM role] @@ -35,9 +31,9 @@ Note: whatever you pick for `` will be shown to other users with AWS a When prompted for a default region name, enter: `us-east-1`. ## Usage -To run tests, activate the virtual environment you created above and run: +To run tests, activate the virtual environment you created above and from this directory run: ``` ->python multitester.py targets.yaml /path/to/your/key.pem scripts/ +>letstest targets/targets.yaml /path/to/your/key.pem scripts/ ``` You can only run up to two tests at once. The following error is often indicative of there being too many AWS instances running on our account: @@ -52,7 +48,7 @@ aws ec2 terminate-instances --profile --instance-ids $(aws ec2 de It will take a minute for these instances to shut down and become available again. Running this will invalidate any in progress tests. -A folder named `letest-` is also created with a log file from each instance of the test and a file named "results" containing the output above. +A temporary directory whose name is output by the tests is also created with a log file from each instance of the test and a file named "results" containing the output above. The tests take quite a while to run. ## Scripts diff --git a/letstest/letstest/__init__.py b/letstest/letstest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/letstest/multitester.py b/letstest/letstest/multitester.py similarity index 98% rename from tests/letstest/multitester.py rename to letstest/letstest/multitester.py index 360bb979e..a56bf0f37 100644 --- a/tests/letstest/multitester.py +++ b/letstest/letstest/multitester.py @@ -22,7 +22,7 @@ Usage: >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \ --query 'KeyMaterial' --output text > MyKeyPair.pem then: ->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh +>letstest targets/targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh see: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html @@ -33,6 +33,7 @@ from multiprocessing import Manager import os import socket import sys +import tempfile import time import traceback import urllib.error as urllib_error @@ -374,9 +375,8 @@ def main(): # Set up local copy of git repo #------------------------------------------------------------------------------- - log_dir = "letest-%d"%int(time.time()) #points to logging / working directory - print("Making local dir for test repo and logs: %s"%log_dir) - local_cxn.local('mkdir %s'%log_dir) + log_dir = tempfile.mkdtemp() # points to logging / working directory + print("Local dir for test repo and logs: %s"%log_dir) try: # figure out what git object to test and locally create it in log_dir diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/letstest/scripts/bootstrap_os_packages.sh similarity index 100% rename from tests/letstest/scripts/bootstrap_os_packages.sh rename to letstest/scripts/bootstrap_os_packages.sh diff --git a/tests/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh similarity index 96% rename from tests/letstest/scripts/test_apache2.sh rename to letstest/scripts/test_apache2.sh index 77dc35f1e..9d9ca6c12 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/letstest/scripts/test_apache2.sh @@ -59,7 +59,7 @@ fi cd letsencrypt echo "Bootstrapping dependencies..." -sudo tests/letstest/scripts/bootstrap_os_packages.sh +sudo letstest/scripts/bootstrap_os_packages.sh if [ $? -ne 0 ] ; then exit 1 fi @@ -113,7 +113,7 @@ elif [ "$OS_TYPE" = "centos" ]; then fi OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) -"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +"venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" if [ $? -ne 0 ] ; then FAIL=1 fi diff --git a/tests/letstest/scripts/test_openssl_version.py b/letstest/scripts/test_openssl_version.py similarity index 100% rename from tests/letstest/scripts/test_openssl_version.py rename to letstest/scripts/test_openssl_version.py diff --git a/tests/letstest/scripts/test_sdists.sh b/letstest/scripts/test_sdists.sh similarity index 92% rename from tests/letstest/scripts/test_sdists.sh rename to letstest/scripts/test_sdists.sh index b0d53fb67..562169524 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/letstest/scripts/test_sdists.sh @@ -2,7 +2,7 @@ cd letsencrypt -BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" +BOOTSTRAP_SCRIPT="letstest/scripts/bootstrap_os_packages.sh" VENV_PATH=venv # install OS packages @@ -41,7 +41,7 @@ for pkg_dir in acme certbot $PLUGINS; do cd - done -VERSION=$(python tests/letstest/scripts/version.py) +VERSION=$(python letstest/scripts/version.py) # test sdists cd $TEMP_DIR for pkg in acme certbot $PLUGINS; do diff --git a/tests/letstest/scripts/version.py b/letstest/scripts/version.py similarity index 85% rename from tests/letstest/scripts/version.py rename to letstest/scripts/version.py index dce2eaec2..6e538b032 100755 --- a/tests/letstest/scripts/version.py +++ b/letstest/scripts/version.py @@ -9,10 +9,10 @@ from os.path import abspath, dirname, join import re -def certbot_version(caller_dir): +def certbot_version(letstest_scripts_dir): """Return the version number stamped in certbot/__init__.py.""" return re.search('''^__version__ = ['"](.+)['"].*''', - file_contents(join(dirname(dirname(dirname(caller_dir))), + file_contents(join(dirname(dirname(letstest_scripts_dir)), 'certbot', 'certbot', '__init__.py')), diff --git a/letstest/setup.py b/letstest/setup.py new file mode 100644 index 000000000..a552cf920 --- /dev/null +++ b/letstest/setup.py @@ -0,0 +1,45 @@ +from setuptools import find_packages +from setuptools import setup + +setup( + name='letstest', + version='1.0', + description='Test Certbot on different AWS images', + url='https://github.com/certbot/certbot', + author='Certbot Project', + author_email='certbot-dev@eff.org', + license='Apache License 2.0', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=[ + # awscli isn't required by the tests themselves, but it is a useful + # tool to have when using these tests to generate keys and control + # running instances so the dependency is declared here for convenience. + 'awscli', + 'boto3', + 'botocore', + # The API from Fabric 2.0+ is used instead of the 1.0 API. + 'fabric>=2', + 'pyyaml', + ], + entry_points={ + 'console_scripts': [ + 'letstest=letstest.multitester:main', + ], + } +) diff --git a/tests/letstest/apache2_targets.yaml b/letstest/targets/apache2_targets.yaml similarity index 100% rename from tests/letstest/apache2_targets.yaml rename to letstest/targets/apache2_targets.yaml diff --git a/tests/letstest/targets.yaml b/letstest/targets/targets.yaml similarity index 100% rename from tests/letstest/targets.yaml rename to letstest/targets/targets.yaml diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml deleted file mode 100644 index 01d410227..000000000 --- a/tests/letstest/auto_targets.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# These images are located in us-east-1. - -targets: - #----------------------------------------------------------------------------- - #Ubuntu - - ami: ami-095192256fe1477ad - name: ubuntu18.04LTS - type: ubuntu - virt: hvm - user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu - #----------------------------------------------------------------------------- - # Debian - - ami: ami-01db78123b2b99496 - name: debian10 - type: ubuntu - virt: hvm - user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin - - ami: ami-0ed54dd1b25657636 - name: debian9_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - #----------------------------------------------------------------------------- - # Other Redhat Distros - - ami: ami-0916c408cb02e310b - name: RHEL7 - type: centos - virt: hvm - user: ec2-user - - ami: ami-0c322300a1dd5dc79 - name: RHEL8 - type: centos - virt: hvm - user: ec2-user - #----------------------------------------------------------------------------- - # CentOS - # These Marketplace AMIs must, irritatingly, have their terms manually - # agreed to on the AWS marketplace site for any new AWS account using them... - - ami: ami-9887c6e7 - name: centos7 - type: centos - virt: hvm - user: centos - - ami: ami-01ca03df4a6012157 - name: centos8 - type: centos - virt: hvm - user: centos diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt deleted file mode 100644 index b49489283..000000000 --- a/tests/letstest/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -awscli==1.19.36 -bcrypt==3.2.0 -boto3==1.17.36 -botocore==1.20.36 -cffi==1.14.5 -colorama==0.4.3 -cryptography==3.4.6 -docutils==0.15.2 -fabric==2.6.0 -invoke==1.5.0 -jmespath==0.10.0 -paramiko==2.7.2 -pathlib2==2.3.5 -pyasn1==0.4.8 -pycparser==2.20 -PyNaCl==1.4.0 -python-dateutil==2.8.1 -PyYAML==5.4.1 -rsa==4.5 -s3transfer==0.3.6 -six==1.15.0 -urllib3==1.26.4 diff --git a/tools/_release.sh b/tools/_release.sh index ba54536bf..4e118c2d7 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -211,8 +211,8 @@ git add certbot/CHANGELOG.md git commit -m "Add contents to certbot/CHANGELOG.md for next version" echo "New root: $root" -echo "Test commands (in the letstest repo):" -echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' +echo "Test commands (in the letstest directory):" +echo 'letstest --saveinstances targets/targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' echo "In order to upload packages run the following command:" echo twine upload "$root/dist.$version/*/*" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 34fea18a6..50611e7da 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -32,9 +32,18 @@ certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} certbot = {path = "../../certbot", extras = ["dev", "docs"]} acme = {path = "../../acme", extras = ["dev", "docs"]} +letstest = {path = "../../letstest"} windows-installer = {path = "../../windows-installer"} # Extra dependencies +# awscli is just listed here as a performance optimization. As of writing this, +# there are some conflicts in shared dependencies between it and other packages +# we depend on. To try and resolve them, poetry searches through older versions +# of awscli to see if that resolves the conflict, but there are over 1000 +# versions of awscli on PyPI so this process takes too long. Providing a high +# minimum version here prevents poetry from searching this path and it fairly +# quickly resolves the dependency conflict in another way. +awscli = ">=1.19.62" # As of writing this, cython is a build dependency of pyyaml. Since there # doesn't appear to be a good way to automatically track down and pin build # dependencies in Python (see diff --git a/tools/requirements.txt b/tools/requirements.txt index b662744c6..4fb1f87e6 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -12,25 +12,26 @@ appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or pyt appnope==0.1.2 astroid==2.3.3; python_version >= "3.6" attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +awscli==1.19.62; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") azure-devops==6.0.0b4; python_version >= "3.6" -babel==2.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" backcall==0.2.0 -bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +bcrypt==3.2.0; python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cloudflare==2.8.15; python_version >= "3.6" -colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" configobj==5.0.6; python_version >= "3.6" coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") @@ -47,14 +48,15 @@ docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0 docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +fabric==2.6.0; python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.2.0; python_version >= "3.6" +google-api-python-client==2.3.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==1.29.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-auth==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" httplib2==0.19.1; python_version >= "3.6" @@ -62,6 +64,7 @@ idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +invoke==1.5.0; python_version >= "3.6" ipdb==0.13.7; python_version >= "3.6" ipython-genutils==0.2.0; python_version == "3.6" ipython==7.16.1; python_version == "3.6" @@ -93,6 +96,7 @@ paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or py parsedatetime==2.6; python_version >= "3.6" parso==0.8.2; python_version == "3.6" pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pathlib2==2.3.5; python_version >= "3.6" pexpect==4.8.0 pickleshare==0.7.5 pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" @@ -105,14 +109,14 @@ protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or p ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -pygithub==1.54.1; python_version >= "3.6" +pygithub==1.55; python_version >= "3.6" pygments==2.8.1 -pyjwt==1.7.1; python_version >= "3.6" +pyjwt==2.1.0; python_version >= "3.6" pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" pylint==2.4.3; python_version >= "3.5" -pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" pynsist==2.7; python_version >= "3.6" pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -126,7 +130,7 @@ pytest==3.2.5 python-augeas==0.5.0 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" python-digitalocean==1.16.0; python_version >= "3.6" -python-dotenv==0.17.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") @@ -140,10 +144,10 @@ requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0 requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" rfc3986==1.4.0; python_version >= "3.6" rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -s3transfer==0.3.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +six==1.15.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6" snowballstemmer==2.1.0; python_version >= "3.6" soupsieve==2.2.1; python_version >= "3.6" sphinx-rtd-theme==0.5.2; python_version >= "3.6" @@ -166,7 +170,7 @@ typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" an typing-extensions==3.7.4.3; python_version >= "3.6" uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" -virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" wcwidth==0.2.5; python_version == "3.6" webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" diff --git a/tools/venv.py b/tools/venv.py index 2e0607e70..f3f5781fa 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -44,6 +44,7 @@ REQUIREMENTS = [ '-e certbot-nginx', '-e certbot-compatibility-test', '-e certbot-ci', + '-e letstest', ] if sys.platform == 'win32': diff --git a/tox.ini b/tox.ini index f7db07b52..a45766d87 100644 --- a/tox.ini +++ b/tox.ini @@ -277,22 +277,26 @@ passenv = DOCKER_* setenv = {[testenv:oldest]setenv} [testenv:test-farm-tests-base] -changedir = tests/letstest -deps = -rtests/letstest/requirements.txt +changedir = letstest +# The package to install is in the current working directory because of the +# value of changedir. +commands = {[base]pip_install} . passenv = AWS_* setenv = AWS_DEFAULT_REGION=us-east-1 [testenv:test-farm-apache2] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} [testenv:test-farm-sdists] changedir = {[testenv:test-farm-tests-base]changedir} -commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} -deps = {[testenv:test-farm-tests-base]deps} +commands = + {[testenv:test-farm-tests-base]commands} + {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} From bb6a076fda04fb26458b9e856878edfba292307a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 4 May 2021 11:48:09 -0700 Subject: [PATCH 82/99] Update changelog for 1.15.0 release --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 7648d0ec3..46d9117d7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.15.0 - master +## 1.15.0 - 2021-05-04 ### Added From 67e3c5474447a20fb8a574be8745c4b2dac0d480 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 4 May 2021 11:50:10 -0700 Subject: [PATCH 83/99] Release 1.15.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- certbot/docs/cli-help.txt | 16 +++++++--------- 20 files changed, 26 insertions(+), 28 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 2e2eb6a6d..e414442de 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index efc9ea38a..9902bfdfb 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 51d555fbb..82094458f 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index b9c0244fb..6c6aa9cbd 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index b3185a32d..1f7927a3b 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 656800f72..f158a5c85 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 09713b928..3813cb7a2 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 18bff0761..65b06b420 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 8d8656fdc..13b179acb 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index b7e984f2b..7ce24c65d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 372e2d5dd..ddc1843f3 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 5136ed274..b49a47813 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e9e1a53b0..a5b81c9af 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index c6c27bdd6..02e0cec90 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 39315c866..15f75e4df 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index afff18efa..6c4642955 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index bbd85944f..775dd6dce 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 740218449..fd01f09dd 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.15.0.dev0' +version = '1.15.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index fbf8160fc..0cdbd46a5 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.15.0.dev0' +__version__ = '1.15.0' diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index c5490c100..5d772d90e 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -118,12 +118,12 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.14.0 - (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX - Installer/YYY (SUBCOMMAND; flags: FLAGS) - Py/major.minor.patchlevel). The flags encoded in the - user agent are: --duplicate, --force-renew, --allow- - subset-of-names, -n, and whether any hooks are set. + "". (default: CertbotACMEClient/1.15.0 (certbot; + OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). + The flags encoded in the user agent are: --duplicate, + --force-renew, --allow-subset-of-names, -n, and + whether any hooks are set. --user-agent-comment USER_AGENT_COMMENT Add a comment to the default user agent string. May be used when repackaging Certbot or calling it from @@ -216,9 +216,7 @@ testing: (invalid) certificates; equivalent to --server https://acme-staging-v02.api.letsencrypt.org/directory (default: False) - --debug Show tracebacks in case of errors, and allow certbot- - auto execution on experimental platforms (default: - False) + --debug Show tracebacks in case of errors (default: False) --no-verify-ssl Disable verification of the ACME server's certificate. (default: False) --http-01-port HTTP01_PORT From 484309ed954bdca735cac08b2527ae98530abe42 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 4 May 2021 11:50:12 -0700 Subject: [PATCH 84/99] Add contents to certbot/CHANGELOG.md for next version --- certbot/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 46d9117d7..fd2f0ffb4 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,6 +2,22 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). +## 1.16.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +More details about these changes can be found on our GitHub repo. + ## 1.15.0 - 2021-05-04 ### Added From 56c781aec47ac4b7c30f64a6be46eb34a762561a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 4 May 2021 11:50:12 -0700 Subject: [PATCH 85/99] Bump version to 1.16.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/certbot/__init__.py | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e414442de..6fa49dafc 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 9902bfdfb..898e4e3e7 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 82094458f..9567bbef6 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 6c6aa9cbd..3e3a3d1ba 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 1f7927a3b..20b499327 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index f158a5c85..fba2fbc5f 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 3813cb7a2..c0da63d71 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 65b06b420..2feae0bd1 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 13b179acb..d24f6b309 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 7ce24c65d..dd43f4992 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index ddc1843f3..72c7ee2fd 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b49a47813..b2c54779f 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a5b81c9af..a78b6c3b7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 02e0cec90..4831fa480 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 15f75e4df..a19753e79 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 6c4642955..b1c1d786c 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 775dd6dce..7b76bb324 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ import sys from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index fd01f09dd..f42a6e85d 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,7 +1,7 @@ from setuptools import find_packages from setuptools import setup -version = '1.15.0' +version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 0cdbd46a5..4557ee399 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,3 +1,3 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.15.0' +__version__ = '1.16.0.dev0' From 934de48d44e431b01dcf750de88d24ca740119d5 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 5 May 2021 15:49:06 -0700 Subject: [PATCH 86/99] fix typo (#8828) --- certbot-ci/snap_integration_tests/dns_tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-ci/snap_integration_tests/dns_tests/test_main.py b/certbot-ci/snap_integration_tests/dns_tests/test_main.py index 721352c04..d008efc67 100644 --- a/certbot-ci/snap_integration_tests/dns_tests/test_main.py +++ b/certbot-ci/snap_integration_tests/dns_tests/test_main.py @@ -44,4 +44,4 @@ def test_dns_plugin_install(dns_snap_path): 'certbot:certbot-metadata']) subprocess.check_call(['snap', 'install', '--dangerous', dns_snap_path]) finally: - subprocess.call(['snap', 'remove', 'plugin_name']) + subprocess.call(['snap', 'remove', plugin_name]) From 7eae058af595f526b42031c646b1febebdcffeca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 7 May 2021 13:10:02 -0700 Subject: [PATCH 87/99] Remove OS instructions (#8833) Fixes https://github.com/certbot/certbot/issues/8832. [These instructions are creating confusion among users](https://github.com/certbot/certbot/issues/8832) and [frustration among packagers](https://pagure.io/fesco/issue/2570) for whom the warning at the top of the OS packaging section doesn't apply. Because of this, I think we should remove them in favor of our instruction generator and snap/docker/pip instructions. I also told Fedora packagers that we could probably do this in response to them continuing to improve their Certbot packages which they've done through things like the renewal timer that is now enabled by default. --- certbot/docs/install.rst | 111 --------------------------------------- 1 file changed, 111 deletions(-) diff --git a/certbot/docs/install.rst b/certbot/docs/install.rst index 99cf11c16..4533cfcc1 100644 --- a/certbot/docs/install.rst +++ b/certbot/docs/install.rst @@ -125,117 +125,6 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. .. _Docker: https://docker.com .. _`install Docker`: https://docs.docker.com/engine/installation/ -Operating System Packages -------------------------- - -.. warning:: While the Certbot team tries to keep the Certbot packages offered - by various operating systems working in the most basic sense, due to - distribution policies and/or the limited resources of distribution - maintainers, Certbot OS packages often have problems that other distribution - mechanisms do not. The packages are often old resulting in a lack of bug - fixes and features and a worse TLS configuration than is generated by newer - versions of Certbot. They also may not configure certificate renewal for you - or have all of Certbot's plugins available. For reasons like these, we - recommend most users follow the instructions at - https://certbot.eff.org/instructions and OS packages are only documented - here as an alternative. - -**Arch Linux** - -.. code-block:: shell - - sudo pacman -S certbot - -**Debian** - -If you run Debian Buster or Debian testing/Sid, you can easily install certbot -packages through commands like: - -.. code-block:: shell - - sudo apt-get update - sudo apt-get install certbot - -If you run Debian Stretch, we recommend you use the packages in Debian -backports repository. First you'll have to follow the instructions at -https://backports.debian.org/Instructions/ to enable the Stretch backports repo, -if you have not already done so. Then run: - -.. code-block:: shell - - sudo apt-get install certbot -t stretch-backports - -In all of these cases, there also packages available to help Certbot integrate -with Apache, nginx, or various DNS services. If you are using Apache or nginx, -we strongly recommend that you install the ``python-certbot-apache`` or -``python-certbot-nginx`` package so that Certbot can fully automate HTTPS -configuration for your server. A full list of these packages can be found -through a command like: - -.. code-block:: shell - - apt search 'python-certbot*' - -They can be installed by running the same installation command above but -replacing ``certbot`` with the name of the desired package. - -**Ubuntu** - -If you run Ubuntu, certbot can be installed using: - -.. code-block:: shell - - sudo apt-get install certbot - -Optionally to install the Certbot Apache plugin, you can use: - -.. code-block:: shell - - sudo apt-get install python3-certbot-apache - -**Fedora** - -.. code-block:: shell - - sudo dnf install certbot python3-certbot-apache - -**FreeBSD** - - * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py37-certbot`` - -**Gentoo** - -The official Certbot client is available in Gentoo Portage. From the -official Certbot plugins, three of them are also available in Portage. -They need to be installed separately if you require their functionality. - -.. code-block:: shell - - emerge -av app-crypt/certbot - emerge -av app-crypt/certbot-apache - emerge -av app-crypt/certbot-nginx - emerge -av app-crypt/certbot-dns-nsone - -.. Note:: The ``app-crypt/certbot-dns-nsone`` package has a different - maintainer than the other packages and can lag behind in version. - -**NetBSD** - - * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` - * Install pre-compiled package: ``pkg_add py37-certbot`` - -**OpenBSD** - - * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` - * Package: ``pkg_add letsencrypt`` - -**Other Operating Systems** - -OS packaging is an ongoing effort. If you'd like to package -Certbot for your distribution of choice please have a -look at the :doc:`packaging`. - .. _certbot-auto: Certbot-Auto From b0552e1939a8a120bf65a1c7caaaebc45b10f040 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 10 May 2021 21:11:31 +0200 Subject: [PATCH 88/99] Fix Sphinx builds (#8838) Since Saturday the CI pipeline is failing due to several Sphinx errors. See https://dev.azure.com/certbot/certbot/_build/results?buildId=3928&view=logs&j=d74e04fe-9740-597d-e9fa-1d0400037dfd&t=dde413a4-f24c-59a0-9684-e33d79f9aa02 First, the build of certbot-dns-google is failing because of a particular configuration. It seems that this configuration has been written here to activate the support of the RST instruction `.. code-block:: json` in documentation. However, it does not seem to be necessary for a similar situation in certbot-dns-route53 documentation. So let's try to remove it and fix the Sphinx builds. Second, Sphinx builds were not pinning dependencies, so Sphinx 4.x (that has been released yesterday) started to be used in the pipeline. Sadly this new version is not compatible with the plugin `repoze.sphinx.autointerface`, used to extract documentation from `zope.interface`. So I fixed the pinning and also explicitly pin Sphinx to 3.5.x for now. Technically speaking the second action is sufficient to fix the first error, but I keep the dedicated solution because it improves the documentation in my opinion. This situation could be fixed by not requiring `repoze.sphinx.autointerface`, but this is possible only if we remove `zope.interface` from Certbot. Luckily I started the work few days ago ;). * Remove explicit lexer call in certbot-dns-google doc builds. * Write a valid JSON file in the documentation * Apply constraints to sphinx build environments * Pin Sphinx to 3.5.4 * Update dependencies * Pin traitlets --- .../templates/steps/sphinx-steps.yml | 2 +- .../certbot_dns_google/__init__.py | 12 +++++-- certbot-dns-google/docs/_ext/jsonlexer.py | 16 ---------- certbot-dns-google/docs/conf.py | 3 +- tools/pinning/pyproject.toml | 11 +++++++ tools/requirements.txt | 32 +++++++++---------- 6 files changed, 39 insertions(+), 37 deletions(-) delete mode 100644 certbot-dns-google/docs/_ext/jsonlexer.py diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml index 23c258bbc..afc273f8f 100644 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -9,7 +9,7 @@ steps: do echo "" echo "##[group]Building $doc_path" - pip install -q -e $doc_path/..[docs] + pip install -q -e $doc_path/..[docs] -c tools/requirements.txt if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then FINAL_STATUS=1 FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py index 2d448c590..67ed34a45 100644 --- a/certbot-dns-google/certbot_dns_google/__init__.py +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -51,8 +51,16 @@ are automatically obtained by certbot through the `metadata service :caption: Example credentials file: { - "type": "service_account", - ... + "type": "service_account", + "project_id": "...", + "private_key_id": "...", + "private_key": "...", + "client_email": "...", + "client_id": "...", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "..." } The path to this file can be provided interactively or using the diff --git a/certbot-dns-google/docs/_ext/jsonlexer.py b/certbot-dns-google/docs/_ext/jsonlexer.py deleted file mode 100644 index 1ad004d2b..000000000 --- a/certbot-dns-google/docs/_ext/jsonlexer.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Copied from https://stackoverflow.com/a/16863232""" - -def setup(app): - # enable Pygments json lexer - try: - import pygments - if pygments.__version__ >= '1.5': - # use JSON lexer included in recent versions of Pygments - from pygments.lexers import JsonLexer - else: - # use JSON lexer from pygments-json if installed - from pygson.json_lexer import JSONLexer as JsonLexer - except ImportError: - pass # not fatal if we have old (or no) Pygments and no pygments-json - else: - app.add_lexer('json', JsonLexer()) diff --git a/certbot-dns-google/docs/conf.py b/certbot-dns-google/docs/conf.py index 06bb99f46..f4c1f661e 100644 --- a/certbot-dns-google/docs/conf.py +++ b/certbot-dns-google/docs/conf.py @@ -35,8 +35,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'jsonlexer'] + 'sphinx.ext.viewcode'] autodoc_member_order = 'bysource' autodoc_default_flags = ['show-inheritance'] diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 50611e7da..0ff11ac2f 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -70,6 +70,17 @@ pytest-forked = "0.2" # library for now because it causes Certbot tests on Windows to fail. See # https://github.com/certbot/certbot/issues/8732. python-augeas = "0.5.0" +# Because some parts of Certbot documentation are generated from zope.interfaces, +# we need the Sphinx plugin repoze.sphinx.autointerface. But this plugin is not +# maintained anymore, and it is not compatible with Sphinx 4.x. So we need to +# pin Sphinx to 3.5 to make it work. This situation will be unblocked by the +# removal of zope.interface in Certbot. +sphinx = "<4" +# Library traitlets is a transitive dependency of ipdb (traitlets -> ipython -> ipdb). +# Version 5.x is incompatible with Python 3.6 but for some reasons, poetry fails to +# add the appropriate marker and allows this version to be installed under Python 3.6. +# We add a pinning to not create a set of requirements incompatible with Python 3.6. +traitlets = "<5" [tool.poetry.dev-dependencies] diff --git a/tools/requirements.txt b/tools/requirements.txt index 4fb1f87e6..9e442f411 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -11,16 +11,16 @@ apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" appnope==0.1.2 astroid==2.3.3; python_version >= "3.6" -attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -awscli==1.19.62; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +awscli==1.19.69; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") azure-devops==6.0.0b4; python_version >= "3.6" babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" backcall==0.2.0 bcrypt==3.2.0; python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.69; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -42,7 +42,7 @@ decorator==5.0.7 deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -dns-lexicon==3.5.6; python_version >= "3.6" and python_version < "4.0" +dns-lexicon==3.6.0; python_version >= "3.6" and python_version < "4.0" dnspython==2.1.0; python_version >= "3.6" docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" @@ -52,7 +52,6 @@ docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" fabric==2.6.0; python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" -future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" google-api-python-client==2.3.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" @@ -68,7 +67,7 @@ invoke==1.5.0; python_version >= "3.6" ipdb==0.13.7; python_version >= "3.6" ipython-genutils==0.2.0; python_version == "3.6" ipython==7.16.1; python_version == "3.6" -ipython==7.22.0; python_version >= "3.7" +ipython==7.23.1; python_version >= "3.7" isodate==0.6.0; python_version >= "3.6" isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" jedi==0.18.0 @@ -83,6 +82,7 @@ keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_ lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" lockfile==0.12.2 markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +matplotlib-inline==0.1.2; python_version >= "3.7" mccabe==0.6.1; python_version >= "3.6" mock==4.0.3; python_version >= "3.6" msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" @@ -105,14 +105,14 @@ ply==3.11; python_version >= "3.6" poetry-core==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" poetry==1.1.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" prompt-toolkit==3.0.3 -protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +protobuf==3.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pygithub==1.55; python_version >= "3.6" -pygments==2.8.1 +pygments==2.9.0 pyjwt==2.1.0; python_version >= "3.6" pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" pylint==2.4.3; python_version >= "3.5" @@ -142,16 +142,16 @@ requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" -rfc3986==1.4.0; python_version >= "3.6" +rfc3986==1.5.0; python_version >= "3.6" rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -six==1.15.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6" +six==1.16.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6" snowballstemmer==2.1.0; python_version >= "3.6" soupsieve==2.2.1; python_version >= "3.6" sphinx-rtd-theme==0.5.2; python_version >= "3.6" -sphinx==3.5.4; python_version >= "3.6" +sphinx==3.5.4; python_version >= "3.5" sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" @@ -162,18 +162,18 @@ texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or p tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" tqdm==4.60.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" -typing-extensions==3.7.4.3; python_version >= "3.6" +typing-extensions==3.10.0.0; python_version >= "3.6" uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" -virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +virtualenv==20.4.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" wcwidth==0.2.5; python_version == "3.6" webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") yarg==0.1.9; python_version >= "3.6" zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" From c788820f5dd9fefd3234c42e658d86b54eb06539 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 10 May 2021 14:32:37 -0700 Subject: [PATCH 89/99] Fix sphinx followup (#8841) I think we should use our `pip_install*` scripts wherever we can and I'm not quite sure yet if I'd call `repoze.sphinx.autointerface` unmaintained. * use pip_install_editable * update sphinx comment --- .azure-pipelines/templates/steps/sphinx-steps.yml | 2 +- tools/pinning/pyproject.toml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml index afc273f8f..7e1d2ee9d 100644 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -9,7 +9,7 @@ steps: do echo "" echo "##[group]Building $doc_path" - pip install -q -e $doc_path/..[docs] -c tools/requirements.txt + tools/pip_install_editable.py $doc_path/..[docs] if ! sphinx-build -W --keep-going -b html $doc_path $doc_path/_build/html; then FINAL_STATUS=1 FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml index 0ff11ac2f..4dee88d51 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/pyproject.toml @@ -71,10 +71,11 @@ pytest-forked = "0.2" # https://github.com/certbot/certbot/issues/8732. python-augeas = "0.5.0" # Because some parts of Certbot documentation are generated from zope.interfaces, -# we need the Sphinx plugin repoze.sphinx.autointerface. But this plugin is not -# maintained anymore, and it is not compatible with Sphinx 4.x. So we need to -# pin Sphinx to 3.5 to make it work. This situation will be unblocked by the -# removal of zope.interface in Certbot. +# we need the Sphinx plugin repoze.sphinx.autointerface. This plugin is not +# currently compatible with Sphinx>=4 though so we're pinning an older version +# for now. See https://github.com/repoze/repoze.sphinx.autointerface/issues/16 +# for more info. This pinning could also be removed by the removal of +# zope.interface in Certbot. sphinx = "<4" # Library traitlets is a transitive dependency of ipdb (traitlets -> ipython -> ipdb). # Version 5.x is incompatible with Python 3.6 but for some reasons, poetry fails to From c48adc57538f15baf98ef14ea86550015709838c Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 11 May 2021 09:03:35 +1000 Subject: [PATCH 90/99] docker: delete CARGO_HOME (#8839) --- tools/docker/core/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index 4c614d5e2..14bd3323a 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -39,4 +39,5 @@ RUN apk add --no-cache --virtual .build-deps \ && python tools/pip_install.py --no-cache-dir \ --editable src/acme \ --editable src/certbot \ - && apk del .build-deps + && apk del .build-deps \ + && rm -rf ${HOME}/.cargo From 89396cefa26dc08d369e9836f477e475cc654571 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 11 May 2021 15:42:52 -0700 Subject: [PATCH 91/99] Remove unnecessary release script output (#8820) --- tools/_release.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tools/_release.sh b/tools/_release.sh index 4e118c2d7..7dd4601ff 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -189,15 +189,6 @@ while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do done git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" -cd .. -echo Now in $PWD -name=${root_without_le%.*} -ext="${root_without_le##*.}" -rev="$(git rev-parse --short HEAD)" -echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz -cd ~- - # Add master section to CHANGELOG.md header=$(head -n 4 certbot/CHANGELOG.md) body=$(sed s/nextversion/$nextversion/ tools/_changelog_top.txt) @@ -210,12 +201,6 @@ $footer" > certbot/CHANGELOG.md git add certbot/CHANGELOG.md git commit -m "Add contents to certbot/CHANGELOG.md for next version" -echo "New root: $root" -echo "Test commands (in the letstest directory):" -echo 'letstest --saveinstances targets/targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' -echo "In order to upload packages run the following command:" -echo twine upload "$root/dist.$version/*/*" - if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 git commit -m "Bump version to $nextversion" From db409747888887c779a1640e0f9d15e141e6b367 Mon Sep 17 00:00:00 2001 From: miigotu Date: Wed, 12 May 2021 19:22:31 -0400 Subject: [PATCH 92/99] Add 3rd party certbot-dns-godaddy to the docs (#8844) * Add 3rd party certbot-dns-godaddy to the docs * fix up rst syntax for godaddy link Co-authored-by: alexzorin --- certbot/docs/using.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index cc061b622..be4d96c4f 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -285,6 +285,7 @@ dns-clouddns_ Y N DNS Authentication using CloudDNS API dns-lightsail_ Y N DNS Authentication using Amazon Lightsail DNS API dns-inwx_ Y Y DNS Authentication for INWX through the XML API dns-azure_ Y N DNS Authentication using Azure DNS +dns-godaddy_ Y N DNS Authentication using Godaddy DNS ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -300,6 +301,7 @@ dns-azure_ Y N DNS Authentication using Azure DNS .. _dns-lightsail: https://github.com/noi/certbot-dns-lightsail .. _dns-inwx: https://github.com/oGGy990/certbot-dns-inwx/ .. _dns-azure: https://github.com/binkhq/certbot-dns-azure +.. _dns-godaddy: https://github.com/miigotu/certbot-dns-godaddy If you're interested, you can also :ref:`write your own plugin `. From ee3b3656eae85b0fa029280b36579e934ab809ca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 14 May 2021 11:27:47 -0700 Subject: [PATCH 93/99] Remove old apache tests (#8843) Apache test farm tests started failing last night due to a change in pyenv. See https://dev.azure.com/certbot/certbot/_build/results?buildId=3948&view=logs&j=f67c2a39-2c4f-5190-915f-6f32a7a4306f&t=96f0f394-f513-5158-f5e7-a26e55aeadbf&l=26943. I managed to fix that in https://github.com/certbot/certbot/commit/d94f20f8b709e088b0c3036683bbda88b354f254, however, the OSes the tests were failing on were Debian 9 and Ubuntu 16.04. [Debian 9 reached its end-of-life in July 2020](https://wiki.debian.org/DebianReleases) and [Ubuntu 16.04 reached its end of standard support in April 2021](https://wiki.ubuntu.com/Releases). As shown at the same links, Debian 9 still has support from the LTS team and Ubuntu 16.04 has ESM support. Do we still want to support either of these OSes? If so, we can use the commit I linked in the first sentence of the last paragraph, but I think supporting the OSes through their standard support is good enough. The Certbot team has enough on their plate and especially when the OSes are so old that we can't even use their packaged version of Python anymore which complicates our tests, I think we can just drop support and move on. I don't have a strong opinion here though so if someone else does, let me know what you'd like to see or make the PR yourself based on the changes in my linked commit and I'll merge it. You can see the tests passing with this change at https://dev.azure.com/certbot/certbot/_build/results?buildId=3955&view=results. * Remove apache tests on old OSes * remove unused pyenv code --- letstest/scripts/test_apache2.sh | 19 ------------------- letstest/targets/apache2_targets.yaml | 10 ---------- 2 files changed, 29 deletions(-) diff --git a/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh index 9d9ca6c12..830ae44b2 100755 --- a/letstest/scripts/test_apache2.sh +++ b/letstest/scripts/test_apache2.sh @@ -12,25 +12,6 @@ then # For apache 2.4, set up ServerName sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE - if [ $(python3 -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -lt 36 ] - then - # Upgrade python version using pyenv because py3.5 is deprecated - # Don't upgrade if it's already 3.8 because pyenv doesn't work great on arm, and - # our arm representative happens to be ubuntu20, which already has a perfectly - # good version of python. - sudo apt-get install -y make gcc build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev python-openssl git # pyenv deps - curl https://pyenv.run | bash - export PATH="~/.pyenv/bin:$PATH" - pyenv init - - pyenv virtualenv-init - - pyenv install 3.8.5 - pyenv global 3.8.5 - # you do, in fact need to run these again, exactly like this. - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - fi elif [ "$OS_TYPE" = "centos" ] then CONFFILE=/etc/httpd/conf/httpd.conf diff --git a/letstest/targets/apache2_targets.yaml b/letstest/targets/apache2_targets.yaml index 2663782ce..c0c08be12 100644 --- a/letstest/targets/apache2_targets.yaml +++ b/letstest/targets/apache2_targets.yaml @@ -21,11 +21,6 @@ targets: type: ubuntu virt: hvm user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu #----------------------------------------------------------------------------- # Debian - ami: ami-01db78123b2b99496 @@ -33,11 +28,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin #----------------------------------------------------------------------------- # CentOS - ami: ami-9887c6e7 From 466e437a2065cb117a32bcaa9a875421718ef20d Mon Sep 17 00:00:00 2001 From: Mads Jensen Date: Fri, 14 May 2021 20:43:58 +0200 Subject: [PATCH 94/99] Use new GitHub templates. Add funding link (#8845) --- .github/FUNDING.yml | 1 + ISSUE_TEMPLATE.md => .github/issue_template.md | 0 pull_request_template.md => .github/pull_request_template.md | 0 3 files changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml rename ISSUE_TEMPLATE.md => .github/issue_template.md (100%) rename pull_request_template.md => .github/pull_request_template.md (100%) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..f02dc2086 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://supporters.eff.org/donate/support-work-on-certbot diff --git a/ISSUE_TEMPLATE.md b/.github/issue_template.md similarity index 100% rename from ISSUE_TEMPLATE.md rename to .github/issue_template.md diff --git a/pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from pull_request_template.md rename to .github/pull_request_template.md From bc23e07ee5c144aa9b1d5cf9e0ba303d9e91a695 Mon Sep 17 00:00:00 2001 From: Thomas G Date: Sun, 16 May 2021 15:03:53 +0200 Subject: [PATCH 95/99] Fix incompatibility with lexicon >= v3.6.0 (#8819) --- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-linode/local-oldest-requirements.txt | 1 - certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/local-oldest-requirements.txt | 1 - certbot-dns-ovh/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot/CHANGELOG.md | 4 ++-- certbot/certbot/plugins/dns_common_lexicon.py | 4 ++-- certbot/certbot/plugins/dns_test_common_lexicon.py | 6 +++--- tools/oldest_constraints.txt | 6 +++--- 15 files changed, 19 insertions(+), 21 deletions(-) diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 20b499327..153e373bd 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -9,7 +9,7 @@ version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index c0da63d71..70181ae78 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -32,7 +32,7 @@ if os.environ.get('SNAP_BUILD'): # which allows us to potentially upgrade our packages in these distros # as necessary. if os.environ.get('CERTBOT_OLDEST') == '1': - install_requires.append('dns-lexicon>=2.2.1') + install_requires.append('dns-lexicon>=3.1.0') # Changed parameter name else: install_requires.append('dns-lexicon>=3.2.1') diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 2feae0bd1..852eac606 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -9,7 +9,7 @@ version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index d24f6b309..85058ab90 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -8,7 +8,7 @@ version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.1.22', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index a8bd7449a..1307698d4 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,4 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.31.0 certbot[dev]==1.1.0 -dns-lexicon==2.2.3 diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 72c7ee2fd..293dc25c3 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -8,7 +8,7 @@ version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.2.3', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b2c54779f..9ecacdd26 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -9,7 +9,7 @@ version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index a78b6c3b7..cf88a118a 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -9,7 +9,7 @@ version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index c55e0d570..1307698d4 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,4 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.31.0 certbot[dev]==1.1.0 -dns-lexicon==2.7.14 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 4831fa480..e4dd4d712 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -9,7 +9,7 @@ version = '1.16.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 7b76bb324..353111ec2 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -8,7 +8,7 @@ version = '1.16.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'dns-lexicon>=2.1.23', + 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index fd2f0ffb4..e98ccc0af 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -10,11 +10,11 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed -* +* DNS plugins based on lexicon now require dns-lexicon >= v3.1.0 ### Fixed -* +* Fix TypeError due to incompatibility with lexicon >= v3.6.0 More details about these changes can be found on our GitHub repo. diff --git a/certbot/certbot/plugins/dns_common_lexicon.py b/certbot/certbot/plugins/dns_common_lexicon.py index a4d46587e..32213999d 100644 --- a/certbot/certbot/plugins/dns_common_lexicon.py +++ b/certbot/certbot/plugins/dns_common_lexicon.py @@ -45,7 +45,7 @@ class LexiconClient: self._find_domain_id(domain) try: - self.provider.create_record(type='TXT', name=record_name, content=record_content) + self.provider.create_record(rtype='TXT', name=record_name, content=record_content) except RequestException as e: logger.debug('Encountered error adding TXT record: %s', e, exc_info=True) raise errors.PluginError('Error adding TXT record: {0}'.format(e)) @@ -67,7 +67,7 @@ class LexiconClient: return try: - self.provider.delete_record(type='TXT', name=record_name, content=record_content) + self.provider.delete_record(rtype='TXT', name=record_name, content=record_content) except RequestException as e: logger.debug('Encountered error deleting TXT record: %s', e, exc_info=True) diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index 5c6f09d20..203adf009 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -94,7 +94,7 @@ class BaseLexiconClientTest: def test_add_txt_record(self: _LexiconAwareTestCase): self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.create_record.assert_called_with(type='TXT', + self.provider_mock.create_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) @@ -103,7 +103,7 @@ class BaseLexiconClientTest: self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.create_record.assert_called_with(type='TXT', + self.provider_mock.create_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) @@ -147,7 +147,7 @@ class BaseLexiconClientTest: def test_del_txt_record(self: _LexiconAwareTestCase): self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) - self.provider_mock.delete_record.assert_called_with(type='TXT', + self.provider_mock.delete_record.assert_called_with(rtype='TXT', name=self.record_name, content=self.record_content) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index f6528f396..c0f114639 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -59,9 +59,6 @@ zope.hookable==4.0.4 # Ubuntu Bionic constraints. cryptography==2.1.4 distro==1.0.1 -# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins -# using their local-oldest-requirements.txt -dns-lexicon==2.2.1 httplib2==0.9.2 idna==2.6 setuptools==39.0.1 @@ -77,3 +74,6 @@ parsedatetime==2.4 # Tracking at https://github.com/certbot/certbot/issues/6473 boto3==1.4.7 botocore==1.7.41 +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==3.1.0 From 5040495741ee924052eb2da3f43e55970a4e1200 Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Sun, 16 May 2021 15:17:41 +0200 Subject: [PATCH 96/99] Use UTF-8 for renewal configuration file encoding (#8789) --- certbot/CHANGELOG.md | 1 + certbot/certbot/_internal/main.py | 4 +++- certbot/certbot/_internal/storage.py | 19 ++++++++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index e98ccc0af..de6ca2039 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -46,6 +46,7 @@ More details about these changes can be found on our GitHub repo. * The module `acme.magic_typing` is deprecated and will be removed in a future release. Please use the built-in module `typing` instead. * The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. +* Use UTF-8 encoding for renewal configuration files ### Fixed diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index aed265ba3..d7639691e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -616,7 +616,9 @@ def _delete_if_appropriate(config): # don't delete if the archive_dir is used by some other lineage archive_dir = storage.full_archive_path( - configobj.ConfigObj(storage.renewal_file_for_certname(config, config.certname)), + configobj.ConfigObj( + storage.renewal_file_for_certname(config, config.certname), + encoding='utf-8', default_encoding='utf-8'), config, config.certname) try: cert_manager.match_and_check_overlaps(config, [lambda x: archive_dir], diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 11dae33e9..4551356d5 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -67,13 +67,16 @@ def cert_path_for_cert_name(config: interfaces.IConfig, cert_name: str) -> str: """ cert_name_implied_conf = renewal_file_for_certname(config, cert_name) - return configobj.ConfigObj(cert_name_implied_conf)["fullchain"] + return configobj.ConfigObj( + cert_name_implied_conf, encoding='utf-8', default_encoding='utf-8')["fullchain"] def config_with_defaults(config=None): """Merge supplied config, if provided, on top of builtin defaults.""" - defaults_copy = configobj.ConfigObj(constants.RENEWER_DEFAULTS) - defaults_copy.merge(config if config is not None else configobj.ConfigObj()) + defaults_copy = configobj.ConfigObj( + constants.RENEWER_DEFAULTS, encoding='utf-8', default_encoding='utf-8') + defaults_copy.merge(config if config is not None else configobj.ConfigObj( + encoding='utf-8', default_encoding='utf-8')) return defaults_copy @@ -114,7 +117,7 @@ def write_renewal_config(o_filename, n_filename, archive_dir, target, relevant_d :rtype: configobj.ConfigObj """ - config = configobj.ConfigObj(o_filename) + config = configobj.ConfigObj(o_filename, encoding='utf-8', default_encoding='utf-8') config["version"] = certbot.__version__ config["archive_dir"] = archive_dir for kind in ALL_FOUR: @@ -196,7 +199,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): write_renewal_config(config_filename, temp_filename, archive_dir, target, values) filesystem.replace(temp_filename, config_filename) - return configobj.ConfigObj(config_filename) + return configobj.ConfigObj(config_filename, encoding='utf-8', default_encoding='utf-8') def get_link_target(link): @@ -324,7 +327,8 @@ def delete_files(config, certname): full_default_archive_dir = full_archive_path(None, config, certname) full_default_live_dir = _full_live_path(config, certname) try: - renewal_config = configobj.ConfigObj(renewal_filename) + renewal_config = configobj.ConfigObj( + renewal_filename, encoding='utf-8', default_encoding='utf-8') except configobj.ConfigObjError: # config is corrupted logger.warning("Could not parse %s. You may wish to manually " @@ -434,7 +438,8 @@ class RenewableCert(interfaces.RenewableCert): # systemwide renewal configuration; self.configfile should be # used to make and save changes. try: - self.configfile = configobj.ConfigObj(config_filename) + self.configfile = configobj.ConfigObj( + config_filename, encoding='utf-8', default_encoding='utf-8') except configobj.ConfigObjError: raise errors.CertStorageError( "error parsing {0}".format(config_filename)) From 352ee258b70290409a8a97494d35a29758bd85b4 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Mon, 17 May 2021 19:22:47 +0200 Subject: [PATCH 97/99] [Windows] Cleanup Certbot pkg dir before installing to avoid dependencies conflicts (#8836) Fixes #8824 This PR makes the installer first delete (if exist) the previous `pkg` directory in the Certbot installation in order to avoid dependencies conflicts when a new version of Certbot (with new versions of dependencies) is intaller other an existing one. I took the simplest approach here, which is to delete specifically the directories known to create conflicts, instead of more complex approaches that involve to factor in some way the complete uninstaller logic. This is because the complexity added without a clear improvement does not worth it in my opinion. More specifically: * factorizing in some way the uninstaller section in the NSIS template make the installer use any potential new logic of a new installation of Certbot instead of the one applying for the current installation, and may create unexpected errors during installation or at runtime * calling the existing `uninstaller.exe` would be better, but I could not find a proper way to let NSIS wait for the actual end of the uninstall logic, and again may create unexpected errors during installation or at runtime * Cleanup Certbot pkg dir before installing to avoid dependencies conflicts * Add a changelog Co-authored-by: Brad Warren --- certbot/CHANGELOG.md | 2 ++ windows-installer/assets/template.nsi | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index de6ca2039..f9589d24e 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * DNS plugins based on lexicon now require dns-lexicon >= v3.1.0 +* Windows installer now cleans up old Certbot dependency packages + before installing the new ones to avoid version conflicts. ### Fixed diff --git a/windows-installer/assets/template.nsi b/windows-installer/assets/template.nsi index 566e1b004..5c551729a 100644 --- a/windows-installer/assets/template.nsi +++ b/windows-installer/assets/template.nsi @@ -83,6 +83,11 @@ SectionEnd [% block sections %] Section "!${PRODUCT_NAME}" sec_app + ; CERTBOT CUSTOM BEGIN + ; Try to cleanup Certbot pkg directory to avoid dependencies conflicts + RMDir /r "$INSTDIR\pkgs" + ; CERTBOT CUSTOM END + SetRegView [[ib.py_bitness]] SectionIn RO File ${PRODUCT_ICON} From 9e6b406218b32a949e0db845803227a2a2b5300d Mon Sep 17 00:00:00 2001 From: osirisinferi Date: Mon, 17 May 2021 20:04:05 +0200 Subject: [PATCH 98/99] Move 5040495 CHANGELOG.md entry to correct version (#8851) The merge of #8789 left the CHANGELOG.md entry at a previous certbot release. This PR puts the entry at the correct certbot version. --- certbot/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index f9589d24e..0220f1039 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -11,6 +11,7 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Changed * DNS plugins based on lexicon now require dns-lexicon >= v3.1.0 +* Use UTF-8 encoding for renewal configuration files * Windows installer now cleans up old Certbot dependency packages before installing the new ones to avoid version conflicts. @@ -48,7 +49,6 @@ More details about these changes can be found on our GitHub repo. * The module `acme.magic_typing` is deprecated and will be removed in a future release. Please use the built-in module `typing` instead. * The DigitalOcean plugin now creates TXT records for the DNS-01 challenge with a lower 30s TTL. -* Use UTF-8 encoding for renewal configuration files ### Fixed From 2df279bc5bb30a38e4f9c1c5e44a7b327c56743c Mon Sep 17 00:00:00 2001 From: alexzorin Date: Tue, 18 May 2021 09:39:04 +1000 Subject: [PATCH 99/99] cli: dont double-print choosing plugins error (#8850) --- certbot/certbot/_internal/main.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index d7639691e..70da95d20 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -1236,12 +1236,8 @@ def renew_cert(config, plugins, lineage): :raises errors.PluginSelectionError: MissingCommandlineFlag if supplied parameters do not pass """ - try: - # installers are used in auth mode to determine domain names - installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: - logger.info("Could not choose appropriate plugin: %s", e) - raise + # installers are used in auth mode to determine domain names + installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") le_client = _init_le_client(config, auth, installer) renewed_lineage = _get_and_save_cert(le_client, config, lineage=lineage) @@ -1279,12 +1275,8 @@ def certonly(config, plugins): """ # SETUP: Select plugins and construct a client instance - try: - # installers are used in auth mode to determine domain names - installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") - except errors.PluginSelectionError as e: - logger.info("Could not choose appropriate plugin: %s", e) - raise + # installers are used in auth mode to determine domain names + installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly") le_client = _init_le_client(config, auth, installer)