diff --git a/.travis.yml b/.travis.yml index 577dcbc40..22bde836e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,13 +78,33 @@ matrix: env: TOXENV=apacheconftest sudo: required - python: "3.3" - env: TOXENV=py33 + env: TOXENV=py33 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.4" - env: TOXENV=py34 + env: TOXENV=py34 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.5" - env: TOXENV=py35 + env: TOXENV=py35 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "3.6" - env: TOXENV=py36 + env: TOXENV=py36 BOULDER_INTEGRATION=1 + sudo: required + after_failure: + - sudo cat /var/log/mysql/error.log + - ps aux | grep mysql + services: docker - python: "2.7" env: TOXENV=nginxroundtrip diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..a8ee45024 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,427 @@ +# Certbot change log + +Certbot adheres to [Semantic Versioning](http://semver.org/). + +## 0.13.0 - 2017-04-06 + +### Added + +* `--debug-challenges` now pauses Certbot after setting up challenges for debugging. +* The Nginx parser can now handle all valid directives in configuration files. +* Nginx ciphersuites have changed to Mozilla Intermediate. +* `certbot-auto --no-bootstrap` provides the option to not install OS dependencies. + +### Fixed + +* `--register-unsafely-without-email` now respects `--quiet`. +* Hyphenated renewal parameters are now saved in renewal config files. +* `--dry-run` no longer persists keys and csrs. +* Certbot no longer hangs when trying to start Nginx in Arch Linux. +* Apache rewrite rules no longer double-encode characters. + +A full list of changes is available on GitHub: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.13.0%20is%3Aclosed%20 + +## 0.12.0 - 2017-03-02 + +### Added + +* Certbot now allows non-camelcase Apache VirtualHost names. +* Certbot now allows more log messages to be silenced. + +### Fixed + +* Fixed a regression around using `--cert-name` when getting new certificates + +More information about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.12.0 + +## 0.11.1 - 2017-02-01 + +### Fixed + +* Resolved a problem where Certbot would crash while parsing command line +arguments in some cases. +* Fixed a typo. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Aclosed + +## 0.11.0 - 2017-02-01 + +### Added + +* When using the standalone plugin while running Certbot interactively +and a required port is bound by another process, Certbot will give you +the option to retry to grab the port rather than immediately exiting. +* You are now able to deactivate your account with the Let's Encrypt +server using the `unregister` subcommand. +* When revoking a certificate using the `revoke` subcommand, you now +have the option to provide the reason the certificate is being revoked +to Let's Encrypt with `--reason`. + +### Changed + +* Providing `--quiet` to `certbot-auto` now silences package manager output. + +### Removed + +* Removed the optional `dnspython` dependency in our `acme` package. +Now the library does not support client side verification of the DNS +challenge. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.11.0+is%3Aclosed + +## 0.10.2 - 2017-01-25 + +### Added + +* If Certbot receives a request with a `badNonce` error, it now +automatically retries the request. Since nonces from Let's Encrypt expire, +this helps people performing the DNS challenge with the `manual` plugin +who may have to wait an extended period of time for their DNS changes to +propagate. + +### Fixed + +* Certbot now saves the `--preferred-challenges` values for renewal. Previously +these values were discarded causing a different challenge type to be used when +renewing certs in some cases. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.2+is%3Aclosed + +## 0.10.1 - 2017-01-13 + +### Fixed + +* Resolve problems where when asking Certbot to update a certificate at +an existing path to include different domain names, the old names would +continue to be used. +* Fix issues successfully running our unit test suite on some systems. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.1+is%3Aclosed + +## 0.10.0 - 2017-01-11 + +## Added + +* Added the ability to customize and automatically complete DNS and HTTP +domain validation challenges with the manual plugin. The flags +`--manual-auth-hook` and `--manual-cleanup-hook` can now be provided +when using the manual plugin to execute commands provided by the user to +perform and clean up challenges provided by the CA. This is best used in +complicated setups where the DNS challenge must be used or Certbot's +existing plugins cannot be used to perform HTTP challenges. For more +information on how this works, see `certbot --help manual`. +* Added a `--cert-name` flag for specifying the name to use for the +certificate in Certbot's configuration directory. Using this flag in +combination with `-d/--domains`, a user can easily request a new +certificate with different domains and save it with the name provided by +`--cert-name`. Additionally, `--cert-name` can be used to select a +certificate with the `certonly` and `run` subcommands so a full list of +domains in the certificate does not have to be provided. +* Added subcommand `certificates` for listing the certificates managed by +Certbot and their properties. +* Added the `delete` subcommand for removing certificates managed by Certbot +from the configuration directory. +* Certbot now supports requesting internationalized domain names (IDNs). +* Hooks provided to Certbot are now saved to be reused during renewal. +If you run Certbot with `--pre-hook`, `--renew-hook`, or `--post-hook` +flags when obtaining a certificate, the provided commands will +automatically be saved and executed again when renewing the certificate. +A pre-hook and/or post-hook can also be given to the `certbot renew` +command either on the command line or in a [configuration +file](https://certbot.eff.org/docs/using.html#configuration-file) to run +an additional command before/after any certificate is renewed. Hooks +will only be run if a certificate is renewed. +* Support Busybox in certbot-auto. + +### Changed + +* Recategorized `-h/--help` output to improve documentation and +discoverability. + +### Removed + +* Removed the ncurses interface. This change solves problems people +were having on many systems, reduces the number of Certbot +dependencies, and simplifies our code. Certbot's only interface now is +the text interface which was available by providing `-t/--text` to +earlier versions of Certbot. + +### Fixed + +* Many small bug fixes. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.10.0is%3Aclosed + +## 0.9.3 - 2016-10-13 + +### Added + +* The Apache plugin uses information about your OS to help determine the +layout of your Apache configuration directory. We added a patch to +ensure this code behaves the same way when testing on different systems +as the tests were failing in some cases. + +### Changed + +* Certbot adopted more conservative behavior about reporting a needed port as +unavailable when using the standalone plugin. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/27?closed=1 + +## 0.9.2 - 2016-10-12 + +### Added + +* Certbot stopped requiring that all possibly required ports are available when +using the standalone plugin. It now only verifies that the ports are available +when they are necessary. + +### Fixed + +* Certbot now verifies that our optional dependencies version matches what is +required by Certbot. +* Certnot now properly copies the `ssl on;` directives as necessary when +performing domain validation in the Nginx plugin. +* Fixed problem where symlinks were becoming files when they were +packaged, causing errors during testing and OS packaging. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/26?closed=1 + +## 0.9.1 - 2016-10-06 + +### Fixed + +* Fixed a bug that was introduced in version 0.9.0 where the command +line flag -q/--quiet wasn't respected in some cases. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/25?closed=1 + +## 0.9.0 - 2016-10-05 + +### Added + +* Added an alpha version of the Nginx plugin. This plugin fully automates the +process of obtaining and installing certificates with Nginx. +Additionally, it is able to automatically configure security +enhancements such as an HTTP to HTTPS redirect and OCSP stapling. To use +this plugin, you must have the `certbot-nginx` package installed (which +is installed automatically when using `certbot-auto`) and provide +`--nginx` on the command line. This plugin is still in its early stages +so we recommend you use it with some caution and make sure you have a +backup of your Nginx configuration. +* Added support for the `DNS` challenge in the `acme` library and `DNS` in +Certbot's `manual` plugin. This allows you to create DNS records to +prove to Let's Encrypt you control the requested domain name. To use +this feature, include `--manual --preferred-challenges dns` on the +command line. +* Certbot now helps with enabling Extra Packages for Enterprise Linux (EPEL) on +CentOS 6 when using `certbot-auto`. To use `certbot-auto` on CentOS 6, +the EPEL repository has to be enabled. `certbot-auto` will now prompt +users asking them if they would like the script to enable this for them +automatically. This is done without prompting users when using +`letsencrypt-auto` or if `-n/--non-interactive/--noninteractive` is +included on the command line. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.9.0+is%3Aclosed + +## 0.8.1 - 2016-06-14 + +### Added + +* Certbot now preserves a certificate's common name when using `renew`. +* Certbot now saves webroot values for renewal when they are entered interactively. +* Certbot now gracefully reports that the Apache plugin isn't usable when Augeas is not installed. +* Added experimental support for Mageia has been added to `certbot-auto`. + +### Fixed + +* Fixed problems with an invalid user-agent string on OS X. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.1+ + +## 0.8.0 - 2016-06-02 + +### Added + +* Added the `register` subcommand which can be used to register an account +with the Let's Encrypt CA. +* You can now run `certbot register --update-registration` to +change the e-mail address associated with your registration. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.8.0+ + +## 0.7.0 - 2016-05-27 + +### Added + +* Added `--must-staple` to request certificates from Let's Encrypt +with the OCSP must staple extension. +* Certbot now automatically configures OSCP stapling for Apache. +* Certbot now allows requesting certificates for domains found in the common name +of a custom CSR. + +### Fixed + +* Fixed a number of miscellaneous bugs + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=milestone%3A0.7.0+is%3Aissue + +## 0.6.0 - 2016-05-12 + +### Added + +* Versioned the datetime dependency in setup.py. + +### Changed + +* Renamed the client from `letsencrypt` to `certbot`. + +### Fixed + +* Fixed a small json deserialization error. +* Certbot now preserves domain order in generated CSRs. +* Fixed some minor bugs. + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue%20milestone%3A0.6.0%20is%3Aclosed%20 + +## 0.5.0 - 2016-04-05 + +### Added + +* Added the ability to use the webroot plugin interactively. +* Added the flags --pre-hook, --post-hook, and --renew-hook which can be used with +the renew subcommand to register shell commands to run in response to +renewal events. Pre-hook commands will be run before any certs are +renewed, post-hook commands will be run after any certs are renewed, +and renew-hook commands will be run after each cert is renewed. If no +certs are due for renewal, no command is run. +* Added a -q/--quiet flag which silences all output except errors. +* Added an --allow-subset-of-domains flag which can be used with the renew +command to prevent renewal failures for a subset of the requested +domains from causing the client to exit. + +### Changed + +* Certbot now uses renewal configuration files. In /etc/letsencrypt/renewal +by default, these files can be used to control what parameters are +used when renewing a specific certificate. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.5.0+is%3Aissue + +## 0.4.2 - 2016-03-03 + +### Fixed + +* Resolved problems encountered when compiling letsencrypt +against the new OpenSSL release. +* Fixed problems encountered when using `letsencrypt renew` with configuration files +from the private beta. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.2 + +## 0.4.1 - 2016-02-29 + +### Fixed + +* Fixed Apache parsing errors encountered with some configurations. +* Fixed Werkzeug dependency problems encountered on some Red Hat systems. +* Fixed bootstrapping failures when using letsencrypt-auto with --no-self-upgrade. +* Fixed problems with parsing renewal config files from private beta. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is:issue+milestone:0.4.1 + +## 0.4.0 - 2016-02-10 + +### Added + +* Added the verb/subcommand `renew` which can be used to renew your existing +certificates as they approach expiration. Running `letsencrypt renew` +will examine all existing certificate lineages and determine if any are +less than 30 days from expiration. If so, the client will use the +settings provided when you previously obtained the certificate to renew +it. The subcommand finishes by printing a summary of which renewals were +successful, failed, or not yet due. +* Added a `--dry-run` flag to help with testing configuration +without affecting production rate limits. Currently supported by the +`renew` and `certonly` subcommands, providing `--dry-run` on the command +line will obtain certificates from the staging server without saving the +resulting certificates to disk. +* Added major improvements to letsencrypt-auto. This script +has been rewritten to include full support for Python 2.6, the ability +for letsencrypt-auto to update itself, and improvements to the +stability, security, and performance of the script. +* Added support for Apache 2.2 to the Apache plugin. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.4.0 + +## 0.3.0 - 2016-01-27 + +### Added + +* Added a non-interactive mode which can be enabled by including `-n` or +`--non-interactive` on the command line. This can be used to guarantee +the client will not prompt when run automatically using cron/systemd. +* Added preparation for the new letsencrypt-auto script. Over the past +couple months, we've been working on increasing the reliability and +security of letsencrypt-auto. A number of changes landed in this +release to prepare for the new version of this script. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.3.0 + +## 0.2.0 - 2016-01-14 + +### Added + +* Added Apache plugin support for non-Debian based systems. Support has been +added for modern Red Hat based systems such as Fedora 23, Red Hat 7, +and CentOS 7 running Apache 2.4. In theory, this plugin should be +able to be configured to run on any Unix-like OS running Apache 2.4. +* Relaxed PyOpenSSL version requirements. This adds support for systems +with PyOpenSSL versions 0.13 or 0.14. +* Improved error messages from the client. + +### Fixed + +* Resolved issues with the Apache plugin enabling an HTTP to HTTPS +redirect on some systems. + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aissue+milestone%3A0.2.0 + +## 0.1.1 - 2015-12-15 + +### Added + +* Added a check that avoids attempting to issue for unqualified domain names like +"localhost". + +### Fixed + +* Fixed a confusing UI path that caused some users to repeatedly renew +their certs while experimenting with the client, in some cases hitting +issuance rate limits. +* Fixed numerous Apache configuration parser problems +* Fixed --webroot permission handling for non-root users + +More details about these changes can be found on our GitHub repo: +https://github.com/letsencrypt/letsencrypt/issues?q=milestone%3A0.1.1 diff --git a/Dockerfile b/Dockerfile index d42b632d4..11d53788b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,70 +1,26 @@ -# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297 -# it is more likely developers will already have ubuntu:trusty rather -# than e.g. debian:jessie and image size differences are negligible -FROM ubuntu:trusty -MAINTAINER Jakub Warmuz -MAINTAINER William Budington - -# Note: this only exposes the port to other docker containers. You -# still have to bind to 443@host at runtime, as per the ACME spec. -EXPOSE 443 - -# TODO: make sure --config-dir and --work-dir cannot be changed -# through the CLI (certbot-docker wrapper that uses standalone -# authenticator and text mode only?) -VOLUME /etc/letsencrypt /var/lib/letsencrypt - -WORKDIR /opt/certbot - -# no need to mkdir anything: -# https://docs.docker.com/reference/builder/#copy -# If doesn't exist, it is created along with all missing -# directories in its path. - -ENV DEBIAN_FRONTEND=noninteractive - -COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto -RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* - -# the above is not likely to change, so by putting it further up the -# Dockerfile we make sure we cache as much as possible - - -COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ - -# all above files are necessary for setup.py and venv setup, however, -# package source code directory has to be copied separately to a -# subdirectory... -# https://docs.docker.com/reference/builder/#copy: "If is a -# directory, the entire contents of the directory are copied, -# including filesystem metadata. Note: The directory itself is not -# copied, just its contents." Order again matters, three files are far -# more likely to be cached than the whole project directory - -COPY certbot /opt/certbot/src/certbot/ -COPY acme /opt/certbot/src/acme/ -COPY certbot-apache /opt/certbot/src/certbot-apache/ -COPY certbot-nginx /opt/certbot/src/certbot-nginx/ - - -RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv - -# PATH is set now so pipstrap upgrades the correct (v)env -ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \ - /opt/certbot/venv/bin/pip install \ - -e /opt/certbot/src/acme \ - -e /opt/certbot/src \ - -e /opt/certbot/src/certbot-apache \ - -e /opt/certbot/src/certbot-nginx - -# install in editable mode (-e) to save space: it's not possible to -# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); -# this might also help in debugging: you can "docker run --entrypoint -# bash" and investigate, apply patches, etc. +FROM python:2-alpine ENTRYPOINT [ "certbot" ] +EXPOSE 80 443 +VOLUME /etc/letsencrypt /var/lib/letsencrypt +WORKDIR /opt/certbot + +COPY CHANGES.rst README.rst setup.py src/ +COPY acme src/acme +COPY certbot src/certbot + +RUN apk add --no-cache --virtual .certbot-deps \ + libffi \ + libssl1.0 \ + ca-certificates \ + binutils +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + linux-headers \ + openssl-dev \ + musl-dev \ + libffi-dev \ + && pip install --no-cache-dir \ + --editable /opt/certbot/src/acme \ + --editable /opt/certbot/src \ + && apk del .build-deps diff --git a/Dockerfile-dev b/Dockerfile-dev index 098fae523..607aa3441 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -13,17 +13,17 @@ EXPOSE 443 # authenticator and text mode only?) VOLUME /etc/letsencrypt /var/lib/letsencrypt -WORKDIR /opt/certbot +WORKDIR /opt/certbot/src # no need to mkdir anything: # https://docs.docker.com/reference/builder/#copy # If doesn't exist, it is created along with all missing # directories in its path. -# TODO: Install non-default Python versions for tox. # TODO: Install Apache/Nginx for plugin development. COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ + apt-get install python3-dev -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ @@ -51,6 +51,8 @@ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tests /opt/certbot/src/tests/ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ + /opt/certbot/venv/bin/pip install -U pip && \ + /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install \ -e /opt/certbot/src/acme \ -e /opt/certbot/src \ diff --git a/Dockerfile-old b/Dockerfile-old new file mode 100644 index 000000000..7bce82e0c --- /dev/null +++ b/Dockerfile-old @@ -0,0 +1,75 @@ +# https://github.com/letsencrypt/letsencrypt/pull/431#issuecomment-103659297 +# it is more likely developers will already have ubuntu:trusty rather +# than e.g. debian:jessie and image size differences are negligible +FROM ubuntu:trusty +MAINTAINER Jakub Warmuz +MAINTAINER William Budington + +# Note: this only exposes the port to other docker containers. You +# still have to bind to 443@host at runtime, as per the ACME spec. +EXPOSE 443 + +# TODO: make sure --config-dir and --work-dir cannot be changed +# through the CLI (certbot-docker wrapper that uses standalone +# authenticator and text mode only?) +VOLUME /etc/letsencrypt /var/lib/letsencrypt + +WORKDIR /opt/certbot + +# no need to mkdir anything: +# https://docs.docker.com/reference/builder/#copy +# If doesn't exist, it is created along with all missing +# directories in its path. + +ENV DEBIAN_FRONTEND=noninteractive + +COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto +RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* + +# the above is not likely to change, so by putting it further up the +# Dockerfile we make sure we cache as much as possible + + +COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/ + +# all above files are necessary for setup.py and venv setup, however, +# package source code directory has to be copied separately to a +# subdirectory... +# https://docs.docker.com/reference/builder/#copy: "If is a +# directory, the entire contents of the directory are copied, +# including filesystem metadata. Note: The directory itself is not +# copied, just its contents." Order again matters, three files are far +# more likely to be cached than the whole project directory + +COPY certbot /opt/certbot/src/certbot/ +COPY acme /opt/certbot/src/acme/ +COPY certbot-apache /opt/certbot/src/certbot-apache/ +COPY certbot-nginx /opt/certbot/src/certbot-nginx/ + + +RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv + +# PATH is set now so pipstrap upgrades the correct (v)env +ENV PATH /opt/certbot/venv/bin:$PATH +RUN /opt/certbot/venv/bin/python /opt/certbot/src/pipstrap.py && \ + /opt/certbot/venv/bin/pip install \ + -e /opt/certbot/src/acme \ + -e /opt/certbot/src \ + -e /opt/certbot/src/certbot-apache \ + -e /opt/certbot/src/certbot-nginx + +# install in editable mode (-e) to save space: it's not possible to +# "rm -rf /opt/certbot/src" (it's stays in the underlaying image); +# this might also help in debugging: you can "docker run --entrypoint +# bash" and investigate, apply patches, etc. + +# set up certbot/letsencrypt wrapper to warn people about Dockerfile changes +COPY tools/docker-warning.sh /opt/certbot/bin/certbot +RUN ln -s /opt/certbot/bin/certbot /opt/certbot/bin/letsencrypt +ENV PATH /opt/certbot/bin:$PATH + +ENTRYPOINT [ "certbot" ] diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..e648a21b4 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,14 @@ +## My operating system is (include version): + + +## I installed Certbot with (certbot-auto, OS package manager, pip, etc): + + +## I ran this command and it produced this output: + + +## Certbot's behavior differed from what I expected because: + + +## Here is a Certbot log showing the issue (if available): +###### Logs are stored in `/var/log/letsencrypt` by default. Feel free to redact domains, e-mail and IP addresses as you see fit. diff --git a/README.rst b/README.rst index f986703ac..c51168216 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,12 @@ .. This file contains a series of comments that are used to include sections of this README in other files. Do not modify these comments unless you know what you are doing. tag:intro-begin -Certbot is part of EFF’s effort to encrypt the entire Internet. Secure communication over the Web relies on HTTPS, which requires the use of a digital certificate that lets browsers verify the identify of web servers (e.g., is that really google.com?). Web servers obtain their certificates from trusted third parties called certificate authorities (CAs). Certbot is an easy-to-use client that fetches a certificate from Let’s Encrypt—an open certificate authority launched by the EFF, Mozilla, and others—and deploys it to a web server. +Certbot is part of EFF’s effort to encrypt the entire Internet. Secure communication over the Web relies on HTTPS, which requires the use of a digital certificate that lets browsers verify the identity of web servers (e.g., is that really google.com?). Web servers obtain their certificates from trusted third parties called certificate authorities (CAs). Certbot is an easy-to-use client that fetches a certificate from Let’s Encrypt—an open certificate authority launched by the EFF, Mozilla, and others—and deploys it to a web server. Anyone who has gone through the trouble of setting up a secure website knows what a hassle getting and maintaining a certificate is. Certbot and Let’s Encrypt can automate away the pain and let you turn on and manage HTTPS with simple commands. Using Certbot and Let's Encrypt is free, so there’s no need to arrange payment. -How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. +How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide `_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access `_ to your web server to run Certbot. -If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issues by Let’s Encrypt. +If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt. Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME @@ -88,7 +88,7 @@ Main Website: https://certbot.eff.org Let's Encrypt Website: https://letsencrypt.org -IRC Channel: #letsencrypt on `Freenode`_ or #certbot on `OFTC`_ +IRC Channel: #letsencrypt on `Freenode`_ Community: https://community.letsencrypt.org @@ -129,19 +129,7 @@ email to client-dev+subscribe@letsencrypt.org) System Requirements =================== -The Let's Encrypt Client presently only runs on Unix-ish OSes that include -Python 2.6 or 2.7; Python 3.x support will hopefully be added in the future. The -client requires root access in order to write to ``/etc/letsencrypt``, -``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 -(if you use the ``standalone`` plugin) and to read and modify webserver -configurations (if you use the ``apache`` or ``nginx`` plugins). If none of -these apply to you, it is theoretically possible to run without root privileges, -but for most users who want to avoid running an ACME client as root, either -`letsencrypt-nosudo `_ or -`simp_le `_ are more appropriate choices. - -The Apache plugin currently requires a Debian-based OS with augeas version -1.0; this includes Ubuntu 12.04+ and Debian 7+. +See https://certbot.eff.org/docs/install.html#system-requirements. .. Do not modify this comment unless you know what you're doing. tag:intro-end diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 83b9b9edd..14641af10 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -5,7 +5,7 @@ import hashlib import logging import socket -from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import hashes # type: ignore import OpenSSL import requests @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) class Challenge(jose.TypedJSONObjectWithFields): # _fields_to_partial_json | pylint: disable=abstract-method """ACME challenge.""" - TYPES = {} + TYPES = {} # type: dict @classmethod def from_json(cls, jobj): @@ -37,7 +37,7 @@ class Challenge(jose.TypedJSONObjectWithFields): class ChallengeResponse(jose.TypedJSONObjectWithFields): # _fields_to_partial_json | pylint: disable=abstract-method """ACME challenge response.""" - TYPES = {} + TYPES = {} # type: dict resource_type = 'challenge' resource = fields.Resource(resource_type) @@ -425,7 +425,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): # TODO: domain is not necessary if host is provided if "host" not in kwargs: host = socket.gethostbyname(domain) - logging.debug('%s resolved to %s', domain, host) + logger.debug('%s resolved to %s', domain, host) kwargs["host"] = host kwargs.setdefault("port", self.PORT) @@ -445,7 +445,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse): """ # pylint: disable=protected-access sans = crypto_util._pyopenssl_cert_or_req_san(cert) - logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), sans) return self.z_domain.decode() in sans def simple_verify(self, chall, domain, account_public_key, diff --git a/acme/acme/client.py b/acme/acme/client.py index d01555c75..a069876d5 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -28,11 +28,13 @@ logger = logging.getLogger(__name__) # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning if sys.version_info < (2, 7, 9): # pragma: no cover try: - requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() + requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() # type: ignore except AttributeError: import urllib3.contrib.pyopenssl # pylint: disable=import-error urllib3.contrib.pyopenssl.inject_into_urllib3() +DEFAULT_NETWORK_TIMEOUT = 45 + DER_CONTENT_TYPE = 'application/pkix-cert' @@ -127,8 +129,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes update = regr.body if update is None else update body = messages.UpdateRegistration(**dict(update)) updated_regr = self._send_recv_regr(regr, body=body) - if updated_regr != regr: - raise errors.UnexpectedUpdate(regr) return updated_regr def deactivate_registration(self, regr): @@ -175,22 +175,25 @@ class Client(object): # pylint: disable=too-many-instance-attributes raise errors.UnexpectedUpdate(authzr) return authzr - def request_challenges(self, identifier): + def request_challenges(self, identifier, new_authzr_uri=None): """Request challenges. :param .messages.Identifier identifier: Identifier to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ + if new_authzr_uri is not None: + logger.debug("request_challenges with new_authzr_uri deprecated.") new_authz = messages.NewAuthorization(identifier=identifier) response = self.net.post(self.directory.new_authz, new_authz) # TODO: handle errors assert response.status_code == http_client.CREATED return self._authzr_from_response(response, identifier) - def request_domain_challenges(self, domain): + def request_domain_challenges(self, domain, new_authzr_uri=None): """Request challenges for domain names. This is simply a convenience function that wraps around @@ -199,13 +202,14 @@ class Client(object): # pylint: disable=too-many-instance-attributes documentation. :param str domain: Domain name to be challenged. + :param str new_authzr_uri: Deprecated. Do not use. :returns: Authorization Resource. :rtype: `.AuthorizationResource` """ return self.request_challenges(messages.Identifier( - typ=messages.IDENTIFIER_FQDN, value=domain)) + typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri) def answer_challenge(self, challb, response): """Answer challenge. @@ -281,7 +285,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes response = self.net.get(authzr.uri) updated_authzr = self._authzr_from_response( response, authzr.body.identifier, authzr.uri) - # TODO: check and raise UnexpectedUpdate return updated_authzr, response def request_issuance(self, csr, authzrs): @@ -357,14 +360,18 @@ class Client(object): # pylint: disable=too-many-instance-attributes # priority queue with datetime.datetime (based on Retry-After) as key, # and original Authorization Resource as value - waiting = [(datetime.datetime.now(), authzr) for authzr in authzrs] + waiting = [ + (datetime.datetime.now(), index, authzr) + for index, authzr in enumerate(authzrs) + ] + heapq.heapify(waiting) # mapping between original Authorization Resource and the most # recently updated one updated = dict((authzr, authzr) for authzr in authzrs) while waiting: # find the smallest Retry-After, and sleep if necessary - when, authzr = heapq.heappop(waiting) + when, index, authzr = heapq.heappop(waiting) now = datetime.datetime.now() if when > now: seconds = (when - now).seconds @@ -383,7 +390,7 @@ class Client(object): # pylint: disable=too-many-instance-attributes if attempts[authzr] < max_attempts: # push back to the priority queue, with updated retry_after heapq.heappush(waiting, (self.retry_after( - response, default=mintime), authzr)) + response, default=mintime), index, authzr)) else: exhausted.add(authzr) @@ -502,13 +509,14 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes REPLAY_NONCE_HEADER = 'Replay-Nonce' def __init__(self, key, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python'): + user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT): self.key = key self.alg = alg self.verify_ssl = verify_ssl self._nonces = set() self.user_agent = user_agent self.session = requests.Session() + self._default_timeout = timeout def __del__(self): self.session.close() @@ -600,13 +608,14 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes """ if method == "POST": - logging.debug('Sending POST request to %s:\n%s', + logger.debug('Sending POST request to %s:\n%s', url, kwargs['data']) else: - logging.debug('Sending %s request to %s.', method, url) + logger.debug('Sending %s request to %s.', method, url) kwargs['verify'] = self.verify_ssl kwargs.setdefault('headers', {}) kwargs['headers'].setdefault('User-Agent', self.user_agent) + kwargs.setdefault('timeout', self._default_timeout) response = self.session.request(method, url, *args, **kwargs) # If content is DER, log the base64 of it instead of raw bytes, to keep # binary data out of the logs. @@ -650,7 +659,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes def _get_nonce(self, url): if not self._nonces: - logging.debug('Requesting fresh nonce') + logger.debug('Requesting fresh nonce') self._add_nonce(self.head(url)) return self._nonces.pop() diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 0f4506203..09bb38c00 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -115,8 +115,6 @@ class ClientTest(unittest.TestCase): # TODO: split here and separate test self.response.json.return_value = self.regr.body.update( contact=()).to_json() - self.assertRaises( - errors.UnexpectedUpdate, self.client.update_registration, self.regr) def test_deactivate_account(self): self.response.headers['Location'] = self.regr.uri @@ -124,14 +122,6 @@ class ClientTest(unittest.TestCase): self.assertEqual(self.regr, self.client.deactivate_registration(self.regr)) - def test_deactivate_account_bad_registration_returned(self): - self.response.headers['Location'] = self.regr.uri - self.response.json.return_value = "some wrong registration thing" - self.assertRaises( - errors.UnexpectedUpdate, - self.client.deactivate_registration, - self.regr) - def test_query_registration(self): self.response.json.return_value = self.regr.body.to_json() self.assertEqual(self.regr, self.client.query_registration(self.regr)) @@ -154,6 +144,13 @@ class ClientTest(unittest.TestCase): self.directory.new_authz, messages.NewAuthorization(identifier=self.identifier)) + def test_request_challenges_deprecated_arg(self): + self._prepare_response_for_request_challenges() + self.client.request_challenges(self.identifier, new_authzr_uri="hi") + self.net.post.assert_called_once_with( + self.directory.new_authz, + messages.NewAuthorization(identifier=self.identifier)) + def test_request_challenges_custom_uri(self): self._prepare_response_for_request_challenges() self.client.request_challenges(self.identifier) @@ -532,7 +529,7 @@ class ClientNetworkTest(unittest.TestCase): 'HEAD', 'http://example.com/', 'foo', bar='baz')) self.net.session.request.assert_called_once_with( 'HEAD', 'http://example.com/', 'foo', - headers=mock.ANY, verify=mock.ANY, bar='baz') + headers=mock.ANY, verify=mock.ANY, timeout=mock.ANY, bar='baz') @mock.patch('acme.client.logger') def test_send_request_get_der(self, mock_logger): @@ -542,8 +539,9 @@ class ClientNetworkTest(unittest.TestCase): headers={"Content-Type": "application/pkix-cert"}, content=b"hi") # pylint: disable=protected-access - self.net._send_request('HEAD', 'http://example.com/', 'foo', bar='baz') - mock_logger.debug.assert_called_once_with( + self.net._send_request('HEAD', 'http://example.com/', 'foo', + timeout=mock.ANY, bar='baz') + mock_logger.debug.assert_called_with( 'Received response:\nHTTP %d\n%s\n\n%s', 200, 'Content-Type: application/pkix-cert', b'aGk=') @@ -555,7 +553,7 @@ class ClientNetworkTest(unittest.TestCase): 'POST', 'http://example.com/', 'foo', data='qux', bar='baz')) self.net.session.request.assert_called_once_with( 'POST', 'http://example.com/', 'foo', - headers=mock.ANY, verify=mock.ANY, data='qux', bar='baz') + headers=mock.ANY, verify=mock.ANY, timeout=mock.ANY, data='qux', bar='baz') def test_send_request_verify_ssl(self): # pylint: disable=protected-access @@ -568,7 +566,8 @@ class ClientNetworkTest(unittest.TestCase): self.response, self.net._send_request('GET', 'http://example.com/')) self.net.session.request.assert_called_once_with( - 'GET', 'http://example.com/', verify=verify, headers=mock.ANY) + 'GET', 'http://example.com/', verify=verify, + timeout=mock.ANY, headers=mock.ANY) def test_send_request_user_agent(self): self.net.session = mock.MagicMock() @@ -577,13 +576,23 @@ class ClientNetworkTest(unittest.TestCase): headers={'bar': 'baz'}) self.net.session.request.assert_called_once_with( 'GET', 'http://example.com/', verify=mock.ANY, + timeout=mock.ANY, headers={'User-Agent': 'acme-python-test', 'bar': 'baz'}) self.net._send_request('GET', 'http://example.com/', headers={'User-Agent': 'foo2'}) self.net.session.request.assert_called_with( 'GET', 'http://example.com/', - verify=mock.ANY, headers={'User-Agent': 'foo2'}) + verify=mock.ANY, timeout=mock.ANY, headers={'User-Agent': 'foo2'}) + + def test_send_request_timeout(self): + self.net.session = mock.MagicMock() + # pylint: disable=protected-access + self.net._send_request('GET', 'http://example.com/', + headers={'bar': 'baz'}) + self.net.session.request.assert_called_once_with( + mock.ANY, mock.ANY, verify=mock.ANY, headers=mock.ANY, + timeout=45) def test_del(self): sess = mock.MagicMock() diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 266f2c0c7..f86a9971a 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni # should be changed to use "set_options" to disable SSLv2 and SSLv3, # in case it's used for things other than probing/serving! -_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD +_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD # type: ignore class SSLSocket(object): # pylint: disable=too-few-public-methods @@ -149,6 +149,36 @@ def probe_sni(name, host, port=443, timeout=300, raise errors.Error(error) return client_ssl.get_peer_certificate() +def make_csr(private_key_pem, domains, must_staple=False): + """Generate a CSR containing a list of domains as subjectAltNames. + + :param buffer private_key_pem: Private key, in PEM PKCS#8 format. + :param list domains: List of DNS names to include in subjectAltNames of CSR. + :param bool must_staple: Whether to include the TLS Feature extension (aka + OCSP Must Staple: https://tools.ietf.org/html/rfc7633). + :returns: buffer PEM-encoded Certificate Signing Request. + """ + private_key = OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, private_key_pem) + csr = OpenSSL.crypto.X509Req() + extensions = [ + OpenSSL.crypto.X509Extension( + b'subjectAltName', + critical=False, + value=', '.join('DNS:' + d for d in domains).encode('ascii') + ), + ] + if must_staple: + extensions.append(OpenSSL.crypto.X509Extension( + b"1.3.6.1.5.5.7.1.24", + critical=False, + value=b"DER:30:03:02:01:05")) + csr.add_extensions(extensions) + csr.set_pubkey(private_key) + csr.set_version(2) + csr.sign(private_key, 'sha256') + return OpenSSL.crypto.dump_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr) def _pyopenssl_cert_or_req_san(cert_or_req): """Get Subject Alternative Names from certificate or CSR using pyOpenSSL. diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index ebb4010a6..845f43914 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -6,7 +6,7 @@ import time import unittest import six -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver #type: ignore # pylint: disable=import-error import OpenSSL @@ -151,6 +151,53 @@ class RandomSnTest(unittest.TestCase): self.serial_num.append(cert.get_serial_number()) self.assertTrue(len(set(self.serial_num)) > 1) +class MakeCSRTest(unittest.TestCase): + """Test for standalone functions.""" + + @classmethod + def _call_with_key(cls, *args, **kwargs): + privkey = OpenSSL.crypto.PKey() + privkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + privkey_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, privkey) + from acme.crypto_util import make_csr + return make_csr(privkey_pem, *args, **kwargs) + + def test_make_csr(self): + csr_pem = self._call_with_key(["a.example", "b.example"]) + self.assertTrue(b'--BEGIN CERTIFICATE REQUEST--' in csr_pem) + self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem) + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_pem) + # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr + # objects don't have a get_extensions() method, so we skip this test if + # the method isn't available. + if hasattr(csr, 'get_extensions'): + self.assertEquals(len(csr.get_extensions()), 1) + self.assertEquals(csr.get_extensions()[0].get_data(), + OpenSSL.crypto.X509Extension( + b'subjectAltName', + critical=False, + value=b'DNS:a.example, DNS:b.example', + ).get_data(), + ) + + def test_make_csr_must_staple(self): + csr_pem = self._call_with_key(["a.example"], must_staple=True) + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_pem) + + # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr + # objects don't have a get_extensions() method, so we skip this test if + # the method isn't available. + if hasattr(csr, 'get_extensions'): + self.assertEquals(len(csr.get_extensions()), 2) + # NOTE: Ideally we would filter by the TLS Feature OID, but + # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID, + # and the shortname field is just "UNDEF" + must_staple_exts = [e for e in csr.get_extensions() + if e.get_data() == b"0\x03\x02\x01\x05"] + self.assertEqual(len(must_staple_exts), 1, + "Expected exactly one Must Staple extension") if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/jose/jwa.py b/acme/acme/jose/jwa.py index 1853e0107..9b682ecab 100644 --- a/acme/acme/jose/jwa.py +++ b/acme/acme/jose/jwa.py @@ -9,9 +9,9 @@ import logging import cryptography.exceptions from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import hmac -from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives import hashes # type: ignore +from cryptography.hazmat.primitives import hmac # type: ignore +from cryptography.hazmat.primitives.asymmetric import padding # type: ignore from acme.jose import errors from acme.jose import interfaces @@ -28,9 +28,9 @@ class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method """JSON Web Algorithm.""" -class JWASignature(JWA, collections.Hashable): +class JWASignature(JWA, collections.Hashable): # type: ignore """JSON Web Signature Algorithm.""" - SIGNATURES = {} + SIGNATURES = {} # type: dict def __init__(self, name): self.name = name diff --git a/acme/acme/jose/jwk.py b/acme/acme/jose/jwk.py index 5b6965c4d..54423f670 100644 --- a/acme/acme/jose/jwk.py +++ b/acme/acme/jose/jwk.py @@ -6,9 +6,9 @@ import logging import cryptography.exceptions from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import hashes # type: ignore from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import ec # type: ignore from cryptography.hazmat.primitives.asymmetric import rsa import six @@ -25,8 +25,8 @@ class JWK(json_util.TypedJSONObjectWithFields): # pylint: disable=too-few-public-methods """JSON Web Key.""" type_field_name = 'kty' - TYPES = {} - cryptography_key_types = () + TYPES = {} # type: dict + cryptography_key_types = () # type: tuple """Subclasses should override.""" required = NotImplemented diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py index 9c14cf729..8fa8d7670 100644 --- a/acme/acme/jose/jws.py +++ b/acme/acme/jose/jws.py @@ -121,12 +121,12 @@ class Header(json_util.JSONObjectWithFields): # x5c does NOT use JOSE Base64 (4.1.6) - @x5c.encoder + @x5c.encoder # type: ignore def x5c(value): # pylint: disable=missing-docstring,no-self-argument return [base64.b64encode(OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_ASN1, cert.wrapped)) for cert in value] - @x5c.decoder + @x5c.decoder # type: ignore def x5c(value): # pylint: disable=missing-docstring,no-self-argument try: return tuple(util.ComparableX509(OpenSSL.crypto.load_certificate( @@ -157,12 +157,12 @@ class Signature(json_util.JSONObjectWithFields): 'signature', decoder=json_util.decode_b64jose, encoder=json_util.encode_b64jose) - @protected.encoder + @protected.encoder # type: ignore def protected(value): # pylint: disable=missing-docstring,no-self-argument # wrong type guess (Signature, not bytes) | pylint: disable=no-member return json_util.encode_b64jose(value.encode('utf-8')) - @protected.decoder + @protected.decoder # type: ignore def protected(value): # pylint: disable=missing-docstring,no-self-argument return json_util.decode_b64jose(value).decode('utf-8') diff --git a/acme/acme/jose/util.py b/acme/acme/jose/util.py index 6be9a6602..26b7e0c5a 100644 --- a/acme/acme/jose/util.py +++ b/acme/acme/jose/util.py @@ -134,7 +134,7 @@ class ComparableRSAKey(ComparableKey): # pylint: disable=too-few-public-methods return hash((self.__class__, pub.n, pub.e)) -class ImmutableMap(collections.Mapping, collections.Hashable): +class ImmutableMap(collections.Mapping, collections.Hashable): # type: ignore # pylint: disable=too-few-public-methods """Immutable key to value mapping with attribute access.""" @@ -180,7 +180,7 @@ class ImmutableMap(collections.Mapping, collections.Hashable): for key, value in six.iteritems(self))) -class frozendict(collections.Mapping, collections.Hashable): +class frozendict(collections.Mapping, collections.Hashable): # type: ignore # pylint: disable=invalid-name,too-few-public-methods """Frozen dictionary.""" __slots__ = ('_items', '_keys') diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 54bc26d94..79e96edcb 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -1,14 +1,18 @@ -"""ACME JOSE JWS.""" +"""ACME-specific JWS. + +The JWS implementation in acme.jose only implements the base JOSE standard. In +order to support the new header fields defined in ACME, this module defines some +ACME-specific classes that layer on top of acme.jose. +""" from acme import jose class Header(jose.Header): - """ACME JOSE Header. - - .. todo:: Implement ``acmePath``. - + """ACME-specific JOSE Header. Implements nonce, kid, and url. """ nonce = jose.Field('nonce', omitempty=True, encoder=jose.encode_b64jose) + kid = jose.Field('kid', omitempty=True) + url = jose.Field('url', omitempty=True) @nonce.decoder def nonce(value): # pylint: disable=missing-docstring,no-self-argument @@ -20,7 +24,7 @@ class Header(jose.Header): class Signature(jose.Signature): - """ACME Signature.""" + """ACME-specific Signature. Uses ACME-specific Header for customer fields.""" __slots__ = jose.Signature._orig_slots # pylint: disable=no-member # TODO: decoder/encoder should accept cls? Otherwise, subclassing @@ -34,11 +38,17 @@ class Signature(jose.Signature): class JWS(jose.JWS): - """ACME JWS.""" + """ACME-specific JWS. Includes none, url, and kid in protected header.""" signature_cls = Signature __slots__ = jose.JWS._orig_slots # pylint: disable=no-member @classmethod - def sign(cls, payload, key, alg, nonce): # pylint: disable=arguments-differ + # pylint: disable=arguments-differ,too-many-arguments + def sign(cls, payload, key, alg, nonce, url=None, kid=None): + # Per ACME spec, jwk and kid are mutually exclusive, so only include a + # jwk field if kid is not provided. + include_jwk = kid is None return super(JWS, cls).sign(payload, key=key, alg=alg, - protect=frozenset(['nonce']), nonce=nonce) + protect=frozenset(['nonce', 'url', 'kid']), + nonce=nonce, url=url, kid=kid, + include_jwk=include_jwk) diff --git a/acme/acme/jws_test.py b/acme/acme/jws_test.py index e8f8e871a..62409227e 100644 --- a/acme/acme/jws_test.py +++ b/acme/acme/jws_test.py @@ -37,16 +37,30 @@ class JWSTest(unittest.TestCase): self.privkey = KEY self.pubkey = self.privkey.public_key() self.nonce = jose.b64encode(b'Nonce') + self.url = 'hi' + self.kid = 'baaaaa' - def test_it(self): + def test_kid_serialize(self): from acme.jws import JWS jws = JWS.sign(payload=b'foo', key=self.privkey, - alg=jose.RS256, nonce=self.nonce) + alg=jose.RS256, nonce=self.nonce, + url=self.url, kid=self.kid) self.assertEqual(jws.signature.combined.nonce, self.nonce) + self.assertEqual(jws.signature.combined.url, self.url) + self.assertEqual(jws.signature.combined.kid, self.kid) + self.assertEqual(jws.signature.combined.jwk, None) # TODO: check that nonce is in protected header self.assertEqual(jws, JWS.from_json(jws.to_json())) + def test_jwk_serialize(self): + from acme.jws import JWS + jws = JWS.sign(payload=b'foo', key=self.privkey, + alg=jose.RS256, nonce=self.nonce, + url=self.url) + self.assertEqual(jws.signature.combined.kid, None) + self.assertEqual(jws.signature.combined.jwk, self.pubkey) + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/messages.py b/acme/acme/messages.py index c3df4998c..5784e8e11 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -98,7 +98,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None) -class _Constant(jose.JSONDeSerializable, collections.Hashable): +class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented @@ -132,7 +132,7 @@ class _Constant(jose.JSONDeSerializable, collections.Hashable): class Status(_Constant): """ACME "status" field.""" - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: dict STATUS_UNKNOWN = Status('unknown') STATUS_PENDING = Status('pending') STATUS_PROCESSING = Status('processing') @@ -143,7 +143,7 @@ STATUS_REVOKED = Status('revoked') class IdentifierType(_Constant): """ACME identifier type.""" - POSSIBLE_NAMES = {} + POSSIBLE_NAMES = {} # type: dict IDENTIFIER_FQDN = IdentifierType('dns') # IdentifierDNS in Boulder @@ -161,7 +161,7 @@ class Identifier(jose.JSONObjectWithFields): class Directory(jose.JSONDeSerializable): """Directory.""" - _REGISTERED_TYPES = {} + _REGISTERED_TYPES = {} # type: dict class Meta(jose.JSONObjectWithFields): """Directory Meta.""" @@ -237,10 +237,6 @@ class Registration(ResourceBody): :ivar tuple contact: Contact information following ACME spec, `tuple` of `unicode`. :ivar unicode agreement: - :ivar unicode authorizations: URI where - `messages.Registration.Authorizations` can be found. - :ivar unicode certificates: URI where - `messages.Registration.Certificates` can be found. """ # on new-reg key server ignores 'key' and populates it based on @@ -248,26 +244,8 @@ class Registration(ResourceBody): key = jose.Field('key', omitempty=True, decoder=jose.JWK.from_json) contact = jose.Field('contact', omitempty=True, default=()) agreement = jose.Field('agreement', omitempty=True) - authorizations = jose.Field('authorizations', omitempty=True) - certificates = jose.Field('certificates', omitempty=True) status = jose.Field('status', omitempty=True) - class Authorizations(jose.JSONObjectWithFields): - """Authorizations granted to Account in the process of registration. - - :ivar tuple authorizations: URIs to Authorization Resources. - - """ - authorizations = jose.Field('authorizations') - - class Certificates(jose.JSONObjectWithFields): - """Certificates granted to Account in the process of registration. - - :ivar tuple certificates: URIs to Certificate Resources. - - """ - certificates = jose.Field('certificates') - phone_prefix = 'tel:' email_prefix = 'mailto:' @@ -315,10 +293,12 @@ class RegistrationResource(ResourceWithURI): """Registration Resource. :ivar acme.messages.Registration body: + :ivar unicode new_authzr_uri: Deprecated. Do not use. :ivar unicode terms_of_service: URL for the CA TOS. """ body = jose.Field('body', decoder=Registration.from_json) + new_authzr_uri = jose.Field('new_authzr_uri', omitempty=True) terms_of_service = jose.Field('terms_of_service', omitempty=True) @@ -423,9 +403,11 @@ class AuthorizationResource(ResourceWithURI): """Authorization Resource. :ivar acme.messages.Authorization body: + :ivar unicode new_cert_uri: Deprecated. Do not use. """ body = jose.Field('body', decoder=Authorization.from_json) + new_cert_uri = jose.Field('new_cert_uri', omitempty=True) @Directory.register diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index e84c3e992..ab05a89b7 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -170,8 +170,7 @@ class RegistrationTest(unittest.TestCase): from acme.messages import Registration self.reg = Registration(key=key, contact=contact, agreement=agreement) - self.reg_none = Registration(authorizations='uri/authorizations', - certificates='uri/certificates') + self.reg_none = Registration() self.jobj_to = { 'contact': contact, diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 02cc2daf5..087240c15 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -6,9 +6,9 @@ import logging import os import sys -from six.moves import BaseHTTPServer # pylint: disable=import-error +from six.moves import BaseHTTPServer # type: ignore # pylint: disable=import-error from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver # type: ignore # pylint: disable=import-error import OpenSSL diff --git a/acme/acme/standalone_test.py b/acme/acme/standalone_test.py index 58469d470..613258c97 100644 --- a/acme/acme/standalone_test.py +++ b/acme/acme/standalone_test.py @@ -7,7 +7,7 @@ import time import unittest from six.moves import http_client # pylint: disable=import-error -from six.moves import socketserver # pylint: disable=import-error +from six.moves import socketserver # type: ignore # pylint: disable=import-error import requests diff --git a/acme/setup.py b/acme/setup.py index d1e91e5ec..a640ae6bb 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,35 +4,30 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ + 'argparse', # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', # Connection.set_tlsext_host_name (>=0.13) + 'mock', 'PyOpenSSL>=0.13', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + # requests>=2.10 is required to fix + # https://github.com/shazow/urllib3/issues/556. This requirement can be + # relaxed to 'requests[security]>=2.4.1', however, less useful errors + # will be raised for some network/SSL errors. + 'requests[security]>=2.10', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', 'six', ] -# env markers in extras_require cause problems with older pip: #517 -# Keep in sync with conditional_requirements.py. -if sys.version_info < (2, 7): - install_requires.extend([ - # only some distros recognize stdlib argparse as already satisfying - 'argparse', - 'mock<1.1.0', - ]) -else: - install_requires.append('mock') - dev_extras = [ 'nose', 'tox', diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index a956341eb..39d25619d 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -174,6 +174,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Set Version if self.version is None: self.version = self.get_version() + logger.debug('Apache version is %s', + '.'.join(str(i) for i in self.version)) if self.version < (2, 2): raise errors.NotSupportedError( "Apache Version %s not supported.", str(self.version)) @@ -254,9 +256,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): raise errors.PluginError( "Unable to find cert and/or key directives") - logger.info("Deploying Certificate to VirtualHost %s", vhost.filep) - logger.debug("Apache version is %s", - ".".join(str(i) for i in self.version)) + logger.info("Deploying Certificate for %s to VirtualHost %s", domain, vhost.filep) if self.version < (2, 4, 8) or (chain_path and not fullchain_path): # install SSLCertificateFile, SSLCertificateKeyFile, @@ -580,7 +580,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ("/files%s//*[label()=~regexp('%s')]" % (vhost_path, parser.case_i("VirtualHost")))) paths = [path for path in paths if - os.path.basename(path) == "VirtualHost"] + os.path.basename(path.lower()) == "virtualhost"] for path in paths: new_vhost = self._create_vhost(path) if not new_vhost: @@ -1315,18 +1315,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # even with save() and load() if not self._is_rewrite_engine_on(general_vh): self.parser.add_dir(general_vh.path, "RewriteEngine", "on") + names = ssl_vhost.get_names() for idx, name in enumerate(names): args = ["%{SERVER_NAME}", "={0}".format(name), "[OR]"] if idx == len(names) - 1: args.pop() self.parser.add_dir(general_vh.path, "RewriteCond", args) - if self.get_version() >= (2, 3, 9): - self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS_WITH_END) - else: - self.parser.add_dir(general_vh.path, "RewriteRule", - constants.REWRITE_HTTPS_ARGS) + + self._set_https_redirection_rewrite_rule(general_vh) self.save_notes += ("Redirecting host in %s to ssl vhost in %s\n" % (general_vh.filep, ssl_vhost.filep)) @@ -1336,12 +1333,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.info("Redirecting vhost in %s to ssl vhost in %s", general_vh.filep, ssl_vhost.filep) + def _set_https_redirection_rewrite_rule(self, vhost): + if self.get_version() >= (2, 3, 9): + self.parser.add_dir(vhost.path, "RewriteRule", + constants.REWRITE_HTTPS_ARGS_WITH_END) + else: + self.parser.add_dir(vhost.path, "RewriteRule", + constants.REWRITE_HTTPS_ARGS) + + def _verify_no_certbot_redirect(self, vhost): """Checks to see if a redirect was already installed by certbot. Checks to see if virtualhost already contains a rewrite rule that is identical to Certbot's redirection rewrite rule. + For graceful transition to new rewrite rules for HTTPS redireciton we + delete certbot's old rewrite rules and set the new one instead. + :param vhost: vhost to check :type vhost: :class:`~certbot_apache.obj.VirtualHost` @@ -1355,19 +1364,29 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # rewrite_args_dict keys are directive ids and the corresponding value # for each is a list of arguments to that directive. rewrite_args_dict = defaultdict(list) - pat = r'.*(directive\[\d+\]).*' + pat = r'(.*directive\[\d+\]).*' for match in rewrite_path: m = re.match(pat, match) if m: - dir_id = m.group(1) - rewrite_args_dict[dir_id].append(match) + dir_path = m.group(1) + rewrite_args_dict[dir_path].append(match) if rewrite_args_dict: redirect_args = [constants.REWRITE_HTTPS_ARGS, constants.REWRITE_HTTPS_ARGS_WITH_END] - for matches in rewrite_args_dict.values(): - if [self.aug.get(x) for x in matches] in redirect_args: + for dir_path, args_paths in rewrite_args_dict.items(): + arg_vals = [self.aug.get(x) for x in args_paths] + + # Search for past redirection rule, delete it, set the new one + if arg_vals in constants.OLD_REWRITE_HTTPS_ARGS: + self.aug.remove(dir_path) + self._set_https_redirection_rewrite_rule(vhost) + self.save() + raise errors.PluginEnhancementAlreadyPresent( + "Certbot has already enabled redirection") + + if arg_vals in redirect_args: raise errors.PluginEnhancementAlreadyPresent( "Certbot has already enabled redirection") diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index dcc635c4b..3cfeb4dd6 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -136,15 +136,19 @@ AUGEAS_LENS_DIR = pkg_resources.resource_filename( """Path to the Augeas lens directory""" REWRITE_HTTPS_ARGS = [ - "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,NE,R=permanent]"] """Apache version<2.3.9 rewrite rule arguments used for redirections to https vhost""" REWRITE_HTTPS_ARGS_WITH_END = [ - "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"] + "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,NE,R=permanent]"] """Apache version >= 2.3.9 rewrite rule arguments used for redirections to https vhost""" +OLD_REWRITE_HTTPS_ARGS = [ + ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"], + ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[END,QSA,R=permanent]"]] + HSTS_ARGS = ["always", "set", "Strict-Transport-Security", "\"max-age=31536000\""] """Apache header arguments for HSTS""" diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index b29b0e0ee..30cb24844 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -25,6 +25,11 @@ class Addr(common.Addr): def __repr__(self): return "certbot_apache.obj.Addr(" + repr(self.tup) + ")" + def __hash__(self): + # Python 3 requires explicit overridden for __hash__ if __eq__ or + # __cmp__ is overridden. See https://bugs.python.org/issue2235 + return super(Addr, self).__hash__() + def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" # pylint: disable=protected-access @@ -174,6 +179,11 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def __ne__(self, other): return not self.__eq__(other) + def __hash__(self): + return hash((self.filep, self.path, + tuple(self.addrs), tuple(self.get_names()), + self.ssl, self.enabled, self.modmacro)) + def conflicts(self, addrs): """See if vhost conflicts with any of the addrs. diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py index 6bb6ff170..67984a26c 100644 --- a/certbot-apache/certbot_apache/parser.py +++ b/certbot-apache/certbot_apache/parser.py @@ -1,10 +1,12 @@ """ApacheParser is a member object of the ApacheConfigurator class.""" import fnmatch -import itertools import logging import os import re import subprocess +import sys + +import six from certbot import errors @@ -87,7 +89,7 @@ class ApacheParser(object): while len(self.modules) != prev_size: prev_size = len(self.modules) - for match_name, match_filename in itertools.izip( + for match_name, match_filename in six.moves.zip( iterator, iterator): self.modules.add(self.get_arg(match_name)) self.modules.add( @@ -134,7 +136,8 @@ class ApacheParser(object): proc = subprocess.Popen( constants.os_constant("define_cmd"), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) stdout, stderr = proc.communicate() except (OSError, ValueError): @@ -460,8 +463,12 @@ class ApacheParser(object): :rtype: str """ - # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] + if sys.version_info < (3, 6): + # This strips off final /Z(?ms) + return fnmatch.translate(clean_fn_match)[:-7] + else: # pragma: no cover + # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z + return fnmatch.translate(clean_fn_match)[4:-3] def _parse_file(self, filepath): """Parse file with Augeas diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 01361c8f0..45e701bd5 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -6,6 +6,8 @@ import socket import unittest import mock +# six is used in mock.patch() +import six # pylint: disable=unused-import from acme import challenges @@ -16,6 +18,7 @@ from certbot.tests import acme_util from certbot.tests import util as certbot_util from certbot_apache import configurator +from certbot_apache import constants from certbot_apache import parser from certbot_apache import obj @@ -517,12 +520,12 @@ class MultipleVhostsTest(util.ApacheTest): # Test self.config.prepare_server_https("8080", temp=True) self.assertEqual(mock_add_dir.call_count, 3) - self.assertEqual(mock_add_dir.call_args_list[0][0][2], - ["1.2.3.4:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[1][0][2], - ["[::1]:8080", "https"]) - self.assertEqual(mock_add_dir.call_args_list[2][0][2], - ["1.1.1.1:8080", "https"]) + call_args_list = [mock_add_dir.call_args_list[i][0][2] for i in range(3)] + self.assertEqual( + sorted(call_args_list), + sorted([["1.2.3.4:8080", "https"], + ["[::1]:8080", "https"], + ["1.1.1.1:8080", "https"]])) # mock_get.side_effect = ["1.2.3.4:80", "[::1]:80"] # mock_find.return_value = ["test1", "test2", "test3"] @@ -662,7 +665,7 @@ class MultipleVhostsTest(util.ApacheTest): # This calls open self.config.reverter.register_file_creation = mock.Mock() mock_open.side_effect = IOError - with mock.patch("__builtin__.open", mock_open): + with mock.patch("six.moves.builtins.open", mock_open): self.assertRaises( errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) @@ -1045,6 +1048,36 @@ class MultipleVhostsTest(util.ApacheTest): self.assertTrue("rewrite_module" in self.config.parser.modules) + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_redirect_with_old_https_redirection(self, mock_exe, _): + self.config.parser.update_runtime_variables = mock.Mock() + mock_exe.return_value = True + self.config.get_version = mock.Mock(return_value=(2, 2, 0)) + + ssl_vhost = self.config.choose_vhost("certbot.demo") + + # pylint: disable=protected-access + http_vhost = self.config._get_http_vhost(ssl_vhost) + + # Create an old (previously suppoorted) https redirectoin rewrite rule + self.config.parser.add_dir( + http_vhost.path, "RewriteRule", + ["^", + "https://%{SERVER_NAME}%{REQUEST_URI}", + "[L,QSA,R=permanent]"]) + + self.config.save() + + try: + self.config.enhance("certbot.demo", "redirect") + except errors.PluginEnhancementAlreadyPresent: + args_paths = self.config.parser.find_dir( + "RewriteRule", None, http_vhost.path, False) + arg_vals = [self.config.aug.get(x) for x in args_paths] + self.assertEqual(arg_vals, constants.REWRITE_HTTPS_ARGS) + + def test_redirect_with_conflict(self): self.config.parser.modules.add("rewrite_module") ssl_vh = obj.VirtualHost( @@ -1132,7 +1165,7 @@ class MultipleVhostsTest(util.ApacheTest): http_vhost.path, "RewriteRule", ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,QSA,R=permanent]"]) + "[L,NE,R=permanent]"]) self.config.save() ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -1143,7 +1176,7 @@ class MultipleVhostsTest(util.ApacheTest): conf_text = open(ssl_vhost.filep).read() commented_rewrite_rule = ("# RewriteRule ^ " "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,QSA,R=permanent]") + "[L,NE,R=permanent]") self.assertTrue(commented_rewrite_rule in conf_text) mock_get_utility().add_message.assert_called_once_with(mock.ANY, @@ -1162,7 +1195,7 @@ class MultipleVhostsTest(util.ApacheTest): "RewriteCond", ["%{DOCUMENT_ROOT}/%{REQUEST_FILENAME}", "!-f"]) self.config.parser.add_dir( http_vhost.path, "RewriteRule", - ["^(.*)$", "b://u%{REQUEST_URI}", "[P,QSA,L]"]) + ["^(.*)$", "b://u%{REQUEST_URI}", "[P,NE,L]"]) # Add a chunk that should be commented out. self.config.parser.add_dir(http_vhost.path, @@ -1173,7 +1206,7 @@ class MultipleVhostsTest(util.ApacheTest): http_vhost.path, "RewriteRule", ["^", "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,QSA,R=permanent]"]) + "[L,NE,R=permanent]"]) self.config.save() @@ -1184,13 +1217,13 @@ class MultipleVhostsTest(util.ApacheTest): not_commented_cond1 = ("RewriteCond " "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f") not_commented_rewrite_rule = ("RewriteRule " - "^(.*)$ b://u%{REQUEST_URI} [P,QSA,L]") + "^(.*)$ b://u%{REQUEST_URI} [P,NE,L]") commented_cond1 = "# RewriteCond %{HTTPS} !=on" commented_cond2 = "# RewriteCond %{HTTPS} !^$" commented_rewrite_rule = ("# RewriteRule ^ " "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,QSA,R=permanent]") + "[L,NE,R=permanent]") self.assertTrue(not_commented_cond1 in conf_line_set) self.assertTrue(not_commented_rewrite_rule in conf_line_set) @@ -1208,13 +1241,13 @@ class MultipleVhostsTest(util.ApacheTest): achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"), + token=b"jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"), "pending"), domain="encryption-example.demo", account_key=account_key) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), + token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"), "pending"), domain="certbot.demo", account_key=account_key) diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index ec6eee3f2..f8b75022e 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -38,7 +38,7 @@ class SelectVhostTest(unittest.TestCase): try: self._call(self.vhosts) except errors.MissingCommandlineFlag as e: - self.assertTrue("vhost ambiguity" in e.message) + self.assertTrue("vhost ambiguity" in str(e)) @certbot_util.patch_get_utility() def test_more_info_cancel(self, mock_util): diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index 5e369e3db..62464d5d0 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -105,7 +105,7 @@ class TlsSniPerformTest(util.ApacheTest): for achall in self.achalls: self.sni.add_chall(achall) z_domain = achall.response(self.auth_key).z_domain - z_domains.append(set([z_domain])) + z_domains.append(set([z_domain.decode('ascii')])) self.sni._mod_config() # pylint: disable=protected-access self.sni.configurator.save() diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index d9e294119..65a66d2fd 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -184,7 +184,7 @@ class ApacheTlsSni01(common.TLSSNI01): # https://docs.python.org/2.7/reference/lexical_analysis.html return self.VHOST_TEMPLATE.format( vhost=ips, - server_name=achall.response(achall.account_key).z_domain, + server_name=achall.response(achall.account_key).z_domain.decode('ascii'), ssl_options_conf_path=self.configurator.mod_ssl_conf, cert_path=self.get_cert_path(achall), key_path=self.get_key_path(achall), diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 87ea1a281..9a473c584 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,13 +4,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), - 'python-augeas', + 'mock', + 'python-augeas<=0.5.0', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', @@ -18,11 +19,6 @@ install_requires = [ 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-auto b/certbot-auto index ac721638f..fc8007c9e 100755 --- a/certbot-auto +++ b/certbot-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.11.1" +LE_AUTO_VERSION="0.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -576,21 +586,23 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -615,7 +627,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 @@ -833,18 +845,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1093,6 +1105,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1209,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/certbot-compatibility-test/certbot_compatibility_test/interfaces.py b/certbot-compatibility-test/certbot_compatibility_test/interfaces.py index cd367d9af..7d3daee09 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/interfaces.py +++ b/certbot-compatibility-test/certbot_compatibility_test/interfaces.py @@ -20,20 +20,20 @@ class IPluginProxy(zope.interface.Interface): def __init__(args): """Initializes the plugin with the given command line args""" - def cleanup_from_tests(): + def cleanup_from_tests(): # type: ignore """Performs any necessary cleanup from running plugin tests. This is guaranteed to be called before the program exits. """ - def has_more_configs(): + def has_more_configs(): # type: ignore """Returns True if there are more configs to test""" - def load_config(): + def load_config(): # type: ignore """Loads the next config and returns its name""" - def get_testable_domain_names(): + def get_testable_domain_names(): # type: ignore """Returns the domain names that can be used in testing""" @@ -44,7 +44,7 @@ class IAuthenticatorProxy(IPluginProxy, certbot.interfaces.IAuthenticator): class IInstallerProxy(IPluginProxy, certbot.interfaces.IInstaller): """Wraps a Certbot installer""" - def get_all_names_answer(): + def get_all_names_answer(): # type: ignore """Returns all names that should be found by the installer""" diff --git a/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf b/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf index a201fe659..5fbc76676 100644 --- a/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf +++ b/certbot-compatibility-test/nginx/nginx-roundtrip-testdata/simplepythonfcgi/weird-spacing.conf @@ -1,16 +1,16 @@ # static files location ~ ^/(images|javascript|js|css|flash|media|static)/ { - root ${PROJECTBASE}/${PROJECTNAME}/static; + root "${PROJECTBASE}/${PROJECTNAME}/static"; } location = /favicon.ico { - root ${PROJECTBASE}/${PROJECTNAME}/static/images; + root "${PROJECTBASE}/${PROJECTNAME}/static/images"; } # pass all requests to FastCGI TG server listening on ${HOST}:${PORT} # location / { - fastcgi_pass ${HOST}:${PORT}; + fastcgi_pass "${HOST}:${PORT}"; fastcgi_index index; fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; include conf/fastcgi_params; diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index e2d226a72..aecae329f 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,21 +4,17 @@ from setuptools import setup from setuptools import find_packages -version = '0.12.0.dev0' +version = '0.14.0.dev0' install_requires = [ 'certbot', 'certbot-apache', + 'mock', 'six', 'requests', 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 382251a2f..2fd2c0226 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -5,9 +5,11 @@ import re import shutil import socket import subprocess +import tempfile import time import OpenSSL +import six import zope.interface from acme import challenges @@ -30,16 +32,16 @@ from certbot_nginx import parser logger = logging.getLogger(__name__) REDIRECT_BLOCK = [[ - ['\n ', 'if', ' ', '($scheme != "https") '], - [['\n ', 'return', ' ', '301 https://$host$request_uri'], + ['\n ', 'if', ' ', '($scheme', ' ', '!=', ' ', '"https") '], + [['\n ', 'return', ' ', '301', ' ', 'https://$host$request_uri'], '\n '] ], ['\n']] TEST_REDIRECT_BLOCK = [ [ - ['if', '($scheme != "https")'], + ['if', '($scheme', '!=', '"https")'], [ - ['return', '301 https://$host$request_uri'] + ['return', '301', 'https://$host$request_uri'] ] ], ['#', ' managed by Certbot'] @@ -262,7 +264,7 @@ class NginxConfigurator(common.Plugin): """ if not matches: return None - elif matches[0]['rank'] in xrange(2, 6): + elif matches[0]['rank'] in six.moves.range(2, 6): # Wildcard match - need to find the longest one rank = matches[0]['rank'] wildcards = [x for x in matches if x['rank'] == rank] @@ -628,10 +630,11 @@ class NginxConfigurator(common.Plugin): proc = subprocess.Popen( [self.conf('ctl'), "-c", self.nginx_conf, "-V"], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) text = proc.communicate()[1] # nginx prints output to stderr except (OSError, ValueError) as error: - logging.debug(error, exc_info=True) + logger.debug(error, exc_info=True) raise errors.PluginError( "Unable to run %s -V" % self.conf('ctl')) @@ -819,7 +822,7 @@ class NginxConfigurator(common.Plugin): self.restart() -def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): +def nginx_restart(nginx_ctl, nginx_conf): """Restarts the Nginx Server. .. todo:: Nginx restart is fatal if the configuration references @@ -830,22 +833,22 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): """ try: - proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() + proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf, "-s", "reload"]) + proc.communicate() if proc.returncode != 0: # Maybe Nginx isn't running - nginx_proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = nginx_proc.communicate() - - if nginx_proc.returncode != 0: - # Enter recovery routine... - raise errors.MisconfigurationError( - "nginx restart failed:\n%s\n%s" % (stdout, stderr)) + # Write to temporary files instead of piping because of communication issues on Arch + # https://github.com/certbot/certbot/issues/4324 + with tempfile.TemporaryFile() as out: + with tempfile.TemporaryFile() as err: + nginx_proc = subprocess.Popen([nginx_ctl, "-c", nginx_conf], + stdout=out, stderr=err) + nginx_proc.communicate() + if nginx_proc.returncode != 0: + # Enter recovery routine... + raise errors.MisconfigurationError( + "nginx restart failed:\n%s\n%s" % (out.read(), err.read())) except (OSError, ValueError): raise errors.MisconfigurationError("nginx restart failed") diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index f6437c589..20aeeb554 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -2,11 +2,9 @@ # Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) import copy import logging -import string from pyparsing import ( - Literal, White, Word, alphanums, CharsNotIn, Combine, Forward, Group, - Optional, OneOrMore, Regex, ZeroOrMore) + Literal, White, Forward, Group, Optional, OneOrMore, QuotedString, Regex, ZeroOrMore, Combine) from pyparsing import stringEnd from pyparsing import restOfLine @@ -14,73 +12,42 @@ logger = logging.getLogger(__name__) class RawNginxParser(object): # pylint: disable=expression-not-assigned + # pylint: disable=pointless-statement """A class that parses nginx configuration with pyparsing.""" # constants - space = Optional(White()) - nonspace = Regex(r"\S+") + space = Optional(White()).leaveWhitespace() + required_space = White().leaveWhitespace() + left_bracket = Literal("{").suppress() - right_bracket = space.leaveWhitespace() + Literal("}").suppress() + right_bracket = space + Literal("}").suppress() semicolon = Literal(";").suppress() - key = Word(alphanums + "_/+-.") - dollar_var = Combine(Literal('$') + Regex(r"[^\{\};,\s]+")) - condition = Regex(r"\(.+\)") - # Matches anything that is not a special character, and ${SHELL_VARS}, AND - # any chars in single or double quotes - # All of these COULD be upgraded to something like - # https://stackoverflow.com/a/16130746 - dquoted = Regex(r'(\".*\")') - squoted = Regex(r"(\'.*\')") - nonspecial = Regex(r"[^\{\};,]") - varsub = Regex(r"(\$\{\w+\})") - # nonspecial nibbles one character at a time, but the other objects take - # precedence. We use ZeroOrMore to allow entries like "break ;" to be - # parsed as assignments - value = Combine(ZeroOrMore(dquoted | squoted | varsub | nonspecial)) + dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\') + squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\') + quoted = dquoted | squoted + head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space) + tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else + tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars)) + paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars)) + # note: ')' allows extension, but then we fall into else, not last_space. - location = CharsNotIn("{};," + string.whitespace) - # modifier for location uri [ = | ~ | ~* | ^~ ] - modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~") + token = paren_quote_extend | tokenchars | quoted + + whitespace_token_group = space + token + ZeroOrMore(required_space + token) + space + assignment = whitespace_token_group + semicolon - # rules comment = space + Literal('#') + restOfLine - assignment = space + key + Optional(space + value, default=None) + semicolon - location_statement = space + Optional(modifier) + Optional(space + location + space) - if_statement = space + Literal("if") + space + condition + space - charset_map_statement = space + Literal("charset_map") + space + value + space + value - - map_statement = space + Literal("map") + space + nonspace + space + dollar_var + space - # This is NOT an accurate way to parse nginx map entries; it's almost - # certainly too permissive and may be wrong in other ways, but it should - # preserve things correctly in mmmmost or all cases. - # - # - I can neither prove nor disprove that it is correct wrt all escaped - # semicolon situations - # Addresses https://github.com/fatiherikli/nginxparser/issues/19 - map_pattern = Regex(r'".*"') | Regex(r"'.*'") | nonspace - map_entry = space + map_pattern + space + value + space + semicolon - map_block = Group( - Group(map_statement).leaveWhitespace() + - left_bracket + - Group(ZeroOrMore(Group(comment | map_entry)) + space).leaveWhitespace() + - right_bracket) - block = Forward() - # key could for instance be "server" or "http", or "location" (in which case - # location_statement needs to have a non-empty location) + # order matters! see issue 518, and also http { # server { \n} + contents = Group(comment) | Group(block) | Group(assignment) - block_begin = (Group(space + key + location_statement) ^ - Group(if_statement) ^ - Group(charset_map_statement)).leaveWhitespace() + block_begin = Group(whitespace_token_group) + block_innards = Group(ZeroOrMore(contents) + space).leaveWhitespace() + block << block_begin + left_bracket + block_innards + right_bracket - block_innards = Group(ZeroOrMore(Group(comment | assignment) | block | map_block) - + space).leaveWhitespace() - - block << Group(block_begin + left_bracket + block_innards + right_bracket) - - script = OneOrMore(Group(comment | assignment) ^ block ^ map_block) + space + stringEnd + script = OneOrMore(contents) + space + stringEnd script.parseWithTabs().leaveWhitespace() def __init__(self, source): @@ -107,30 +74,23 @@ class RawNginxDumper(object): if isinstance(b0, str): yield b0 continue - b = copy.deepcopy(b0) - if spacey(b[0]): - yield b.pop(0) # indentation - if not b: + item = copy.deepcopy(b0) + if spacey(item[0]): + yield item.pop(0) # indentation + if not item: continue - key, values = b.pop(0), b.pop(0) - if isinstance(key, list): - yield "".join(key) + '{' - for parameter in values: + if isinstance(item[0], list): # block + yield "".join(item.pop(0)) + '{' + for parameter in item.pop(0): for line in self.__iter__([parameter]): # negate "for b0 in blocks" yield line yield '}' - else: - if isinstance(key, str) and key.strip() == '#': # comment - yield key + values - else: # assignment - gap = "" - # Sometimes the parser has stuck some gap whitespace in here; - # if so rotate it into gap - if values and spacey(values): - gap = values - values = b.pop(0) - yield key + gap + values + ';' + else: # not a block - list of strings + semicolon = ";" + if isinstance(item[0], str) and item[0].strip() == '#': # comment + semicolon = "" + yield "".join(item) + semicolon def __str__(self): """Return the parsed block as a string.""" diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 29fa976f3..849cefe1f 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -1,6 +1,8 @@ """Module contains classes used by the Nginx Configurator.""" import re +import six + from certbot.plugins import common REDIRECT_DIRECTIVES = ['return', 'rewrite'] @@ -97,6 +99,11 @@ class Addr(common.Addr): def __repr__(self): return "Addr(" + self.__str__() + ")" + def __hash__(self): + # Python 3 requires explicit overridden for __hash__ + # See certbot-apache/certbot_apache/obj.py for more information + return super(Addr, self).__hash__() + def super_eq(self, other): """Check ip/port equality, with IPv6 support. """ @@ -147,13 +154,15 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods self.path = path def __str__(self): - addr_str = ", ".join(str(addr) for addr in self.addrs) + addr_str = ", ".join(str(addr) for addr in sorted(self.addrs, key=str)) + # names might be a set, and it has different representations in Python + # 2 and 3. Force it to be a list here for consistent outputs return ("file: %s\n" "addrs: %s\n" "names: %s\n" "ssl: %s\n" "enabled: %s" % (self.filep, addr_str, - self.names, self.ssl, self.enabled)) + list(self.names), self.ssl, self.enabled)) def __repr__(self): return "VirtualHost(" + self.__str__().replace("\n", ", ") + ")\n" @@ -161,7 +170,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and - list(self.addrs) == list(other.addrs) and + sorted(self.addrs, key=str) == sorted(other.addrs, key=str) and self.names == other.names and self.ssl == other.ssl and self.enabled == other.enabled and @@ -181,7 +190,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods def contains_list(self, test): """Determine if raw server block contains test list at top level """ - for i in xrange(0, len(self.raw) - len(test)): + for i in six.moves.range(0, len(self.raw) - len(test)): if self.raw[i:i + len(test)] == test: return True return False diff --git a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf index 89c920b3e..e1839909d 100644 --- a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf +++ b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf @@ -4,4 +4,4 @@ ssl_session_timeout 1440m; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_prefer_server_ciphers on; -ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; +ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS"; diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 1a2c85c2c..6f3f344db 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -205,8 +205,8 @@ class NginxParser(object): trees.append(parsed) except IOError: logger.warning("Could not open file: %s", item) - except pyparsing.ParseException: - logger.debug("Could not parse file: %s", item) + except pyparsing.ParseException as err: + logger.debug("Could not parse file: %s due to %s", item, err) return trees def _parse_ssl_options(self, ssl_options): @@ -216,8 +216,8 @@ class NginxParser(object): return nginxparser.load(_file).spaced except IOError: logger.warn("Missing NGINX TLS options file: %s", ssl_options) - except pyparsing.ParseBaseException: - logger.debug("Could not parse file: %s", ssl_options) + except pyparsing.ParseBaseException as err: + logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] def _set_locations(self, ssl_options): @@ -298,9 +298,9 @@ class NginxParser(object): """ server = vhost.raw for directive in server: - if not directive or len(directive) < 2: + if not directive: continue - elif directive[0] == 'ssl' and directive[1] == 'on': + elif _is_ssl_on_directive(directive): return True return False @@ -342,7 +342,7 @@ class NginxParser(object): vhost.names = parsed_server['names'] vhost.raw = new_server except errors.MisconfigurationError as err: - raise errors.MisconfigurationError("Problem in %s: %s" % (filename, err.message)) + raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) def _do_for_subarray(entry, condition, func, path=None): @@ -468,17 +468,17 @@ def _is_include_directive(entry): len(entry) == 2 and entry[0] == 'include' and isinstance(entry[1], str)) +def _is_ssl_on_directive(entry): + """Checks if an nginx parsed entry is an 'ssl on' directive. -def _get_servernames(names): - """Turns a server_name string into a list of server names - - :param str names: server names - :rtype: list + :param list entry: the parsed entry + :returns: Whether it's an 'ssl on' directive + :rtype: bool """ - whitespace_re = re.compile(r'\s+') - names = re.sub(whitespace_re, ' ', names) - return names.split(' ') + return (isinstance(entry, list) and + len(entry) == 2 and entry[0] == 'ssl' and + entry[1] == 'on') def _add_directives(block, directives, replace): """Adds or replaces directives in a config block. @@ -550,12 +550,11 @@ def _add_directive(block, directive, replace): # and there is already a copy of that directive with a different value # in the config file. directive_name = directive[0] - directive_value = directive[1] if location is None or (isinstance(directive_name, str) and directive_name in REPEATABLE_DIRECTIVES): block.append(directive) _comment_directive(block, len(block) - 1) - elif block[location][1] != directive_value: + elif block[location] != directive: raise errors.MisconfigurationError( 'tried to insert directive "{0}" but found ' 'conflicting "{1}".'.format(directive, block[location])) @@ -585,14 +584,14 @@ def _parse_server_raw(server): if not directive: continue if directive[0] == 'listen': - addr = obj.Addr.fromstring(directive[1]) - parsed_server['addrs'].add(addr) - if addr.ssl: - parsed_server['ssl'] = True + addr = obj.Addr.fromstring(" ".join(directive[1:])) + if addr: + parsed_server['addrs'].add(addr) + if addr.ssl: + parsed_server['ssl'] = True elif directive[0] == 'server_name': - parsed_server['names'].update( - _get_servernames(directive[1])) - elif directive[0] == 'ssl' and directive[1] == 'on': + parsed_server['names'].update(directive[1:]) + elif _is_ssl_on_directive(directive): parsed_server['ssl'] = True apply_ssl_to_all_addrs = True diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index cc36aa0de..b9e70cd59 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -27,12 +27,13 @@ class NginxConfiguratorTest(util.NginxTest): super(NginxConfiguratorTest, self).setUp() self.config = util.get_nginx_configurator( - self.config_path, self.config_dir, self.work_dir) + self.config_path, self.config_dir, self.work_dir, self.logs_dir) def tearDown(self): shutil.rmtree(self.temp_dir) shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) + shutil.rmtree(self.logs_dir) @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): @@ -93,7 +94,7 @@ class NginxConfiguratorTest(util.NginxTest): None, [0]) self.config.parser.add_server_directives( mock_vhost, - [['listen', ' ', '5001 ssl']], + [['listen', ' ', '5001', ' ', 'ssl']], replace=False) self.config.save() @@ -104,7 +105,7 @@ class NginxConfiguratorTest(util.NginxTest): ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['#', parser.COMMENT]]]], parsed[0]) @@ -204,13 +205,13 @@ class NginxConfiguratorTest(util.NginxTest): ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['ssl_certificate', 'example/fullchain.pem'], ['ssl_certificate_key', 'example/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) ]], parsed_example_conf) - self.assertEqual([['server_name', 'somename alias another.alias']], + self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], parsed_server_conf) self.assertTrue(util.contains_at_depth( parsed_nginx_conf, @@ -221,8 +222,8 @@ class NginxConfiguratorTest(util.NginxTest): ['include', 'server.conf'], [['location', '/'], [['root', 'html'], - ['index', 'index.html index.htm']]], - ['listen', '5001 ssl'], + ['index', 'index.html', 'index.htm']]], + ['listen', '5001', 'ssl'], ['ssl_certificate', '/etc/nginx/fullchain.pem'], ['ssl_certificate_key', '/etc/nginx/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) @@ -246,7 +247,7 @@ class NginxConfiguratorTest(util.NginxTest): ['server_name', 'summer.com'], ['listen', '80'], - ['listen', '5001 ssl'], + ['listen', '5001', 'ssl'], ['ssl_certificate', 'summer/fullchain.pem'], ['ssl_certificate_key', 'summer/key.pem']] + util.filter_comments(self.config.parser.loc["ssl_options"]) @@ -261,13 +262,13 @@ class NginxConfiguratorTest(util.NginxTest): # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"), + chall=challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), uri="https://ca.org/chall0_uri", status=messages.Status("pending"), ), domain="localhost", account_key=self.rsa512jwk) achall2 = achallenges.KeyAuthorizationAnnotatedChallenge( challb=messages.ChallengeBody( - chall=challenges.TLSSNI01(token="m8TdO1qik4JVFtgPPurJmg"), + chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"), uri="https://ca.org/chall1_uri", status=messages.Status("pending"), ), domain="example.com", account_key=self.rsa512jwk) @@ -407,8 +408,8 @@ class NginxConfiguratorTest(util.NginxTest): # Test that we successfully add a redirect when there is # a listen directive expected = [ - ['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + ['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ] example_conf = self.config.parser.abs_path('sites-enabled/example.com') diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index e83b414cf..dd31ebac8 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -25,15 +25,15 @@ class TestRawNginxParser(unittest.TestCase): def test_blocks(self): parsed = RawNginxParser.block.parseString('foo {}').asList() - self.assertEqual(parsed, [[['foo', ' '], []]]) + self.assertEqual(parsed, [['foo', ' '], []]) parsed = RawNginxParser.block.parseString('location /foo{}').asList() - self.assertEqual(parsed, [[['location', ' ', '/foo'], []]]) + self.assertEqual(parsed, [['location', ' ', '/foo'], []]) parsed = RawNginxParser.block.parseString('foo { bar foo ; }').asList() - self.assertEqual(parsed, [[['foo', ' '], [[' ', 'bar', ' ', 'foo '], ' ']]]) + self.assertEqual(parsed, [['foo', ' '], [[' ', 'bar', ' ', 'foo', ' '], ' ']]) def test_nested_blocks(self): parsed = RawNginxParser.block.parseString('foo { bar {} }').asList() - block, content = FIRST(parsed) + block, content = parsed self.assertEqual(FIRST(content), [[' ', 'bar', ' '], []]) self.assertEqual(FIRST(block), 'foo') @@ -72,8 +72,8 @@ class TestRawNginxParser(unittest.TestCase): [['user', 'www-data'], [['http'], [[['server'], [ - ['listen', '*:80 default_server ssl'], - ['server_name', '*.www.foo.com *.www.example.com'], + ['listen', '*:80', 'default_server', 'ssl'], + ['server_name', '*.www.foo.com', '*.www.example.com'], ['root', '/home/ubuntu/sites/foo/'], [['location', '/status'], [ [['types'], [['image/jpeg', 'jpg']]], @@ -97,17 +97,35 @@ class TestRawNginxParser(unittest.TestCase): [['server'], [['server_name', 'with.if'], [['location', '~', '^/services/.+$'], - [[['if', '($request_filename ~* \\.(ttf|woff)$)'], - [['add_header', 'Access-Control-Allow-Origin "*"']]]]]]], + [[['if', '($request_filename', '~*', '\\.(ttf|woff)$)'], + [['add_header', 'Access-Control-Allow-Origin', '"*"']]]]]]], [['server'], [['server_name', 'with.complicated.headers'], [['location', '~*', '\\.(?:gif|jpe?g|png)$'], - [['add_header', 'Pragma public'], + [['add_header', 'Pragma', 'public'], ['add_header', - 'Cache-Control \'public, must-revalidate, proxy-revalidate\'' - ' "test,;{}" foo'], + 'Cache-Control', '\'public, must-revalidate, proxy-revalidate\'', + '"test,;{}"', 'foo'], ['blah', '"hello;world"'], - ['try_files', '$uri @rewrites']]]]]]) + ['try_files', '$uri', '@rewrites']]]]]]) + + def test_parse_from_file3(self): + with open(util.get_data_filename('multiline_quotes.conf')) as handle: + parsed = util.filter_comments(load(handle)) + self.assertEqual( + parsed, + [[['http'], + [[['server'], + [['listen', '*:443'], + [['location', '/'], + [['body_filter_by_lua', + '\'ngx.ctx.buffered = (ngx.ctx.buffered or "")' + ' .. string.sub(ngx.arg[1], 1, 1000)\n' + ' ' + 'if ngx.arg[2] then\n' + ' ' + 'ngx.var.resp_body = ngx.ctx.buffered\n' + ' end\'']]]]]]]]) def test_abort_on_parse_failure(self): with open(util.get_data_filename('broken.conf')) as handle: @@ -117,7 +135,7 @@ class TestRawNginxParser(unittest.TestCase): with open(util.get_data_filename('nginx.conf')) as handle: parsed = load(handle) parsed[-1][-1].append(UnspacedList([['server'], - [['listen', ' ', '443 ssl'], + [['listen', ' ', '443', ' ', 'ssl'], ['server_name', ' ', 'localhost'], ['ssl_certificate', ' ', 'cert.pem'], ['ssl_certificate_key', ' ', 'cert.key'], @@ -126,9 +144,9 @@ class TestRawNginxParser(unittest.TestCase): ['ssl_ciphers', ' ', 'HIGH:!aNULL:!MD5'], [['location', ' ', '/'], [['root', ' ', 'html'], - ['index', ' ', 'index.html index.htm']]]]])) + ['index', ' ', 'index.html', ' ', 'index.htm']]]]])) - with tempfile.TemporaryFile() as f: + with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) f.seek(0) parsed_new = load(f) @@ -138,7 +156,7 @@ class TestRawNginxParser(unittest.TestCase): with open(util.get_data_filename('minimalistic_comments.conf')) as handle: parsed = load(handle) - with tempfile.TemporaryFile() as f: + with tempfile.TemporaryFile(mode='w+t') as f: dump(parsed, f) f.seek(0) parsed_new = load(f) @@ -161,10 +179,177 @@ class TestRawNginxParser(unittest.TestCase): parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }') self.assertEqual(parsed, [ - [['if', '($http_accept ~* "webp")'], - [['set', '$webp "true"']]] + [['if', '($http_accept', '~*', '"webp")'], + [['set', '$webp', '"true"']]] ]) + def test_comment_in_block(self): + parsed = loads("""http { + # server{ + }""") + + self.assertEqual(parsed, [ + [['http'], + [['#', ' server{']]] + ]) + + def test_access_log(self): + # see issue #3798 + parsed = loads('access_log syslog:server=unix:/dev/log,facility=auth,' + 'tag=nginx_post,severity=info custom;') + + self.assertEqual(parsed, [ + ['access_log', + 'syslog:server=unix:/dev/log,facility=auth,tag=nginx_post,severity=info', + 'custom'] + ]) + + def test_add_header(self): + # see issue #3798 + parsed = loads('add_header Cache-Control no-cache,no-store,must-revalidate,max-age=0;') + + self.assertEqual(parsed, [ + ['add_header', 'Cache-Control', 'no-cache,no-store,must-revalidate,max-age=0'] + ]) + + def test_map_then_assignment_in_block(self): + # see issue #3798 + test_str = """http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + "~Opera Mini" 1; + *.example.com 1; + } + one; + }""" + parsed = loads(test_str) + self.assertEqual(parsed, [ + [['http'], [ + [['map', '$http_upgrade', '$connection_upgrade'], [ + ['default', 'upgrade'], + ["''", 'close'], + ['"~Opera Mini"', '1'], + ['*.example.com', '1'] + ]], + ['one'] + ]] + ]) + + def test_variable_name(self): + parsed = loads('try_files /typo3temp/tx_ncstaticfilecache/' + '$host${request_uri}index.html @nocache;') + + self.assertEqual(parsed, [ + ['try_files', + '/typo3temp/tx_ncstaticfilecache/$host${request_uri}index.html', + '@nocache'] + ]) + + def test_weird_blocks(self): + test = r""" + if ($http_user_agent ~ MSIE) { + rewrite ^(.*)$ /msie/$1 break; + } + + if ($http_cookie ~* "id=([^;]+)(?:;|$)") { + set $id $1; + } + + if ($request_method = POST) { + return 405; + } + + if ($request_method) { + return 403; + } + + if ($args ~ post=140){ + rewrite ^ http://example.com/; + } + + location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ { + alias /data/w3/images/$1; + } + """ + parsed = loads(test) + self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'], + [['rewrite', '^(.*)$', '/msie/$1', 'break']]], + [['if', '($http_cookie', '~*', '"id=([^;]+)(?:;|$)")'], [['set', '$id', '$1']]], + [['if', '($request_method', '=', 'POST)'], [['return', '405']]], + [['if', '($request_method)'], + [['return', '403']]], [['if', '($args', '~', 'post=140)'], + [['rewrite', '^', 'http://example.com/']]], + [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'], + [['alias', '/data/w3/images/$1']]]] + ) + + def test_edge_cases(self): + # quotes + parsed = loads(r'"hello\""; # blah "heh heh"') + self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']]) + + # empty var as block + parsed = loads(r"${}") + self.assertEqual(parsed, [[['$'], []]]) + + # if with comment + parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah ) + }""") + self.assertEqual(parsed, [[['if', '($http_cookie', '~*', '"id=([^;]+)(?:;|$)")'], + [['#', ' blah )']]]]) + + # end paren + test = """ + one"test"; + ("two"); + "test")red; + "test")"blue"; + "test")"three; + (one"test")one; + one"; + one"test; + one"test"one; + """ + parsed = loads(test) + self.assertEqual(parsed, [ + ['one"test"'], + ['("two")'], + ['"test")red'], + ['"test")"blue"'], + ['"test")"three'], + ['(one"test")one'], + ['one"'], + ['one"test'], + ['one"test"one'] + ]) + self.assertRaises(ParseException, loads, r'"test"one;') # fails + self.assertRaises(ParseException, loads, r'"test;') # fails + + # newlines + test = """ + server_name foo.example.com bar.example.com \ + baz.example.com qux.example.com; + server_name foo.example.com bar.example.com + baz.example.com qux.example.com; + """ + parsed = loads(test) + self.assertEqual(parsed, [ + ['server_name', 'foo.example.com', 'bar.example.com', + 'baz.example.com', 'qux.example.com'], + ['server_name', 'foo.example.com', 'bar.example.com', + 'baz.example.com', 'qux.example.com'] + ]) + + # variable weirdness + parsed = loads("directive $var;") + self.assertEqual(parsed, [['directive', '$var']]) + self.assertRaises(ParseException, loads, "server {server_name test.com};") + self.assertRaises(ParseException, loads, "directive ${var};") + self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']]) + self.assertRaises(ParseException, loads, "blag${dfgdf{g};") + + class TestUnspacedList(unittest.TestCase): """Test the UnspacedList data structure""" def setUp(self): @@ -219,18 +404,18 @@ class TestUnspacedList(unittest.TestCase): ['\n ', 'listen', ' ', '127.0.0.1'], ['\n ', 'server_name', ' ', '.example.com'], ['\n ', 'server_name', ' ', 'example.*'], '\n', - ['listen', ' ', '5001 ssl']]) + ['listen', ' ', '5001', ' ', 'ssl']]) x.insert(5, "FROGZ") self.assertEqual(x, [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], ['server_name', '.example.com'], ['server_name', 'example.*'], - ['listen', '5001 ssl'], 'FROGZ']) + ['listen', '5001', 'ssl'], 'FROGZ']) self.assertEqual(x.spaced, [['\n ', 'listen', ' ', '69.50.225.155:9000'], ['\n ', 'listen', ' ', '127.0.0.1'], ['\n ', 'server_name', ' ', '.example.com'], ['\n ', 'server_name', ' ', 'example.*'], '\n', - ['listen', ' ', '5001 ssl'], + ['listen', ' ', '5001', ' ', 'ssl'], 'FROGZ']) def test_rawlists(self): diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py index b0a2d5ad8..ba136bb78 100644 --- a/certbot-nginx/certbot_nginx/tests/obj_test.py +++ b/certbot-nginx/certbot_nginx/tests/obj_test.py @@ -108,8 +108,8 @@ class VirtualHostTest(unittest.TestCase): from certbot_nginx.obj import Addr raw1 = [ ['listen', '69.50.225.155:9000'], - [['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + [['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ], ['#', ' managed by Certbot'] ] @@ -119,8 +119,8 @@ class VirtualHostTest(unittest.TestCase): set(['localhost']), raw1, []) raw2 = [ ['listen', '69.50.225.155:9000'], - [['if', '($scheme != "https") '], - [['return', '301 https://$host$request_uri']] + [['if', '($scheme', '!=', '"https") '], + [['return', '301', 'https://$host$request_uri']] ] ] self.vhost2 = VirtualHost( @@ -129,7 +129,7 @@ class VirtualHostTest(unittest.TestCase): set(['localhost']), raw2, []) raw3 = [ ['listen', '69.50.225.155:9000'], - ['rewrite', '^(.*)$ $scheme://www.domain.com$1 permanent;'] + ['rewrite', '^(.*)$', '$scheme://www.domain.com$1', 'permanent'] ] self.vhost3 = VirtualHost( "filep", @@ -158,7 +158,7 @@ class VirtualHostTest(unittest.TestCase): def test_str(self): stringified = '\n'.join(['file: filep', 'addrs: localhost', - "names: set(['localhost'])", 'ssl: False', + "names: ['localhost']", 'ssl: False', 'enabled: False']) self.assertEqual(stringified, str(self.vhost1)) @@ -181,7 +181,9 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme != "https")'], [['return', '301 https://$host$request_uri']]], + [['if', '($scheme', '!=', '"https")'], + [['return', '301', 'https://$host$request_uri']] + ], ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", @@ -195,7 +197,9 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], ['ssl_certificate_key', '/etc/letsencrypt/live/two.functorkitten.xyz/privkey.pem'], ['#', ' managed by Certbot'], - [['if', '($scheme != "https")'], [['return', '302 https://$host$request_uri']]], + [['if', '($scheme', '!=', '"https")'], + [['return', '302', 'https://$host$request_uri']] + ], ['#', ' managed by Certbot'], []] vhost_bad_haystack = VirtualHost( "filp", diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 921cc3c5a..9c2b8656e 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -52,7 +52,7 @@ class NginxParserTest(util.NginxTest): 'sites-enabled/sslon.com', 'sites-enabled/globalssl.com']]), set(nparser.parsed.keys())) - self.assertEqual([['server_name', 'somename alias another.alias']], + self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], nparser.parsed[nparser.abs_path('server.conf')]) self.assertEqual([[['server'], [['listen', '69.50.225.155:9000'], ['listen', '127.0.0.1'], @@ -168,16 +168,16 @@ class NginxParserTest(util.NginxTest): [['location', '/'], [['root', 'html'], ['index', 'index.html index.htm']]] ], None) self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) - mock_vhost.raw = [['listen', '*:80 default_server ssl'], - ['server_name', '*.www.foo.com *.www.example.com'], + mock_vhost.raw = [['listen', '*:80', 'default_server', 'ssl'], + ['server_name', '*.www.foo.com', '*.www.example.com'], ['root', '/home/ubuntu/sites/foo/']] self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) mock_vhost.raw = [['listen', '80 ssl'], - ['server_name', '*.www.foo.com *.www.example.com']] + ['server_name', '*.www.foo.com', '*.www.example.com']] self.assertFalse(nparser.has_ssl_on_directive(mock_vhost)) mock_vhost.raw = [['listen', '80'], ['ssl', 'on'], - ['server_name', '*.www.foo.com *.www.example.com']] + ['server_name', '*.www.foo.com', '*.www.example.com']] self.assertTrue(nparser.has_ssl_on_directive(mock_vhost)) def test_add_server_directives(self): @@ -309,7 +309,7 @@ class NginxParserTest(util.NginxTest): self.assertFalse(server['ssl']) server = parser._parse_server_raw([ #pylint: disable=protected-access - ['listen', '443 ssl'] + ['listen', '443', 'ssl'] ]) self.assertTrue(server['ssl']) @@ -323,6 +323,12 @@ class NginxParserTest(util.NginxTest): ]) self.assertTrue(server['ssl']) + def test_parse_server_raw_unix(self): + server = parser._parse_server_raw([ #pylint: disable=protected-access + ['listen', 'unix:/var/run/nginx.sock'] + ]) + self.assertEqual(len(server['addrs']), 0) + def test_parse_server_global_ssl_applied(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) server = nparser.parse_server([ @@ -335,16 +341,21 @@ class NginxParserTest(util.NginxTest): self.assertEqual(nginxparser.UnspacedList(nparser.loc["ssl_options"]), [['ssl_session_cache', 'shared:le_nginx_SSL:1m'], ['ssl_session_timeout', '1440m'], - ['ssl_protocols', 'TLSv1 TLSv1.1 TLSv1.2'], + ['ssl_protocols', 'TLSv1', 'TLSv1.1', 'TLSv1.2'], ['ssl_prefer_server_ciphers', 'on'], - ['ssl_ciphers', '"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-'+ - 'AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256'+ - '-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384'+ - ' ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384'+ - ' ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-'+ - 'AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM'+ - '-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-'+ - 'AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"'] + ['ssl_ciphers', '"ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-'+ + 'RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:'+ + 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-'+ + 'SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-'+ + 'SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-'+ + 'SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:'+ + 'ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-'+ + 'AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:'+ + 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-'+ + 'SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-'+ + 'RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:'+ + 'AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:'+ + 'AES256-SHA:DES-CBC3-SHA:!DSS"'] ]) if __name__ == "__main__": diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf new file mode 100644 index 000000000..74cd84bcd --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/multiline_quotes.conf @@ -0,0 +1,16 @@ +# Test nginx configuration file with multiline quoted strings. +# Good example of usage for multilined quoted values is when +# using Openresty's Lua directives and you wish to keep the +# inline Lua code readable. +http { + server { + listen *:443; # because there should be no other port open. + + location / { + body_filter_by_lua 'ngx.ctx.buffered = (ngx.ctx.buffered or "") .. string.sub(ngx.arg[1], 1, 1000) + if ngx.arg[2] then + ngx.var.resp_body = ngx.ctx.buffered + end'; + } + } +} diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index e7dacb400..7a2de44a2 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -3,6 +3,7 @@ import unittest import shutil import mock +import six from acme import challenges @@ -23,25 +24,25 @@ class TlsSniPerformTest(util.NginxTest): achalls = [ achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( - challenges.TLSSNI01(token="kNdwjwOeX0I_A8DXt9Msmg"), "pending"), + challenges.TLSSNI01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"), domain="www.example.com", account_key=account_key), achallenges.KeyAuthorizationAnnotatedChallenge( challb=acme_util.chall_to_challb( challenges.TLSSNI01( - token="\xba\xa9\xda?=1.5.5', # Python3 support; perhaps unnecessary? # For pkg_resources. >=1.0 so pip resolves it to a version cryptography @@ -18,11 +19,6 @@ install_requires = [ 'zope.interface', ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot/__init__.py b/certbot/__init__.py index 6451eb0d5..228a1a2a6 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.12.0.dev0' +__version__ = '0.14.0.dev0' diff --git a/certbot/account.py b/certbot/account.py index 4b5f48411..389f96791 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -74,7 +74,8 @@ class Account(object): # pylint: disable=too-few-public-methods self.meta.creation_dt), self.meta.creation_host, self.id[:4]) def __repr__(self): - return "<{0}({1})>".format(self.__class__.__name__, self.id) + return "<{0}({1}, {2}, {3})>".format( + self.__class__.__name__, self.regr, self.id, self.meta) def __eq__(self, other): return (isinstance(other, self.__class__) and @@ -98,7 +99,7 @@ def report_new_account(config): class AccountMemoryStorage(interfaces.AccountStorage): - """In-memory account strage.""" + """In-memory account storage.""" def __init__(self, initial_accounts=None): self.accounts = initial_accounts if initial_accounts is not None else {} @@ -106,7 +107,8 @@ class AccountMemoryStorage(interfaces.AccountStorage): def find_all(self): return list(six.itervalues(self.accounts)) - def save(self, account): + def save(self, account, acme): + # pylint: disable=unused-argument if account.id in self.accounts: logger.debug("Overwriting account: %s", account.id) self.accounts[account.id] = account @@ -117,6 +119,16 @@ class AccountMemoryStorage(interfaces.AccountStorage): except KeyError: raise errors.AccountNotFound(account_id) +class RegistrationResourceWithNewAuthzrURI(messages.RegistrationResource): + """A backwards-compatible RegistrationResource with a new-authz URI. + + Hack: Certbot versions pre-0.11.1 expect to load + new_authzr_uri as part of the account. Because people + sometimes switch between old and new versions, we will + continue to write out this field for some time so older + clients don't crash in that scenario. + """ + new_authzr_uri = jose.Field('new_authzr_uri') class AccountFileStorage(interfaces.AccountStorage): """Accounts file storage. @@ -181,16 +193,16 @@ class AccountFileStorage(interfaces.AccountStorage): account_id, acc.id)) return acc - def save(self, account): - self._save(account, regr_only=False) + def save(self, account, acme): + self._save(account, acme, regr_only=False) - def save_regr(self, account): + def save_regr(self, account, acme): """Save the registration resource. :param Account account: account whose regr should be saved """ - self._save(account, regr_only=True) + self._save(account, acme, regr_only=True) def delete(self, account_id): """Delete registration info from disk @@ -204,13 +216,19 @@ class AccountFileStorage(interfaces.AccountStorage): "Account at %s does not exist" % account_dir_path) shutil.rmtree(account_dir_path) - def _save(self, account, regr_only): + def _save(self, account, acme, regr_only): account_dir_path = self._account_dir_path(account.id) util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: - regr_file.write(account.regr.json_dumps()) + regr = account.regr + with_uri = RegistrationResourceWithNewAuthzrURI( + new_authzr_uri=acme.directory.new_authz, + body=regr.body, + uri=regr.uri, + terms_of_service=regr.terms_of_service) + regr_file.write(with_uri.json_dumps()) if not regr_only: with util.safe_open(self._key_path(account_dir_path), "w", chmod=0o400) as key_file: diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index 53346a77c..a1f23a895 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -66,11 +66,16 @@ class AuthHandler(object): self.authzr[domain] = self.acme.request_domain_challenges(domain) self._choose_challenges(domains) + config = zope.component.getUtility(interfaces.IConfig) + notify = zope.component.getUtility(interfaces.IDisplay).notification # While there are still challenges remaining... while self.achalls: resp = self._solve_challenges() logger.info("Waiting for verification...") + if config.debug_challenges: + notify('Challenges loaded. Press continue to submit to CA. ' + 'Pass "-v" for more info about challenges.', pause=True) # Send all Responses - this modifies achalls self._respond(resp, best_effort) diff --git a/certbot/cert_manager.py b/certbot/cert_manager.py index 71a5fe6fa..35d539a16 100644 --- a/certbot/cert_manager.py +++ b/certbot/cert_manager.py @@ -100,7 +100,10 @@ def lineage_for_certname(cli_config, certname): configs_dir = cli_config.renewal_configs_dir # Verify the directory is there util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) - renewal_file = storage.renewal_file_for_certname(cli_config, certname) + try: + renewal_file = storage.renewal_file_for_certname(cli_config, certname) + except errors.CertStorageError: + return None try: return storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): diff --git a/certbot/cli.py b/certbot/cli.py index a9714152f..2a071bbb1 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -19,6 +19,7 @@ import certbot from certbot import constants from certbot import crypto_util from certbot import errors +from certbot import hooks from certbot import interfaces from certbot import util @@ -207,7 +208,7 @@ def set_by_cli(var): return False # static housekeeping var -set_by_cli.detector = None +set_by_cli.detector = None # type: ignore def has_default_value(option, value): @@ -515,6 +516,13 @@ class HelpfulArgumentParser(object): return usage + def remove_config_file_domains_for_renewal(self, parsed_args): + """Make "certbot renew" safe if domains are set in cli.ini.""" + # Works around https://github.com/certbot/certbot/issues/4096 + if self.verb == "renew": + for source, flags in self.parser._source_to_settings.items(): # pylint: disable=protected-access + if source.startswith("config_file") and "domains" in flags: + parsed_args.domains = _Default() if self.detect_defaults else [] def parse_args(self): """Parses command line arguments and returns the result. @@ -527,6 +535,8 @@ class HelpfulArgumentParser(object): parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb + self.remove_config_file_domains_for_renewal(parsed_args) + if self.detect_defaults: return parsed_args @@ -556,6 +566,11 @@ class HelpfulArgumentParser(object): if parsed_args.must_staple: parsed_args.staple = True + if parsed_args.validate_hooks: + hooks.validate_hooks(parsed_args) + + possible_deprecation_warning(parsed_args) + return parsed_args def set_test_server(self, parsed_args): @@ -895,7 +910,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "'run' subcommand this means reinstall the existing cert). (default: Ask)") helpful.add( "automation", "--expand", action="store_true", - help="If an existing cert covers some subset of the requested names, " + help="If an existing cert is a strict subset of the requested names, " "always expand and replace it with the additional names. (default: Ask)") helpful.add( "automation", "--version", action="version", @@ -941,6 +956,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis help="(certbot-auto only) prevent the certbot-auto script from" " upgrading itself to newer released versions (default: Upgrade" " automatically)") + helpful.add( + "automation", "--no-bootstrap", action="store_true", + help="(certbot-auto only) prevent the certbot-auto script from" + " installing OS-level dependencies (default: Prompt to install " + " OS-wide dependencies, but exit if the user says 'No')") helpful.add( ["automation", "renew", "certonly", "run"], "-q", "--quiet", dest="quiet", action="store_true", @@ -955,6 +975,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "testing", "--debug", action="store_true", help="Show tracebacks in case of errors, and allow certbot-auto " "execution on experimental platforms") + helpful.add( + [None, "certonly", "renew", "run"], "--debug-challenges", action="store_true", + default=flag_default("debug_challenges"), + help="After setting up challenges, wait for user input before " + "submitting to CA") helpful.add( "testing", "--no-verify-ssl", action="store_true", help=config_help("no_verify_ssl"), @@ -1051,9 +1076,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis "renew", "--renew-hook", help="Command to be run in a shell once for each successfully renewed" " certificate. For this command, the shell variable $RENEWED_LINEAGE" - " will point to the config live subdirectory containing the new certs" + " will point to the config live subdirectory (for example," + " \"/etc/letsencrypt/live/example.com\") containing the new certs" " and keys; the shell variable $RENEWED_DOMAINS will contain a" - " space-delimited list of renewed cert domains") + " space-delimited list of renewed cert domains (for example," + " \"example.com www.example.com\"") helpful.add( "renew", "--disable-hook-validation", action='store_false', dest='validate_hooks', default=True, @@ -1156,7 +1183,7 @@ def _paths_parser(helpful): default_cp = None if verb == "certonly": default_cp = flag_default("auth_chain_path") - add(["install", "paths"], "--fullchain-path", default=default_cp, type=os.path.abspath, + add(["paths", "install"], "--fullchain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a full certificate chain (cert plus chain).") add("paths", "--chain-path", default=default_cp, type=os.path.abspath, help="Accompanying path to a certificate chain.") diff --git a/certbot/client.py b/certbot/client.py index fc4b3bfdb..1ef6fc877 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -8,6 +8,7 @@ import OpenSSL import zope.component from acme import client as acme_client +from acme import errors as acme_errors from acme import jose from acme import messages @@ -117,7 +118,7 @@ def register(config, account_storage, tos_cb=None): logger.warning(msg) raise errors.Error(msg) if not config.dry_run: - logger.warning("Registering without email!") + logger.info("Registering without email!") # Each new registration shall use a fresh new key key = jose.JWKRSA(key=jose.ComparableRSAKey( @@ -138,7 +139,7 @@ def register(config, account_storage, tos_cb=None): acc = account.Account(regr, key) account.report_new_account(config) - account_storage.save(acc) + account_storage.save(acc, acme) eff.handle_subscription(config) @@ -155,8 +156,6 @@ def perform_registration(acme, config): :returns: Registration Resource. :rtype: `acme.messages.RegistrationResource` - - :raises .UnexpectedUpdate: """ try: return acme.register(messages.NewRegistration.from_data(email=config.email)) @@ -168,7 +167,7 @@ def perform_registration(acme, config): "registration again." % config.email) raise errors.Error(msg) else: - config.namespace.email = display_ops.get_email(invalid=True) + config.email = display_ops.get_email(invalid=True) return perform_registration(acme, config) else: raise @@ -208,15 +207,14 @@ class Client(object): else: self.auth_handler = None - def obtain_certificate_from_csr(self, domains, csr, - typ=OpenSSL.crypto.FILETYPE_ASN1, authzr=None): + def obtain_certificate_from_csr(self, domains, csr, authzr=None): """Obtain certificate. Internal function with precondition that `domains` are consistent with identifiers present in the `csr`. :param list domains: Domain names. - :param .util.CSR csr: DER-encoded Certificate Signing + :param .util.CSR csr: PEM-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. :param list authzr: List of @@ -242,9 +240,30 @@ class Client(object): certr = self.acme.request_issuance( jose.ComparableX509( - OpenSSL.crypto.load_certificate_request(typ, csr.data)), + OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr.data)), authzr) - return certr, self.acme.fetch_chain(certr) + + notify = zope.component.getUtility(interfaces.IDisplay).notification + retries = 0 + chain = None + + while retries <= 1: + if retries: + notify('Failed to fetch chain, please check your network ' + 'and continue', pause=True) + try: + chain = self.acme.fetch_chain(certr) + break + except acme_errors.Error: + logger.debug('Failed to fetch chain', exc_info=True) + retries += 1 + + if chain is None: + raise acme_errors.Error( + 'Failed to fetch chain. You should not deploy the generated ' + 'certificate, please rerun the command for a new one.') + + return certr, chain def obtain_certificate(self, domains): """Obtains a certificate from the ACME server. @@ -305,10 +324,12 @@ class Client(object): key = crypto_util.save_key( key_pem, self.config.key_dir) csr = crypto_util.init_save_csr(key, domains, self.config.csr_dir) + certr, chain = self.obtain_certificate_from_csr( + domains, csr, authzr=authzr) - return (self.obtain_certificate_from_csr(domains, csr, authzr=authzr) - + (key, csr)) + return certr, chain, key, csr + # pylint: disable=no-member def obtain_and_enroll_certificate(self, domains, certname): """Obtain and enroll certificate. diff --git a/certbot/colored_logging.py b/certbot/colored_logging.py deleted file mode 100644 index 93bf3a55a..000000000 --- a/certbot/colored_logging.py +++ /dev/null @@ -1,45 +0,0 @@ -"""A formatter and StreamHandler for colorizing logging output.""" -import logging -import sys - -from certbot import util - - -class StreamHandler(logging.StreamHandler): - """Sends colored logging output to a stream. - - If the specified stream is not a tty, the class works like the - standard logging.StreamHandler. Default red_level is logging.WARNING. - - :ivar bool colored: True if output should be colored - :ivar bool red_level: The level at which to output - - """ - - def __init__(self, stream=None): - if sys.version_info < (2, 7): - # pragma: no cover - # pylint: disable=non-parent-init-called - logging.StreamHandler.__init__(self, stream) - else: - super(StreamHandler, self).__init__(stream) - self.colored = (sys.stderr.isatty() if stream is None else - stream.isatty()) - self.red_level = logging.WARNING - - def format(self, record): - """Formats the string representation of record. - - :param logging.LogRecord record: Record to be formatted - - :returns: Formatted, string representation of record - :rtype: str - - """ - out = (logging.StreamHandler.format(self, record) - if sys.version_info < (2, 7) - else super(StreamHandler, self).format(record)) - if self.colored and record.levelno >= self.red_level: - return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) - else: - return out diff --git a/certbot/configuration.py b/certbot/configuration.py index d25378922..30c6f0437 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -42,7 +42,7 @@ class NamespaceConfig(object): """ def __init__(self, namespace): - self.namespace = namespace + object.__setattr__(self, 'namespace', namespace) self.namespace.config_dir = os.path.abspath(self.namespace.config_dir) self.namespace.work_dir = os.path.abspath(self.namespace.work_dir) @@ -54,6 +54,9 @@ class NamespaceConfig(object): def __getattr__(self, name): return getattr(self.namespace, name) + def __setattr__(self, name, value): + setattr(self.namespace, name, value) + @property def server_path(self): """File path based on ``server``.""" diff --git a/certbot/constants.py b/certbot/constants.py index 3b4fcf5be..8ef44cac9 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -18,6 +18,7 @@ CLI_DEFAULTS = dict( os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), ], + dry_run=False, verbose_count=-int(logging.INFO / 10), server="https://acme-v01.api.letsencrypt.org/directory", rsa_key_size=2048, @@ -34,6 +35,7 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", strict_permissions=False, + debug_challenges=False, ) STAGING_URI = "https://acme-staging.api.letsencrypt.org/directory" diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b615de2dd..f8f0c2105 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -6,7 +6,6 @@ """ import logging import os -import traceback import OpenSSL import pyrfc3339 @@ -49,16 +48,20 @@ def save_key(key_pem, key_dir, keyname="key-certbot.pem"): # Save file util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), config.strict_permissions) - key_f, key_path = util.unique_file( - os.path.join(key_dir, keyname), 0o600, "wb") - with key_f: - logger.info("Saving key to: %s", key_path) - key_f.write(key_pem) + if config.dry_run: + key_path = None + logger.info("Dry run, not saving private key to file") + else: + key_f, key_path = util.unique_file( + os.path.join(key_dir, keyname), 0o600, "wb") + with key_f: + key_f.write(key_pem) + logger.info("Saving private key: %s", key_path) return util.Key(key_path, key_pem) -def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): +def init_save_csr(privkey, names, path): """Initialize a CSR with the given private key. :param privkey: Key to include in the CSR @@ -74,61 +77,23 @@ def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): """ config = zope.component.getUtility(interfaces.IConfig) - csr_pem, csr_der = make_csr(privkey.pem, names, - must_staple=config.must_staple) + csr_pem = acme_crypto_util.make_csr( + privkey.pem, names, must_staple=config.must_staple) # Save CSR util.make_or_verify_dir(path, 0o755, os.geteuid(), config.strict_permissions) - csr_f, csr_filename = util.unique_file( - os.path.join(path, csrname), 0o644, "wb") - csr_f.write(csr_pem) - csr_f.close() + if config.dry_run: + csr_filename = None + logger.info("Creating CSR: not saving to file") + else: + csr_f, csr_filename = util.unique_file( + os.path.join(path, "csr-certbot.pem"), 0o644, "wb") + with csr_f: + csr_f.write(csr_pem) + logger.info("Creating CSR: %s", csr_filename) - logger.info("Creating CSR: %s", csr_filename) - - return util.CSR(csr_filename, csr_der, "der") - - -# Lower level functions -def make_csr(key_str, domains, must_staple=False): - """Generate a CSR. - - :param str key_str: PEM-encoded RSA key. - :param list domains: Domains included in the certificate. - - .. todo:: Detect duplicates in `domains`? Using a set doesn't - preserve order... - - :returns: new CSR in PEM and DER form containing all domains - :rtype: tuple - - """ - assert domains, "Must provide one or more hostnames for the CSR." - pkey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, key_str) - req = OpenSSL.crypto.X509Req() - req.get_subject().CN = domains[0] - # TODO: what to put into req.get_subject()? - # TODO: put SAN if len(domains) > 1 - extensions = [ - OpenSSL.crypto.X509Extension( - b"subjectAltName", - critical=False, - value=", ".join("DNS:%s" % d for d in domains).encode('ascii') - ) - ] - if must_staple: - extensions.append(OpenSSL.crypto.X509Extension( - b"1.3.6.1.5.5.7.1.24", - critical=False, - value=b"DER:30:03:02:01:05")) - req.add_extensions(extensions) - req.set_version(2) - req.set_pubkey(pkey) - req.sign(pkey, "sha256") - return tuple(OpenSSL.crypto.dump_certificate_request(method, req) - for method in (OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.FILETYPE_ASN1)) + return util.CSR(csr_filename, csr_pem, "pem") # WARNING: the csr and private key file are possible attack vectors for TOCTOU @@ -182,22 +147,27 @@ def import_csr_file(csrfile, data): :param str csrfile: CSR filename :param str data: contents of the CSR file - :returns: (`OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`, + :returns: (`OpenSSL.crypto.FILETYPE_PEM`, util.CSR object representing the CSR, list of domains requested in the CSR) :rtype: tuple """ - for form, typ in (("der", OpenSSL.crypto.FILETYPE_ASN1,), - ("pem", OpenSSL.crypto.FILETYPE_PEM,),): + PEM = OpenSSL.crypto.FILETYPE_PEM + load = OpenSSL.crypto.load_certificate_request + try: + # Try to parse as DER first, then fall back to PEM. + csr = load(OpenSSL.crypto.FILETYPE_ASN1, data) + except OpenSSL.crypto.Error: try: - domains = get_names_from_csr(data, typ) + csr = load(PEM, data) except OpenSSL.crypto.Error: - logger.debug("CSR parse error (form=%s, typ=%s):", form, typ) - logger.debug(traceback.format_exc()) - continue - return typ, util.CSR(file=csrfile, data=data, form=form), domains - raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) + raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) + + domains = _get_names_from_loaded_cert_or_req(csr) + # Internally we always use PEM, so re-encode as PEM before returning. + data_pem = OpenSSL.crypto.dump_certificate_request(PEM, csr) + return PEM, util.CSR(file=csrfile, data=data_pem, form="pem"), domains def make_key_rsa(bits): @@ -303,22 +273,12 @@ def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): cert, OpenSSL.crypto.load_certificate, typ) -def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): - """Get a list of Subject Alternative Names from a CSR. - - :param str csr: CSR (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` - - :returns: A list of Subject Alternative Names. - :rtype: list - - """ - return _get_sans_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate_request, typ) - - def _get_names_from_cert_or_req(cert_or_req, load_func, typ): loaded_cert_or_req = _load_cert_or_req(cert_or_req, load_func, typ) + return _get_names_from_loaded_cert_or_req(loaded_cert_or_req) + + +def _get_names_from_loaded_cert_or_req(loaded_cert_or_req): common_name = loaded_cert_or_req.get_subject().CN # pylint: disable=protected-access sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) @@ -343,20 +303,6 @@ def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): csr, OpenSSL.crypto.load_certificate, typ) -def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): - """Get a list of domains from a CSR, including the CN if it is set. - - :param str csr: CSR (encoded). - :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` - - :returns: A list of domain names. - :rtype: list - - """ - return _get_names_from_cert_or_req( - csr, OpenSSL.crypto.load_certificate_request, typ) - - def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. diff --git a/certbot/display/completer.py b/certbot/display/completer.py index 37564954a..08b55fdea 100644 --- a/certbot/display/completer.py +++ b/certbot/display/completer.py @@ -4,7 +4,7 @@ import glob try: import readline except ImportError: - import certbot.display.dummy_readline as readline + import certbot.display.dummy_readline as readline # type: ignore class Completer(object): diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 85343fdc3..4bbf9e5b8 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -192,12 +192,7 @@ def _choose_names_manually(prompt_prefix=""): try: domain_list[i] = util.enforce_domain_sanity(domain) except errors.ConfigurationError as e: - try: # Python 2 - # pylint: disable=no-member - err_msg = e.message.encode('utf-8') - except AttributeError: - err_msg = str(e) - invalid_domains[domain] = err_msg + invalid_domains[domain] = str(e) if len(invalid_domains): retry_message = ( diff --git a/certbot/hooks.py b/certbot/hooks.py index 5cda478cc..75d7a3b20 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -13,12 +13,14 @@ from certbot.plugins import util as plug_util logger = logging.getLogger(__name__) + def validate_hooks(config): """Check hook commands are executable.""" validate_hook(config.pre_hook, "pre") validate_hook(config.post_hook, "post") validate_hook(config.renew_hook, "renew") + def _prog(shell_cmd): """Extract the program run by a shell command. @@ -44,10 +46,15 @@ def validate_hook(shell_cmd, hook_name): cmd = shell_cmd.split(None, 1)[0] if not _prog(cmd): path = os.environ["PATH"] - msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format( - cmd, path, hook_name) + if os.path.exists(cmd): + msg = "{1}-hook command {0} exists, but is not executable.".format(cmd, hook_name) + else: + msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format( + cmd, path, hook_name) + raise errors.HookCommandNotFound(msg) + def pre_hook(config): "Run pre-hook if it's defined and hasn't been run." cmd = config.pre_hook @@ -58,7 +65,7 @@ def pre_hook(config): elif cmd: logger.info("Pre-hook command already run, skipping: %s", cmd) -pre_hook.already = set() +pre_hook.already = set() # type: ignore def post_hook(config): @@ -78,7 +85,8 @@ def post_hook(config): logger.info("Running post-hook command: %s", cmd) _run_hook(cmd) -post_hook.eventually = [] +post_hook.eventually = [] # type: ignore + def run_saved_post_hooks(): """Run any post hooks that were saved up in the course of the 'renew' verb""" diff --git a/certbot/interfaces.py b/certbot/interfaces.py index 48828f35e..7bcfd681c 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -32,7 +32,7 @@ class AccountStorage(object): raise NotImplementedError() @abc.abstractmethod - def save(self, account): # pragma: no cover + def save(self, account, client): # pragma: no cover """Save account. :raises .AccountStorageError: if account could not be saved @@ -99,7 +99,7 @@ class IPluginFactory(zope.interface.Interface): class IPlugin(zope.interface.Interface): """Certbot plugin.""" - def prepare(): + def prepare(): # type: ignore """Prepare the plugin. Finish up any additional initialization. @@ -118,7 +118,7 @@ class IPlugin(zope.interface.Interface): """ - def more_info(): + def more_info(): # type: ignore """Human-readable string to help the user. Should describe the steps taken and any relevant info to help the user @@ -243,6 +243,22 @@ class IConfig(zope.interface.Interface): "This only affects the port Certbot listens on. " "A conforming ACME server will still attempt to connect on port 80.") + pref_challs = zope.interface.Attribute( + "Sorted user specified preferred challenges" + "type strings with the most preferred challenge listed first") + + allow_subset_of_names = zope.interface.Attribute( + "When performing domain validation, do not consider it a failure " + "if authorizations can not be obtained for a strict subset of " + "the requested domains. This may be useful for allowing renewals for " + "multiple domains to succeed even if some domains no longer point " + "at this system. This is a boolean") + + strict_permissions = zope.interface.Attribute( + "Require that all configuration files are owned by the current " + "user; only needed if your config is somewhere unsafe like /tmp/." + "This is a boolean") + class IInstaller(IPlugin): """Generic Certbot Installer Interface. @@ -259,7 +275,7 @@ class IInstaller(IPlugin): """ - def get_all_names(): + def get_all_names(): # type: ignore """Returns all names that may be authenticated. :rtype: `collections.Iterable` of `str` @@ -296,7 +312,7 @@ class IInstaller(IPlugin): """ - def supported_enhancements(): + def supported_enhancements(): # type: ignore """Returns a `collections.Iterable` of supported enhancements. :returns: supported enhancements which should be a subset of @@ -334,7 +350,7 @@ class IInstaller(IPlugin): """ - def recovery_routine(): + def recovery_routine(): # type: ignore """Revert configuration to most recent finalized checkpoint. Remove all changes (temporary and permanent) that have not been @@ -345,21 +361,21 @@ class IInstaller(IPlugin): """ - def view_config_changes(): + def view_config_changes(): # type: ignore """Display all of the LE config changes. :raises .PluginError: when config changes cannot be parsed """ - def config_test(): + def config_test(): # type: ignore """Make sure the configuration is valid. :raises .MisconfigurationError: when the config is not in a usable state """ - def restart(): + def restart(): # type: ignore """Restart or refresh the server content. :raises .PluginError: when server cannot be restarted diff --git a/certbot/log.py b/certbot/log.py new file mode 100644 index 000000000..7660846a6 --- /dev/null +++ b/certbot/log.py @@ -0,0 +1,321 @@ +"""Logging utilities for Certbot. + +The best way to use this module is through `pre_arg_parse_setup` and +`post_arg_parse_setup`. `pre_arg_parse_setup` configures a minimal +terminal logger and ensures a detailed log is written to a secure +temporary file if Certbot exits before `post_arg_parse_setup` is called. +`post_arg_parse_setup` relies on the parsed command line arguments and +does the full logging setup with terminal and rotating file handling as +configured by the user. Any logged messages before +`post_arg_parse_setup` is called are sent to the rotating file handler. +Special care is taken by both methods to ensure all errors are logged +and properly flushed before program exit. + +""" +from __future__ import print_function +import functools +import logging +import logging.handlers +import os +import sys +import tempfile +import time +import traceback + +from acme import messages + +from certbot import constants +from certbot import errors +from certbot import util + +# Logging format +CLI_FMT = "%(message)s" +FILE_FMT = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + + +logger = logging.getLogger(__name__) + + +def pre_arg_parse_setup(): + """Setup logging before command line arguments are parsed. + + Terminal logging is setup using + `certbot.constants.QUIET_LOGGING_LEVEL` so Certbot is as quiet as + possible. File logging is setup so that logging messages are + buffered in memory. If Certbot exits before `post_arg_parse_setup` + is called, these buffered messages are written to a temporary file. + If Certbot doesn't exit, `post_arg_parse_setup` writes the messages + to the normal log files. + + This function also sets `logging.shutdown` to be called on program + exit which automatically flushes logging handlers and + `sys.excepthook` to properly log/display fatal exceptions. + + """ + temp_handler = TempHandler() + temp_handler.setFormatter(logging.Formatter(FILE_FMT)) + temp_handler.setLevel(logging.DEBUG) + memory_handler = MemoryHandler(temp_handler) + + stream_handler = ColoredStreamHandler() + stream_handler.setFormatter(logging.Formatter(CLI_FMT)) + stream_handler.setLevel(constants.QUIET_LOGGING_LEVEL) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.DEBUG) # send all records to handlers + root_logger.addHandler(memory_handler) + root_logger.addHandler(stream_handler) + + # logging.shutdown will flush the memory handler because flush() and + # close() are explicitly called + util.atexit_register(logging.shutdown) + sys.excepthook = functools.partial( + except_hook, debug='--debug' in sys.argv, log_path=temp_handler.path) + + +def post_arg_parse_setup(config): + """Setup logging after command line arguments are parsed. + + This function assumes `pre_arg_parse_setup` was called earlier and + the root logging configuration has not been modified. A rotating + file logging handler is created and the buffered log messages are + sent to that handler. Terminal logging output is set to the level + requested by the user. + + :param certbot.interface.IConfig config: Configuration object + + """ + file_handler, file_path = setup_log_file_handler( + config, 'letsencrypt.log', FILE_FMT) + logs_dir = os.path.dirname(file_path) + + root_logger = logging.getLogger() + memory_handler = stderr_handler = None + for handler in root_logger.handlers: + if isinstance(handler, ColoredStreamHandler): + stderr_handler = handler + elif isinstance(handler, MemoryHandler): + memory_handler = handler + msg = 'Previously configured logging handlers have been removed!' + assert memory_handler is not None and stderr_handler is not None, msg + + root_logger.addHandler(file_handler) + root_logger.removeHandler(memory_handler) + temp_handler = memory_handler.target + memory_handler.setTarget(file_handler) + memory_handler.close() + temp_handler.delete_and_close() + + if config.quiet: + level = constants.QUIET_LOGGING_LEVEL + else: + level = -config.verbose_count * 10 + stderr_handler.setLevel(level) + logger.debug('Root logging level set at %d', level) + logger.info('Saving debug log to %s', file_path) + + sys.excepthook = functools.partial( + except_hook, debug=config.debug, log_path=logs_dir) + + +def setup_log_file_handler(config, logfile, fmt): + """Setup file debug logging. + + :param certbot.interface.IConfig config: Configuration object + :param str logfile: basename for the log file + :param str fmt: logging format string + + :returns: file handler and absolute path to the log file + :rtype: tuple + + """ + # TODO: logs might contain sensitive data such as contents of the + # private key! #525 + util.make_or_verify_core_dir( + config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) + log_file_path = os.path.join(config.logs_dir, logfile) + try: + handler = logging.handlers.RotatingFileHandler( + log_file_path, maxBytes=2 ** 20, backupCount=1000) + except IOError as error: + raise errors.Error(util.PERM_ERR_FMT.format(error)) + # rotate on each invocation, rollover only possible when maxBytes + # is nonzero and backupCount is nonzero, so we set maxBytes as big + # as possible not to overrun in single CLI invocation (1MB). + handler.doRollover() # TODO: creates empty letsencrypt.log.1 file + handler.setLevel(logging.DEBUG) + handler_formatter = logging.Formatter(fmt=fmt) + handler_formatter.converter = time.gmtime # don't use localtime + handler.setFormatter(handler_formatter) + return handler, log_file_path + + +class ColoredStreamHandler(logging.StreamHandler): + """Sends colored logging output to a stream. + + If the specified stream is not a tty, the class works like the + standard `logging.StreamHandler`. Default red_level is + `logging.WARNING`. + + :ivar bool colored: True if output should be colored + :ivar bool red_level: The level at which to output + + """ + def __init__(self, stream=None): + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover + logging.StreamHandler.__init__(self, stream) + else: + super(ColoredStreamHandler, self).__init__(stream) + self.colored = (sys.stderr.isatty() if stream is None else + stream.isatty()) + self.red_level = logging.WARNING + + def format(self, record): + """Formats the string representation of record. + + :param logging.LogRecord record: Record to be formatted + + :returns: Formatted, string representation of record + :rtype: str + + """ + out = (logging.StreamHandler.format(self, record) + if sys.version_info < (2, 7) + else super(ColoredStreamHandler, self).format(record)) + if self.colored and record.levelno >= self.red_level: + return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) + else: + return out + + +class MemoryHandler(logging.handlers.MemoryHandler): + """Buffers logging messages in memory until the buffer is flushed. + + This differs from `logging.handlers.MemoryHandler` in that flushing + only happens when it is done explicitly by calling flush() or + close(). + + """ + def __init__(self, target=None): + # capacity doesn't matter because should_flush() is overridden + capacity = float('inf') + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover + logging.handlers.MemoryHandler.__init__( + self, capacity, target=target) + else: + super(MemoryHandler, self).__init__(capacity, target=target) + + def shouldFlush(self, record): + """Should the buffer be automatically flushed? + + :param logging.LogRecord record: log record to be considered + + :returns: False because the buffer should never be auto-flushed + :rtype: bool + + """ + return False + + +class TempHandler(logging.StreamHandler): + """Safely logs messages to a temporary file. + + The file is created with permissions 600. + + :ivar str path: file system path to the temporary log file + + """ + def __init__(self): + stream = tempfile.NamedTemporaryFile('w', delete=False) + # logging handlers use old style classes in Python 2.6 so + # super() cannot be used + if sys.version_info < (2, 7): # pragma: no cover + logging.StreamHandler.__init__(self, stream) + else: + super(TempHandler, self).__init__(stream) + self.path = stream.name + + def delete_and_close(self): + """Close the handler and delete the temporary log file.""" + self._close(delete=True) + + def close(self): + """Close the handler and the temporary log file.""" + self._close(delete=False) + + def _close(self, delete): + """Close the handler and the temporary log file. + + :param bool delete: True if the log file should be deleted + + """ + self.acquire() + try: + # StreamHandler.close() doesn't close the stream to allow a + # stream like stderr to be used + self.stream.close() + if delete: + os.remove(self.path) + if sys.version_info < (2, 7): # pragma: no cover + logging.StreamHandler.close(self) + else: + super(TempHandler, self).close() + finally: + self.release() + + +def except_hook(exc_type, exc_value, trace, debug, log_path): + """Logs fatal exceptions and reports them to the user. + + If debug is True, the full exception and traceback is shown to the + user, otherwise, it is suppressed. sys.exit is always called with a + nonzero status. + + :param type exc_type: type of the raised exception + :param BaseException exc_value: raised exception + :param traceback trace: traceback of where the exception was raised + :param bool debug: True if the traceback should be shown to the user + :param str log_path: path to file or directory containing the log + + """ + exc_info = (exc_type, exc_value, trace) + # constants.QUIET_LOGGING_LEVEL or higher should be used to + # display message the user, otherwise, a lower level like + # logger.DEBUG should be used + if debug or not issubclass(exc_type, Exception): + assert constants.QUIET_LOGGING_LEVEL <= logging.ERROR + logger.error('Exiting abnormally:', exc_info=exc_info) + else: + logger.debug('Exiting abnormally:', exc_info=exc_info) + if issubclass(exc_type, errors.Error): + sys.exit(exc_value) + print('An unexpected error occurred:', file=sys.stderr) + if messages.is_acme_error(exc_value): + # Remove the ACME error prefix from the exception + _, _, exc_str = str(exc_value).partition(':: ') + print(exc_str, file=sys.stderr) + else: + traceback.print_exception(exc_type, exc_value, None) + exit_with_log_path(log_path) + + +def exit_with_log_path(log_path): + """Print a message about the log location and exit. + + The message is printed to stderr and the program will exit with a + nonzero status. + + :param str log_path: path to file or directory containing the log + + """ + msg = 'Please see the ' + if os.path.isdir(log_path): + msg += 'logfiles in {0} '.format(log_path) + else: + msg += "logfile '{0}' ".format(log_path) + msg += 'for more details.' + sys.exit(msg) diff --git a/certbot/main.py b/certbot/main.py index 118c0f958..023c09aee 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,47 +1,37 @@ """Certbot main entry point.""" from __future__ import print_function -import atexit -import functools import logging.handlers import os import sys -import time -import traceback import zope.component from acme import jose -from acme import messages from acme import errors as acme_errors import certbot from certbot import account from certbot import cert_manager -from certbot import client from certbot import cli -from certbot import crypto_util -from certbot import colored_logging +from certbot import client from certbot import configuration from certbot import constants +from certbot import crypto_util from certbot import eff from certbot import errors from certbot import hooks from certbot import interfaces -from certbot import util -from certbot import reporter +from certbot import log from certbot import renewal +from certbot import reporter +from certbot import util from certbot.display import util as display_util, ops as display_ops from certbot.plugins import disco as plugins_disco from certbot.plugins import selection as plug_sel -_PERM_ERR_FMT = os.linesep.join(( - "The following error was encountered:", "{0}", - "If running as non-root, set --config-dir, " - "--work-dir, and --logs-dir to writeable paths.")) - USER_CANCELLED = ("User chose to cancel the operation and may " "reinvoke the client.") @@ -359,7 +349,7 @@ def _determine_account(config): acc = accounts[0] else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: - config.namespace.email = display_ops.get_email() + config.email = display_ops.get_email() def _tos_cb(regr): if config.tos: @@ -382,7 +372,7 @@ def _determine_account(config): raise errors.Error( "Unable to register an account with ACME server") - config.namespace.account = acc.id + config.account = acc.id return acc, acme @@ -417,10 +407,10 @@ def unregister(config, unused_plugins): return "Deactivation aborted." acc, acme = _determine_account(config) - acme_client = client.Client(config, acc, None, None, acme=acme) + cb_client = client.Client(config, acc, None, None, acme=acme) # delete on boulder - acme_client.acme.deactivate_registration(acc.regr) + cb_client.acme.deactivate_registration(acc.regr) account_files = account.AccountFileStorage(config) # delete local account files account_files.delete(config.account) @@ -459,14 +449,14 @@ def register(config, unused_plugins): return ("--register-unsafely-without-email provided, however, a " "new e-mail address must\ncurrently be provided when " "updating a registration.") - config.namespace.email = display_ops.get_email(optional=False) + config.email = display_ops.get_email(optional=False) acc, acme = _determine_account(config) - acme_client = client.Client(config, acc, None, None, acme=acme) + cb_client = client.Client(config, acc, None, None, acme=acme) # We rely on an exception to interrupt this process if it didn't work. - acc.regr = acme_client.acme.update_registration(acc.regr.update( + acc.regr = cb_client.acme.update_registration(acc.regr.update( body=acc.regr.body.update(contact=('mailto:' + config.email,)))) - account_storage.save_regr(acc) + account_storage.save_regr(acc, cb_client.acme) eff.handle_subscription(config) add_msg("Your e-mail address was updated to {0}.".format(config.email)) @@ -487,7 +477,7 @@ def install(config, plugins): try: installer, _ = plug_sel.choose_configurator_plugins(config, plugins, "install") except errors.PluginSelectionError as e: - return e.message + return str(e) domains, _ = _find_domains_or_certname(config, installer) le_client = _init_le_client(config, authenticator=None, installer=installer) @@ -565,7 +555,7 @@ def certificates(config, unused_plugins): def revoke(config, unused_plugins): # TODO: coop with renewal config """Revoke a previously obtained certificate.""" # For user-agent construction - config.namespace.installer = config.namespace.authenticator = "None" + config.installer = config.authenticator = "None" if config.key_path is not None: # revocation by cert key logger.debug("Revoking %s using cert key %s", config.cert_path[0], config.key_path[0]) @@ -581,7 +571,7 @@ def revoke(config, unused_plugins): # TODO: coop with renewal config try: acme.revoke(jose.ComparableX509(cert), config.reason) except acme_errors.ClientError as e: - return e.message + return str(e) display_ops.success_revocation(config.cert_path[0]) @@ -593,7 +583,7 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals try: installer, authenticator = plug_sel.choose_configurator_plugins(config, plugins, "run") except errors.PluginSelectionError as e: - return e.message + return str(e) # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) @@ -627,8 +617,8 @@ def _csr_get_and_save_cert(config, le_client): have the privkey, and therefore can't construct the files for a lineage. So we just save the cert & chain to disk :/ """ - csr, typ = config.actual_csr - certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) + csr, _ = config.actual_csr + certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr) if config.dry_run: logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) @@ -704,138 +694,11 @@ def renew(config, unused_plugins): hooks.run_saved_post_hooks() -def setup_log_file_handler(config, logfile, fmt): - """Setup file debug logging.""" - log_file_path = os.path.join(config.logs_dir, logfile) - try: - handler = logging.handlers.RotatingFileHandler( - log_file_path, maxBytes=2 ** 20, backupCount=1000) - except IOError as error: - raise errors.Error(_PERM_ERR_FMT.format(error)) - # rotate on each invocation, rollover only possible when maxBytes - # is nonzero and backupCount is nonzero, so we set maxBytes as big - # as possible not to overrun in single CLI invocation (1MB). - handler.doRollover() # TODO: creates empty letsencrypt.log.1 file - handler.setLevel(logging.DEBUG) - handler_formatter = logging.Formatter(fmt=fmt) - handler_formatter.converter = time.gmtime # don't use localtime - handler.setFormatter(handler_formatter) - return handler, log_file_path - - -def _cli_log_handler(level, fmt): - handler = colored_logging.StreamHandler() - handler.setFormatter(logging.Formatter(fmt)) - handler.setLevel(level) - return handler - - -def setup_logging(config): - """Sets up logging to logfiles and the terminal. - - :param certbot.interface.IConfig config: Configuration object - - """ - cli_fmt = "%(message)s" - file_fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" - logfile = "letsencrypt.log" - if config.quiet: - level = constants.QUIET_LOGGING_LEVEL - else: - level = -config.verbose_count * 10 - file_handler, log_file_path = setup_log_file_handler( - config, logfile=logfile, fmt=file_fmt) - cli_handler = _cli_log_handler(level, cli_fmt) - - # TODO: use fileConfig? - - root_logger = logging.getLogger() - root_logger.setLevel(logging.DEBUG) # send all records to handlers - root_logger.addHandler(cli_handler) - root_logger.addHandler(file_handler) - - logger.debug("Root logging level set at %d", level) - logger.info("Saving debug log to %s", log_file_path) - - -def _handle_exception(exc_type, exc_value, trace, config): - """Logs exceptions and reports them to the user. - - Config is used to determine how to display exceptions to the user. In - general, if config.debug is True, then the full exception and traceback is - shown to the user, otherwise it is suppressed. If config itself is None, - then the traceback and exception is attempted to be written to a logfile. - If this is successful, the traceback is suppressed, otherwise it is shown - to the user. sys.exit is always called with a nonzero status. - - """ - tb_str = "".join(traceback.format_exception(exc_type, exc_value, trace)) - logger.debug("Exiting abnormally:%s%s", os.linesep, tb_str) - - if issubclass(exc_type, Exception) and (config is None or not config.debug): - if config is None: - logfile = "certbot.log" - try: - with open(logfile, "w") as logfd: - traceback.print_exception( - exc_type, exc_value, trace, file=logfd) - assert "--debug" not in sys.argv # config is None if this explodes - except: # pylint: disable=bare-except - sys.exit(tb_str) - if "--debug" in sys.argv: - sys.exit(tb_str) - - if issubclass(exc_type, errors.Error): - sys.exit(exc_value) - else: - # Here we're passing a client or ACME error out to the client at the shell - # Tell the user a bit about what happened, without overwhelming - # them with a full traceback - err = traceback.format_exception_only(exc_type, exc_value)[0] - # Typical error from the ACME module: - # acme.messages.Error: urn:ietf:params:acme:error:malformed :: The - # request message was malformed :: Error creating new registration - # :: Validation of contact mailto:none@longrandomstring.biz failed: - # Server failure at resolver - if (messages.is_acme_error(err) and ":: " in err and - config.verbose_count <= cli.flag_default("verbose_count")): - # prune ACME error code, we have a human description - _code, _sep, err = err.partition(":: ") - msg = "An unexpected error occurred:\n" + err + "Please see the " - if config is None: - msg += "logfile '{0}' for more details.".format(logfile) - else: - msg += "logfiles in {0} for more details.".format(config.logs_dir) - sys.exit(msg) - else: - sys.exit(tb_str) - - -def make_or_verify_core_dir(directory, mode, uid, strict): - """Make sure directory exists with proper permissions. - - :param str directory: Path to a directory. - :param int mode: Directory mode. - :param int uid: Directory owner. - :param bool strict: require directory to be owned by current user - - :raises .errors.Error: if the directory cannot be made or verified - - """ - try: - util.make_or_verify_dir(directory, mode, uid, strict) - except OSError as error: - raise errors.Error(_PERM_ERR_FMT.format(error)) - def make_or_verify_needed_dirs(config): - """Create or verify existence of config, work, or logs directories""" - make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, + """Create or verify existence of config and work directories""" + util.make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) - make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) - # TODO: logs might contain sensitive data such as contents of the - # private key! #525 - make_or_verify_core_dir(config.logs_dir, 0o700, + util.make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), config.strict_permissions) @@ -851,47 +714,30 @@ def set_displayer(config): config.force_interactive) zope.component.provideUtility(displayer) -def _post_logging_setup(config, plugins, cli_args): - """Perform any setup or configuration tasks that require a logger.""" - # This needs logging, but would otherwise be in HelpfulArgumentParser - if config.validate_hooks: - hooks.validate_hooks(config) - - cli.possible_deprecation_warning(config) +def main(cli_args=sys.argv[1:]): + """Command line argument parsing and main script execution.""" + log.pre_arg_parse_setup() + plugins = plugins_disco.PluginsRegistry.find_all() logger.debug("certbot version: %s", certbot.__version__) # do not log `config`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) logger.debug("Discovered plugins: %r", plugins) - -def main(cli_args=sys.argv[1:]): - """Command line argument parsing and main script execution.""" - sys.excepthook = functools.partial(_handle_exception, config=None) - plugins = plugins_disco.PluginsRegistry.find_all() - # note: arg parser internally handles --help (and exits afterwards) args = cli.prepare_and_parse_args(plugins, cli_args) config = configuration.NamespaceConfig(args) zope.component.provideUtility(config) + log.post_arg_parse_setup(config) make_or_verify_needed_dirs(config) - - # Setup logging ASAP, otherwise "No handlers could be found for - # logger ..." TODO: this should be done before plugins discovery - setup_logging(config) - - _post_logging_setup(config, plugins, cli_args) - - sys.excepthook = functools.partial(_handle_exception, config=config) - set_displayer(config) # Reporter report = reporter.Reporter(config) zope.component.provideUtility(report) - atexit.register(report.atexit_print_messages) + util.atexit_register(report.print_messages) return config.func(config, plugins) diff --git a/certbot/ocsp.py b/certbot/ocsp.py index 8921dbb88..d34110f88 100644 --- a/certbot/ocsp.py +++ b/certbot/ocsp.py @@ -16,7 +16,7 @@ class RevocationChecker(object): self.broken = False if not util.exe_exists("openssl"): - logging.info("openssl not installed, can't check revocation") + logger.info("openssl not installed, can't check revocation") self.broken = True return @@ -61,7 +61,7 @@ class RevocationChecker(object): logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) try: - output, err = util.run_script(cmd, log=logging.debug) + output, err = util.run_script(cmd, log=logger.debug) except errors.SubprocessError: logger.info("OCSP check failed for %s (are we offline?)", cert_path) return False @@ -80,7 +80,7 @@ class RevocationChecker(object): try: url, _err = util.run_script( ["openssl", "x509", "-in", cert_path, "-noout", "-ocsp_uri"], - log=logging.debug) + log=logger.debug) except errors.SubprocessError: logger.info("Cannot extract OCSP URI from %s", cert_path) return None, None diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index e567422e2..a17f8d7b3 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -27,7 +27,7 @@ class PluginEntryPoint(object): """Distributions for which prefix will be omitted.""" # this object is mutable, don't allow it to be hashed! - __hash__ = None + __hash__ = None # type: ignore def __init__(self, entry_point): self.name = self.entry_point_to_plugin_name(entry_point) diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 81387c435..d138001e6 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -137,9 +137,8 @@ noninstaller_plugins = ["webroot", "manual", "standalone"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." - cn = config.namespace - cn.authenticator = plugins.find_init(auth).name if auth else "None" - cn.installer = plugins.find_init(inst).name if inst else "None" + config.authenticator = plugins.find_init(auth).name if auth else "None" + config.installer = plugins.find_init(inst).name if inst else "None" def choose_configurator_plugins(config, plugins, verb): diff --git a/certbot/plugins/util.py b/certbot/plugins/util.py index e45c26735..f0e2f4c5b 100644 --- a/certbot/plugins/util.py +++ b/certbot/plugins/util.py @@ -33,6 +33,6 @@ def path_surgery(cmd): return True else: expanded = " expanded" if any(added) else "" - logger.warning("Failed to find %s in%s PATH: %s", cmd, + logger.warning("Failed to find executable %s in%s PATH: %s", cmd, expanded, path) return False diff --git a/certbot/renewal.py b/certbot/renewal.py index b59d6bc33..eacfc8d29 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -80,7 +80,7 @@ def _reconstitute(config, full_path): except (ValueError, errors.Error) as error: logger.warning( "An error occurred while parsing %s. The error was %s. " - "Skipping the file.", full_path, error.message) + "Skipping the file.", full_path, str(error)) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None @@ -104,13 +104,13 @@ def _restore_webroot_config(config, renewalparams): """ if "webroot_map" in renewalparams: if not cli.set_by_cli("webroot_map"): - config.namespace.webroot_map = renewalparams["webroot_map"] + config.webroot_map = renewalparams["webroot_map"] elif "webroot_path" in renewalparams: logger.debug("Ancient renewal conf file without webroot-map, restoring webroot-path") wp = renewalparams["webroot_path"] if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] - config.namespace.webroot_path = wp + config.webroot_path = wp def _restore_plugin_configs(config, renewalparams): @@ -149,10 +149,10 @@ def _restore_plugin_configs(config, renewalparams): if config_value in ("None", "True", "False"): # bool("False") == True # pylint: disable=eval-used - setattr(config.namespace, config_item, eval(config_value)) + setattr(config, config_item, eval(config_value)) else: cast = cli.argparse_type(config_item) - setattr(config.namespace, config_item, cast(config_value)) + setattr(config, config_item, cast(config_value)) def restore_required_config_elements(config, renewalparams): @@ -173,7 +173,7 @@ def restore_required_config_elements(config, renewalparams): for item_name, restore_func in required_items: if item_name in renewalparams and not cli.set_by_cli(item_name): value = restore_func(item_name, renewalparams[item_name]) - setattr(config.namespace, item_name, value) + setattr(config, item_name, value) def _restore_pref_challs(unused_name, value): diff --git a/certbot/reporter.py b/certbot/reporter.py index 118b13166..e0063d8e5 100644 --- a/certbot/reporter.py +++ b/certbot/reporter.py @@ -3,11 +3,10 @@ from __future__ import print_function import collections import logging -import os import sys import textwrap -from six.moves import queue # pylint: disable=import-error +from six.moves import queue # type: ignore # pylint: disable=import-error import zope.interface from certbot import interfaces @@ -16,11 +15,6 @@ from certbot import util logger = logging.getLogger(__name__) -# Store the pid of the process that first imported this module so that -# atexit_print_messages side-effects such as error reporting can be limited to -# this process and not any fork()'d children. -INITIAL_PID = os.getpid() - @zope.interface.implementer(interfaces.IReporter) class Reporter(object): @@ -60,19 +54,6 @@ class Reporter(object): self.messages.put(self._msg_type(priority, msg, on_crash)) logger.debug("Reporting to user: %s", msg) - def atexit_print_messages(self, pid=None): - """Function to be registered with atexit to print messages. - - :param int pid: Process ID - - """ - if pid is None: - pid = INITIAL_PID - # This ensures that messages are only printed from the process that - # created the Reporter. - if pid == os.getpid(): - self.print_messages() - def print_messages(self): """Prints messages to the user and clears the message queue. diff --git a/certbot/storage.py b/certbot/storage.py index dacc73c4c..c35a268e3 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -19,6 +19,9 @@ from certbot import errors from certbot import error_handler from certbot import util +from certbot.plugins import common as plugins_common +from certbot.plugins import disco as plugins_disco + logger = logging.getLogger(__name__) ALL_FOUR = ("cert", "privkey", "chain", "fullchain") @@ -27,7 +30,14 @@ CURRENT_VERSION = util.get_strict_version(certbot.__version__) def renewal_conf_files(config): - """Return /path/to/*.conf in the renewal conf directory""" + """Build a list of all renewal configuration files. + + :param certbot.interfaces.IConfig config: Configuration object + + :returns: list of renewal configuration files + :rtype: `list` of `str` + + """ return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) def renewal_file_for_certname(config, certname): @@ -179,13 +189,12 @@ def _relevant(option): :rtype: bool """ - # The list() here produces a list of the plugin names as strings. from certbot import renewal - from certbot.plugins import disco as plugins_disco - plugins = list(plugins_disco.PluginsRegistry.find_all()) + plugins = plugins_disco.PluginsRegistry.find_all() + namespaces = [plugins_common.dest_namespace(plugin) for plugin in plugins] return (option in renewal.CONFIG_ITEMS or - any(option.startswith(x + "_") for x in plugins)) + any(option.startswith(namespace) for namespace in namespaces)) def relevant_values(all_values): @@ -1038,10 +1047,10 @@ class RenewableCert(object): is regarded as a successor (used to choose a privkey, if the key has not changed, but otherwise this information is not permanently recorded anywhere) - :param str new_cert: the new certificate, in PEM format - :param str new_privkey: the new private key, in PEM format, + :param bytes new_cert: the new certificate, in PEM format + :param bytes new_privkey: the new private key, in PEM format, or ``None``, if the private key has not changed - :param str new_chain: the new chain, in PEM format + :param bytes new_chain: the new chain, in PEM format :param .NamespaceConfig cli_config: parsed command line arguments @@ -1077,18 +1086,18 @@ class RenewableCert(object): logger.debug("Writing symlink to old private key, %s.", old_privkey) os.symlink(old_privkey, target["privkey"]) else: - with open(target["privkey"], "w") as f: + with open(target["privkey"], "wb") as f: logger.debug("Writing new private key to %s.", target["privkey"]) f.write(new_privkey) # Save everything else - with open(target["cert"], "w") as f: + with open(target["cert"], "wb") as f: logger.debug("Writing certificate to %s.", target["cert"]) f.write(new_cert) - with open(target["chain"], "w") as f: + with open(target["chain"], "wb") as f: logger.debug("Writing chain to %s.", target["chain"]) f.write(new_chain) - with open(target["fullchain"], "w") as f: + with open(target["fullchain"], "wb") as f: logger.debug("Writing full chain to %s.", target["fullchain"]) f.write(new_cert + new_chain) diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 7367717bf..5950dcda9 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -1,9 +1,9 @@ """Tests for certbot.account.""" import datetime +import json import os import shutil import stat -import tempfile import unittest import mock @@ -16,6 +16,8 @@ from certbot import errors from certbot.tests import util +from certbot.tests.util import TempDirTestCase + KEY = jose.JWKRSA.load(util.load_vector("rsa512_key_2.pem")) @@ -31,6 +33,7 @@ class AccountTest(unittest.TestCase): creation_dt=datetime.datetime( 2015, 7, 4, 14, 4, 10, tzinfo=pytz.UTC)) self.acc = Account(self.regr, KEY, self.meta) + self.regr.__repr__ = mock.MagicMock(return_value="i_am_a_regr") with mock.patch("certbot.account.socket") as mock_socket: mock_socket.getfqdn.return_value = "test.certbot.org" @@ -52,10 +55,8 @@ class AccountTest(unittest.TestCase): self.acc.slug, "test.certbot.org@2015-07-04T14:04:10Z (bca5)") def test_repr(self): - self.assertEqual( - repr(self.acc), - "") - + self.assertTrue(repr(self.acc).startswith( + "`. If you're running on Linux, you can run the following commands to +install dependencies and set up a virtual environment where you can run +Certbot. You will need to repeat this when Certbot's dependencies change or when +a new plugin is introduced. + +.. code-block:: shell + cd certbot - ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only + ./certbot-auto --os-packages-only ./tools/venv.sh Then in each shell where you're working on the client, do: @@ -30,17 +40,18 @@ Then in each shell where you're working on the client, do: .. code-block:: shell source ./venv/bin/activate + export SERVER=https://acme-staging.api.letsencrypt.org/directory + source tests/integration/_common.sh After that, your shell will be using the virtual environment, and you run the -client by typing: +client by typing `certbot` or `certbot_test`. The latter is an alias that +includes several flags useful for testing. For instance, it sets various output +directories to point to /tmp/, and uses non-privileged ports for challenges, so +root privileges are not required. -.. code-block:: shell - - certbot - -Activating a shell in this way makes it easier to run unit tests -with ``tox`` and integration tests, as described below. To reverse this, you -can type ``deactivate``. More information can be found in the `virtualenv docs`_. +Activating a shell with `venv/bin/activate` sets environment variables so that +Python pulls in the correct versions of various packages needed by Certbot. +More information can be found in the `virtualenv docs`_. .. _`virtualenv docs`: https://virtualenv.pypa.io @@ -69,10 +80,8 @@ either in the same directory as ``foo.py`` or in the ``tests`` subdirectory (if there isn't, make one). While you are working on your code and tests, run ``python foo_test.py`` to run the relevant tests. -For debugging, we recommend running ``pip install ipdb`` and putting -``import ipdb; ipdb.set_trace()`` statements inside the source -code. Alternatively, you can use Python's standard library `pdb`, -but you won't get TAB completion. +For debugging, we recommend putting +``import ipdb; ipdb.set_trace()`` statements inside the source code. Once you are done with your code changes, and the tests in ``foo_test.py`` pass, run all of the unittests for Certbot with ``tox -e py27`` (this uses Python @@ -109,6 +118,14 @@ and working. Fetch and start Boulder using: If you have problems with Docker, you may want to try `removing all containers and volumes`_ and making sure you have at least 1GB of memory. +Set up a certbot_test alias that enables easily running against the local +Boulder: + +.. code-block:: shell + + export SERVER=http://localhost:4000/directory + source tests/integration/_common.sh + Run the integration tests using: .. code-block:: shell @@ -138,13 +155,15 @@ different webservers, other TLS servers, and operating systems. The interfaces available for plugins to implement are defined in `interfaces.py`_ and `plugins/common.py`_. -The most common kind of plugin is a "Configurator", which is likely to -implement the `~certbot.interfaces.IAuthenticator` and -`~certbot.interfaces.IInstaller` interfaces (though some -Configurators may implement just one of those). +The main two plugin interfaces are `~certbot.interfaces.IAuthenticator`, which +implements various ways of proving domain control to a certificate authority, +and `~certbot.interfaces.IInstaller`, which configures a server to use a +certificate once it is issued. Some plugins, like the built-in Apache and Nginx +plugins, implement both interfaces and perform both tasks. Others, like the +built-in Standalone authenticator, implement just one interface. There are also `~certbot.interfaces.IDisplay` plugins, -which implement bindings to alternative UI libraries. +which can change how prompts are displayed to a user. .. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py .. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 @@ -153,27 +172,20 @@ which implement bindings to alternative UI libraries. Authenticators -------------- -Authenticators are plugins designed to prove that this client deserves a -certificate for some domain name by solving challenges received from -the ACME server. From the protocol, there are essentially two -different types of challenges. Challenges that must be solved by -individual plugins in order to satisfy domain validation (subclasses -of `~.DVChallenge`, i.e. `~.challenges.TLSSNI01`, -`~.challenges.HTTP01`, `~.challenges.DNS`) and continuity specific -challenges (subclasses of `~.ContinuityChallenge`, -i.e. `~.challenges.RecoveryToken`, `~.challenges.RecoveryContact`, -`~.challenges.ProofOfPossession`). Continuity challenges are -always handled by the `~.ContinuityAuthenticator`, while plugins are -expected to handle `~.DVChallenge` types. -Right now, we have two authenticator plugins, the `~.ApacheConfigurator` -and the `~.StandaloneAuthenticator`. The Standalone and Apache -authenticators only solve the `~.challenges.TLSSNI01` challenge currently. -(You can set which challenges your authenticator can handle through the -:meth:`~.IAuthenticator.get_chall_pref`. +Authenticators are plugins that prove control of a domain name by solving a +challenge provided by the ACME server. ACME currently defines three types of +challenges: HTTP, TLS-SNI, and DNS, represented by classes in `acme.challenges`. +An authenticator plugin should implement support for at least one challenge type. -(FYI: We also have a partial implementation for a `~.DNSAuthenticator` -in a separate branch). +An Authenticator indicates which challenges it supports by implementing +`get_chall_pref(domain)` to return a sorted list of challenge types in +preference order. +An Authenticator must also implement `perform(achalls)`, which "performs" a list +of challenges by, for instance, provisioning a file on an HTTP server, or +setting a TXT record in DNS. Once all challenges have succeeded or failed, +Certbot will call the plugin's `cleanup(achalls)` method to remove any files or +DNS records that were needed only during authentication. Installer --------- @@ -211,24 +223,40 @@ Augeas may still find the `~.Reverter` class helpful in handling configuration checkpoints and rollback. -Display -~~~~~~~ - -We currently only offer a "text" mode for displays. Display plugins -implement the `~certbot.interfaces.IDisplay` interface. - .. _dev-plugin: Writing your own plugin -======================= +~~~~~~~~~~~~~~~~~~~~~~~ Certbot client supports dynamic discovery of plugins through the -`setuptools entry points`_. This way you can, for example, create a -custom implementation of `~certbot.interfaces.IAuthenticator` or -the `~certbot.interfaces.IInstaller` without having to merge it +`setuptools entry points`_ using the `certbot.plugins` group. This +way you can, for example, create a custom implementation of +`~certbot.interfaces.IAuthenticator` or the +`~certbot.interfaces.IInstaller` without having to merge it with the core upstream source code. An example is provided in ``examples/plugins/`` directory. +While developing, you can install your plugin into a Certbot development +virtualenv like this: + +.. code-block:: shell + + . venv/bin/activate + . tests/integration/_common.sh + pip install -e examples/plugins/ + certbot_test plugins + +Your plugin should show up in the output of the last command. If not, +it was not installed properly. + +Once you've finished your plugin and published it, you can have your +users install it system-wide with `pip install`. Note that this will +only work for users who have Certbot installed from OS packages or via +pip. Users who run `certbot-auto` are currently unable to use third-party +plugins. It's technically possible to install third-party plugins into +the virtualenv used by `certbot-auto`, but they will be wiped away when +`certbot-auto` upgrades. + .. warning:: Please be aware though that as this client is still in a developer-preview stage, the API may undergo a few changes. If you believe the plugin will be beneficial to the community, please @@ -285,8 +313,7 @@ Steps: including coverage. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. 5. If your code touches communication with an ACME server/Boulder, you - should run the integration tests, see `integration`_. See `Known Issues`_ - for some common failures that have nothing to do with your code. + should run the integration tests, see `integration`_. 6. Submit the PR. 7. Did your tests pass on Travis? If they didn't, fix any errors. @@ -346,56 +373,36 @@ This should generate documentation in the ``docs/_build/html`` directory. -Other methods for running the client -==================================== +.. _docker: -Vagrant -------- +Running the client with Docker +============================== -If you are a Vagrant user, Certbot comes with a Vagrantfile that -automates setting up a development environment in an Ubuntu 14.04 -LTS VM. To set it up, simply run ``vagrant up``. The repository is -synced to ``/vagrant``, so you can get started with: +You can use Docker Compose to quickly set up an environment for running and +testing Certbot. This is especially useful for macOS users. To install Docker +Compose, follow the instructions at https://docs.docker.com/compose/install/. -.. code-block:: shell +.. note:: Linux users can simply run ``pip install docker-compose`` to get + Docker Compose after installing Docker Engine and activating your shell as + described in the :ref:`Getting Started ` section. - vagrant ssh - cd /vagrant - sudo ./venv/bin/certbot +Now you can develop on your host machine, but run Certbot and test your changes +in Docker. When using ``docker-compose`` make sure you are inside your clone of +the Certbot repository. As an example, you can run the following command to +check for linting errors:: -Support for other Linux distributions coming soon. + docker-compose run --rm --service-ports development bash -c 'tox -e lint' -.. note:: - Unfortunately, Python distutils and, by extension, setup.py and - tox, use hard linking quite extensively. Hard linking is not - supported by the default sync filesystem in Vagrant. As a result, - all actions with these commands are *significantly slower* in - Vagrant. One potential fix is to `use NFS`_ (`related issue`_). +You can also leave a terminal open running a shell in the Docker container and +modify Certbot code in another window. The Certbot repo on your host machine is +mounted inside of the container so any changes you make immediately take +effect. To do this, run:: -.. _use NFS: http://docs.vagrantup.com/v2/synced-folders/nfs.html -.. _related issue: https://github.com/ClusterHQ/flocker/issues/516 + docker-compose run --rm --service-ports development bash +Now running the check for linting errors described above is as easy as:: -Docker ------- - -OSX users will probably find it easiest to set up a Docker container for -development. Certbot comes with a Dockerfile (``Dockerfile-dev``) -for doing so. To use Docker on OSX, install and setup docker-machine using the -instructions at https://docs.docker.com/installation/mac/. - -To build the development Docker image:: - - docker build -t certbot -f Dockerfile-dev . - -Now run tests inside the Docker image: - -.. code-block:: shell - - docker run -it certbot bash - cd src - tox -e py27 - + tox -e lint .. _prerequisites: @@ -432,8 +439,12 @@ For squeeze you will need to: FreeBSD ------- -Package installation for FreeBSD uses ``pkg``, not ports. +Packages can be installed on FreeBSD using ``pkg``, +or any other port-management tool (``portupgrade``, ``portmanager``, etc.) +from the pre-built package or can be built and installed from ports. +Either way will ensure proper installation of all the dependencies required +for the package. FreeBSD by default uses ``tcsh``. In order to activate virtualenv (see -below), you will need a compatible shell, e.g. ``pkg install bash && +above), you will need a compatible shell, e.g. ``pkg install bash && bash``. diff --git a/docs/install.rst b/docs/install.rst index aa59e44ec..6c56584be 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,9 +22,8 @@ your system. System Requirements =================== -The Let's Encrypt Client presently only runs on Unix-ish OSes that include -Python 2.6 or 2.7; Python 3.x support will hopefully be added in the future. The -client requires root access in order to write to ``/etc/letsencrypt``, +Certbot currently requires Python 2.6 or 2.7. By default, it requires root +access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver configurations (if you use the ``apache`` or ``nginx`` plugins). If none of @@ -33,11 +32,16 @@ but for most users who want to avoid running an ACME client as root, either `letsencrypt-nosudo `_ or `simp_le `_ are more appropriate choices. -The Apache plugin currently requires OS with augeas version 1.0; currently `it +The Apache plugin currently requires an OS with augeas version 1.0; currently `it supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. +Installing with ``certbot-auto`` requires 512MB of RAM in order to build some +of the dependencies. Installing from pre-built OS packages avoids this +requirement. You can also temporarily set a swap file. See "Problems with +Python virtual environment" below for details. + Alternate installation methods ================================ @@ -76,7 +80,7 @@ For full command line help, you can type:: Problems with Python virtual environment ---------------------------------------- -On a low memory system such as VPS with only 256MB of RAM, the required dependencies of Certbot will failed to build. +On a low memory system such as VPS with less than 512MB of RAM, the required dependencies of Certbot will failed to build. This can be identified if the pip outputs contains something like ``internal compiler error: Killed (program cc1)``. You can workaround this restriction by creating a temporary swapfile:: @@ -120,7 +124,7 @@ to, `install Docker`_, then issue the following command: sudo docker run -it --rm -p 443:443 -p 80:80 --name certbot \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ - quay.io/letsencrypt/letsencrypt:latest certonly + certbot/certbot certonly Running Certbot with the ``certonly`` command will obtain a certificate and place it in the directory ``/etc/letsencrypt/live`` on your system. Because Certonly cannot install the certificate from @@ -136,16 +140,6 @@ of the ``/etc/letsencrypt`` directory, see :ref:`where-certs`. Operating System Packages ------------------------- -**FreeBSD** - - * Port: ``cd /usr/ports/security/py-certbot && make install clean`` - * Package: ``pkg install py27-certbot`` - -**OpenBSD** - - * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` - * Package: ``pkg_add letsencrypt`` - **Arch Linux** .. code-block:: shell @@ -178,6 +172,11 @@ repo, if you have not already done so. Then run: sudo dnf install certbot python2-certbot-apache +**FreeBSD** + + * Port: ``cd /usr/ports/security/py-certbot && make install clean`` + * Package: ``pkg install py27-certbot`` + **Gentoo** The official Certbot client is available in Gentoo Portage. If you @@ -213,6 +212,16 @@ For the time being, this is the only way for the Apache plugin to recognise the appropriate directives when installing the certificate. Note: this change is not required for the other plugins. +**NetBSD** + + * Build from source: ``cd /usr/pkgsrc/security/py-certbot && make install clean`` + * Install pre-compiled package: ``pkg_add py27-certbot`` + +**OpenBSD** + + * Port: ``cd /usr/ports/security/letsencrypt/client && make install clean`` + * Package: ``pkg_add letsencrypt`` + **Other Operating Systems** OS packaging is an ongoing effort. If you'd like to package diff --git a/docs/packaging.rst b/docs/packaging.rst index 1a1b83f50..5b37c2e24 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -16,21 +16,19 @@ The following scripts are used in the process: - https://github.com/letsencrypt/letsencrypt/blob/master/tools/release.sh -We currently version with the following scheme: +We use git tags to identify releases, using `Semantic Versioning`_. For +example: `v0.11.1`. -- ``0.1.0`` -- ``0.2.0dev`` for developement in ``master`` -- ``0.2.0`` (only temporarily in ``master``) -- ... +.. _`Semantic Versioning`: http://semver.org/ Notes for package maintainers ============================= -0. Please use our releases, not ``master``! +0. Please use our tagged releases, not ``master``! 1. Do not package ``certbot-compatibility-test`` or ``letshelp-certbot`` - it's only used internally. -2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. +2. If you'd like to include automated renewal in your package ``certbot renew -q`` should be added to crontab or systemd timer. Additionally you should include a random per-machine time offset to avoid having a large number of your clients hit Let's Encrypt's servers simultaneously. 3. ``jws`` is an internal script for ``acme`` module and it doesn't have to be packaged - it's mostly for debugging: you can use it as ``echo foo | jws sign | jws verify``. @@ -44,34 +42,43 @@ Arch ---- From our official releases: + - https://www.archlinux.org/packages/community/any/python2-acme - https://www.archlinux.org/packages/community/any/certbot - https://www.archlinux.org/packages/community/any/certbot-apache - https://www.archlinux.org/packages/community/any/certbot-nginx -- https://www.archlinux.org/packages/community/any/letshelp-certbot From ``master``: https://aur.archlinux.org/packages/certbot-git Debian (and its derivatives, including Ubuntu) ------ -https://packages.debian.org/sid/certbot -https://packages.debian.org/sid/python-certbot -https://packages.debian.org/sid/python-certbot-apache +- https://packages.debian.org/sid/certbot +- https://packages.debian.org/sid/python-certbot +- https://packages.debian.org/sid/python-certbot-apache Fedora ------ In Fedora 23+. -- https://admin.fedoraproject.org/pkgdb/package/letsencrypt/ - https://admin.fedoraproject.org/pkgdb/package/certbot/ - https://admin.fedoraproject.org/pkgdb/package/python-acme/ FreeBSD ------- -https://svnweb.freebsd.org/ports/head/security/py-certbot/ +- https://svnweb.freebsd.org/ports/head/security/py-certbot/ + +Gentoo +------ + +Currently, all ``certbot`` related packages are in the testing branch: + +- https://packages.gentoo.org/packages/app-crypt/certbot +- https://packages.gentoo.org/packages/app-crypt/certbot-apache +- https://packages.gentoo.org/packages/app-crypt/certbot-nginx +- https://packages.gentoo.org/packages/app-crypt/acme GNU Guix -------- diff --git a/docs/using.rst b/docs/using.rst index 628043ff9..7eaa92f84 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -144,6 +144,10 @@ the ``--nginx`` flag on the commandline. Standalone ---------- +Use standalone mode to obtain a cert if you don't want to use (or don't currently have) +existing server software. The standalone plugin does not rely on any other server +software running on the machine where you obtain the cert. + To obtain a cert using a "standalone" webserver, you can use the standalone plugin by including ``certonly`` and ``--standalone`` on the command line. This plugin needs to bind to port 80 or 443 in @@ -151,13 +155,14 @@ order to perform domain validation, so you may need to stop your existing webserver. To control which port the plugin uses, include one of the options shown below on the command line. - * ``--standalone-supported-challenges http-01`` to use port 80 - * ``--standalone-supported-challenges tls-sni-01`` to use port 443 + * ``--preferred-challenges http`` to use port 80 + * ``--preferred-challenges tls-sni`` to use port 443 -The standalone plugin does not rely on any other server software running -on the machine where you obtain the certificate. It must still be possible -for that machine to accept inbound connections from the Internet on the -specified port using each requested domain name. +It must still be possible for your machine to accept inbound connections from +the Internet on the specified port using each requested domain name. + +.. note:: The ``--standalone-supported-challenges`` option has been + deprecated since ``certbot`` version 0.9.0. Manual ------ @@ -170,6 +175,23 @@ the UI, you can use the plugin to obtain a cert by specifying to copy and paste commands into another terminal session, which may be on a different computer. +The manual plugin can use either the ``http`` or the ``dns`` challenge. You +can use the ``--preferred-challenges`` option to choose the challenge of your +preference. +The ``http`` challenge will ask you to place a file with a specific name and +specific content in the ``/.well-known/acme-challenge/`` directory directly +in the top-level directory (“web root”) containing the files served by your +webserver. In essence it's the same as the webroot_ plugin, but not automated. +When using the ``dns`` plugin, ``certbot`` will ask you to place a TXT DNS +record with specific contents under the domain name consisting of the hostname +for which you want a certificate issued, prepended by ``_acme-challenge``. + +For example, for the domain ``example.com``, a zone file entry would look like: + +:: + + _acme-challenge.example.com. 300 IN TXT "gfj9Xq...Rg85nM" + Additionally you can specify scripts to prepare for validation and perform the authentication procedure and/or clean up after it by using the ``--manual-auth-hook`` and ``--manual-cleanup-hook`` flags. This is described in @@ -356,10 +378,59 @@ then restart it after the plugin is finished. Example:: certbot renew --pre-hook "service nginx stop" --post-hook "service nginx start" -The hooks will only be -run if a certificate is due for renewal, so you can run this command -frequently without unnecessarily stopping your webserver. More -information about renewal hooks can be found by running +If a hook exits with a non-zero exit code, the error will be printed +to ``stderr`` but renewal will be attempted anyway. A failing hook +doesn't directly cause Certbot to exit with a non-zero exit code, but +since Certbot exits with a non-zero exit code when renewals fail, a +failed hook causing renewal failures will indirectly result in a +non-zero exit code. Hooks will only be run if a certificate is due for +renewal, so you can run the above command frequently without +unnecessarily stopping your webserver. + +``--pre-hook`` and ``--post-hook`` hooks run before and after every renewal +attempt. If you want your hook to run only after a successful renewal, use +``--renew-hook`` in a command like this. + +``certbot renew --renew-hook /path/to/renew-hook-script`` + +For example, if you have a daemon that does not read its certificates as the +root user, a renew hook like this can copy them to the correct location and +apply appropriate file permissions. + +/path/to/renew-hook-script + +.. code-block:: none + + #!/bin/sh + + set -e + + for domain in $RENEWED_DOMAINS; do + case $domain in + example.com) + daemon_cert_root=/etc/some-daemon/certs + + # Make sure the certificate and private key files are + # never world readable, even just for an instant while + # we're copying them into daemon_cert_root. + umask 077 + + cp "$RENEWED_LINEAGE/fullchain.pem" "$daemon_cert_root/$domain.cert" + cp "$RENEWED_LINEAGE/privkey.pem" "$daemon_cert_root/$domain.key" + + # Apply the proper file ownership and permissions for + # the daemon to read its certificate and key. + chown some-daemon "$daemon_cert_root/$domain.cert" \ + "$daemon_cert_root/$domain.key" + chmod 400 "$daemon_cert_root/$domain.cert" \ + "$daemon_cert_root/$domain.key" + + service some-daemon restart >/dev/null + ;; + esac + done + + More information about renewal hooks can be found by running ``certbot --help renew``. If you're sure that this command executes successfully without human @@ -409,6 +480,13 @@ Certbot is working hard to improve the renewal process, and we apologize for any inconvenience you encounter in integrating these commands into your individual environment. +.. note:: ``certbot renew`` exit status will only be 1 if a renewal attempt failed. + This means ``certbot renew`` exit status will be 0 if no cert needs to be updated. + If you write a custom script and expect to run a command only after a cert was actually renewed + you will need to use the ``--post-hook`` since the exit status will be 0 both on successful renewal + and when renewal is not necessary. + + Modifying the Renewal Configuration File ---------------------------------------- diff --git a/examples/cli.ini b/examples/cli.ini index 63af3cc49..dbaa9c599 100644 --- a/examples/cli.ini +++ b/examples/cli.ini @@ -1,6 +1,11 @@ # This is an example of the kind of things you can do in a configuration file. # All flags used by the client can be configured here. Run Certbot with # "--help" to learn more about the available options. +# +# Note that these options apply automatically to all use of Certbot for +# obtaining or renewing certificates, so options specific to a single +# certificate on a system with several certificates should not be placed +# here. # Use a 4096 bit RSA key instead of 2048 rsa-key-size = 4096 @@ -8,13 +13,6 @@ rsa-key-size = 4096 # Uncomment and update to register with the specified e-mail address # email = foo@example.com -# Uncomment and update to generate certificates for the specified -# domains. -# domains = example.com, www.example.com - -# Uncomment to use a text interface instead of ncurses -# text = True - # Uncomment to use the standalone authenticator on port 443 # authenticator = standalone # standalone-supported-challenges = tls-sni-01 diff --git a/letsencrypt-auto b/letsencrypt-auto index ac721638f..fc8007c9e 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -23,7 +23,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.11.1" +LE_AUTO_VERSION="0.13.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -36,6 +36,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,6 +55,8 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; --noninteractive|--non-interactive) @@ -160,17 +163,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else echo "FATAL: $1 support is very experimental at present..." echo "if you would like to work on improving it, please ensure you have backups" echo "and then run this script again with the --debug flag!" + echo "Alternatively, you can install OS dependencies yourself and run this script" + echo "again with --no-bootstrap." exit 1 fi } @@ -576,21 +586,23 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" @@ -615,7 +627,7 @@ Bootstrap() { else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." exit 1 @@ -833,18 +845,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1093,6 +1105,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1209,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 44e2246b7..bf267dc25 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -Version: GnuPG v1 +Version: GnuPG v2 -iQEcBAABAgAGBQJYkqlBAAoJEE0XyZXNl3XyXUAH/RqwKDOpceChSdH/aIk891HX -VRDBQxIjZ6EB1iyebfihyZd4a5zGJ9ocMj1GxThMyLgSKbgkSRtjE/+ymDWsL0Us -Y8w9fw76BAImaJZEvjkrpqD2bSYdijnF479hBa/huZHKcQhb/sqxkNJO9SO1uj8z -bnF0UjJNjgn1hm2yHNMWwyEX7xCN/Vxiq/Zwqi7HdPus99sInJA7+04nwXaUtash -87MHpCjIiHh3axOCOjJbAWzIfsDUKeaBHgeYO+2ldOPWVQ0Amp7ghXjohryBkiux -dqhhAuvBTmNqPrbPAjdJ7Kd74NOGDo3HvAUiuXIckDWqxX2Q34w5pwxelZcIEnI= -=vmVS +iQEcBAABCAAGBQJY5WxEAAoJEE0XyZXNl3XyoDYH/joyJ/7cS4+SoTEiPpVcDnK+ +YJVhxP6pir6GaRvl+ebWlo7ichS4c0Kye8e5BPVj5RtZbDT88iplMZ2EyUmeA579 +8Z96p9qoEANeGWiPe+KCDXRHJfCAsphcHSLTeS8lXgG8SP13p7hsML6hn3gosRdu +OG4/SnFBDLLwu4YwUVom4U+Z+dYS1jQstge4sexr85jCX/Lds7M5WM/lFiYMBsJ8 +uZd/IGKwb7jvsc4u58Ruj9xiTcchaxn15NMJR7R967Mt5ortSvZ3C6Cv3NyubJmB +hmGQVU+eNBTeEwPSIN8xAf3fcwh2wlRMaTZOy5nJ3IoDdSQuwO9IGxxdkNDSegE= +=8KUq -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 81e3862c2..50c80f0a4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi @@ -23,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.12.0.dev0" +LE_AUTO_VERSION="0.14.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -36,6 +41,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,9 +60,11 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; @@ -90,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -97,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -142,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -160,17 +178,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -181,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -217,7 +242,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -245,7 +270,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -305,7 +330,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } @@ -325,7 +350,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -339,7 +364,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -351,7 +376,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -393,7 +418,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } @@ -498,15 +523,15 @@ BootstrapFreeBsd() { BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -517,26 +542,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } @@ -556,7 +581,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -568,7 +593,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } @@ -576,28 +601,30 @@ BootstrapMageiaCommon() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -613,11 +640,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -637,7 +664,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -645,7 +672,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -654,7 +681,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -833,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1010,42 +1037,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -1072,7 +1097,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -1093,6 +1118,9 @@ else On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -1194,12 +1222,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 @@ -1212,9 +1240,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -1223,7 +1251,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7aea42dd0..723cc2f8c 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 327799210..f6585a378 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi @@ -36,6 +41,7 @@ Help for certbot itself cannot be provided until it is installed. --debug attempt experimental installation -h, --help print this help -n, --non-interactive, --noninteractive run without asking for user input + --no-bootstrap do not install OS dependencies --no-self-upgrade do not download updates --os-packages-only install OS dependencies and exit -v, --verbose provide more output @@ -54,9 +60,11 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --no-bootstrap) + NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; @@ -90,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -97,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -142,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -160,17 +178,24 @@ else fi fi +BootstrapMessage() { + # Arguments: Platform name + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" +} + ExperimentalBootstrap() { # Arguments: Platform name, bootstrap function name if [ "$DEBUG" = 1 ]; then if [ "$2" != "" ]; then - echo "Bootstrapping dependencies via $1..." + BootstrapMessage $1 $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -181,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -206,28 +231,30 @@ DeterminePythonVersion() { # Install required OS packages: Bootstrap() { - if [ -f /etc/debian_version ]; then - echo "Bootstrapping dependencies for Debian-based OSes..." + if [ "$NO_BOOTSTRAP" = 1 ]; then + return + elif [ -f /etc/debian_version ]; then + BootstrapMessage "Debian-based OSes" BootstrapDebCommon - elif [ -f /etc/mageia-release ] ; then + elif [ -f /etc/mageia-release ]; then # Mageia has both /etc/mageia-release and /etc/redhat-release ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then - echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapMessage "RedHat-based OSes" BootstrapRpmCommon elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then - echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapMessage "openSUSE-based OSes" BootstrapSuseCommon elif [ -f /etc/arch-release ]; then if [ "$DEBUG" = 1 ]; then - echo "Bootstrapping dependencies for Archlinux..." + BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -243,11 +270,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to bootstrap, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -267,7 +294,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -275,7 +302,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -284,7 +311,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -309,42 +336,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -371,7 +396,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -385,9 +410,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -396,7 +421,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index 7735933c4..afd279ac2 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -21,7 +21,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -49,7 +49,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -109,7 +109,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh index c101be7d7..b88e96999 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh @@ -1,14 +1,14 @@ BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -19,26 +19,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh index dd1213a4c..1c76bbcac 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh @@ -8,7 +8,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -20,7 +20,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 44c4625d9..dcd535292 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -13,7 +13,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -27,7 +27,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -39,7 +39,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -81,7 +81,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index 365a5a36a..e7ebb9e0a 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -10,6 +10,9 @@ On failure, return non-zero. """ + +from __future__ import print_function + from distutils.version import LooseVersion from json import loads from os import devnull, environ @@ -111,12 +114,12 @@ def main(): flag = argv[1] try: if flag == '--latest-version': - print latest_stable_version(get) + print(latest_stable_version(get)) elif flag == '--le-auto-script': tag = argv[2] verified_new_le_auto(get, tag, dirname(argv[0])) except ExpectedError as exc: - print exc.args[0], exc.args[1] + print(exc.args[0], exc.args[1]) return 1 else: return 0 diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index 36d93dddd..325bdf84f 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -171,15 +171,15 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.11.1 \ - --hash=sha256:9f4efac6dc4477a3baa7eb2392d4f7583f974e4ad336439aa1961ef805622a77 \ - --hash=sha256:db35258edfc13dfe5839215898fe2d5d3caafc9a084f631a032f3fdf712c694e -certbot==0.11.1 \ - --hash=sha256:ba80552df0f390dbc5fcd14b4ea4b1499ea866f5f78c8c1a375abc25101dedf1 \ - --hash=sha256:6c1724486d500c5163c9313d6a14af5af9f4515f79553627303a6b86df2c3af2 -certbot-apache==0.11.1 \ - --hash=sha256:70132d9013509011b9edeba64fc208961f50ef78457f58d3b80a61094102efcd \ - --hash=sha256:efe2224b531595edee366423c115e2874a3c9011890321d3ccda0367efc776c0 -certbot-nginx==0.11.1 \ - --hash=sha256:1895eea1de92ab3dfd762998a4be7868ec3ec4d42cce7772995e4e9b2e488e6a \ - --hash=sha256:e5e5ffe8930ba10139bb61c2a05a30e84d9a69a7d8fc6a7b391f707eae8bfce5 +acme==0.13.0 \ + --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ + --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 +certbot==0.13.0 \ + --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ + --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 +certbot-apache==0.13.0 \ + --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ + --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 +certbot-nginx==0.13.0 \ + --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ + --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index 5752bdab0..2391a30bb 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -15,6 +15,7 @@ import tarfile import tempfile import textwrap +import six _DESCRIPTION = """ Let's Help is a simple script you can run to help out the Certbot @@ -69,7 +70,7 @@ def make_and_verify_selection(server_root, temp_dir): sys.stdout.write("\nIs it safe to submit these files? ") while True: - ans = raw_input("(Y)es/(N)o: ").lower() + ans = six.moves.input("(Y)es/(N)o: ").lower() if ans.startswith("y"): return elif ans.startswith("n"): @@ -144,7 +145,8 @@ def safe_config_file(config_file): return False proc = subprocess.Popen(["file", config_file], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) file_output, _ = proc.communicate() if "ASCII" in file_output: diff --git a/letshelp-certbot/letshelp_certbot/apache_test.py b/letshelp-certbot/letshelp_certbot/apache_test.py index 0c1b5f2f6..e0656ae05 100644 --- a/letshelp-certbot/letshelp_certbot/apache_test.py +++ b/letshelp-certbot/letshelp_certbot/apache_test.py @@ -9,6 +9,8 @@ import tempfile import unittest import mock +# six is used in mock.patch() +import six # pylint: disable=unused-import import letshelp_certbot.apache as letshelp_le_apache @@ -63,7 +65,7 @@ class LetsHelpApacheTest(unittest.TestCase): def test_make_and_verify_selection(self, mock_copy_config): mock_copy_config.return_value = (["apache2.conf"], ["apache2"]) - with mock.patch("__builtin__.raw_input") as mock_input: + with mock.patch("six.moves.input") as mock_input: with mock.patch(_MODULE_NAME + ".sys.stdout"): mock_input.side_effect = ["Yes", "No"] letshelp_le_apache.make_and_verify_selection("root", "temp") diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b616da688..b26ab41fe 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -7,12 +7,9 @@ from setuptools import find_packages version = '0.7.0.dev0' install_requires = [ + 'mock', 'setuptools', # pkg_resources ] -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags diff --git a/setup.py b/setup.py index 529459cf0..0e8d19a22 100644 --- a/setup.py +++ b/setup.py @@ -36,12 +36,14 @@ version = meta['version'] # https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), + 'argparse', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate + 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'PyOpenSSL', 'pyrfc3339', @@ -54,21 +56,11 @@ install_requires = [ 'zope.interface', ] -# env markers in extras_require cause problems with older pip: #517 -# Keep in sync with conditional_requirements.py. -if sys.version_info < (2, 7): - install_requires.extend([ - # only some distros recognize stdlib argparse as already satisfying - 'argparse', - 'mock<1.1.0', - ]) -else: - install_requires.append('mock') - dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', 'coverage', + 'ipdb', 'nose', 'pylint==1.4.2', # upstream #248 'tox', diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index ef61fe3f5..d9a979667 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -12,5 +12,7 @@ fi cd ${BOULDERPATH} FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') +[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) +[ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml docker-compose up -d diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 2ddd3c04b..08c482676 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -1,4 +1,4 @@ -#!/bin/sh -xe +#!/bin/bash # Simple integration test. Make sure to activate virtualenv beforehand # (source venv/bin/activate) and that you are running Boulder test # instance (see ./boulder-fetch.sh). @@ -8,12 +8,11 @@ # # Note: this script is called by Boulder integration test suite! +set -eux + . ./tests/integration/_common.sh export PATH="$PATH:/usr/sbin" # /usr/sbin/nginx -export GOPATH="${GOPATH:-/tmp/go}" -export PATH="$GOPATH/bin:$PATH" - if [ `uname` = "Darwin" ];then readlink="greadlink" else @@ -27,6 +26,14 @@ cleanup_and_exit() { echo Kill server subprocess, left running by abnormal exit kill $SERVER_STILL_RUNNING fi + # Dump boulder logs in case they contain useful debugging information. + : "------------------ ------------------ ------------------" + : "------------------ begin boulder logs ------------------" + : "------------------ ------------------ ------------------" + docker logs boulder_boulder_1 + : "------------------ ------------------ ------------------" + : "------------------ end boulder logs ------------------" + : "------------------ ------------------ ------------------" exit $EXIT_STATUS } @@ -75,7 +82,7 @@ CheckHooks() { # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python -m SimpleHTTPServer $http_01_port & +python ./tests/run_http_server.py $http_01_port & python_server_pid=$! common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ @@ -83,7 +90,7 @@ common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --renew-hook 'echo renew >> "$HOOK_TEST"' kill $python_server_pid -python -m SimpleHTTPServer $tls_sni_01_port & +python ./tests/run_http_server.py $tls_sni_01_port & python_server_pid=$! common --domains le2.wtf --preferred-challenges http-01 run \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ @@ -100,6 +107,8 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 \ common certonly -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni \ --manual-auth-hook ./tests/manual-dns-auth.sh +common certonly --cert-name newname -d newname.le.wtf + export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ OPENSSL_CNF=examples/openssl.cnf ./examples/generate-csr.sh le3.wtf diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 9b44631d4..8d4baff95 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -1,12 +1,7 @@ -#!/bin/sh - -if [ "xxx$root" = "xxx" ]; -then - # The -t is required on macOS. It provides a template file path for - # the kernel to use. - root="$(mktemp -d -t leitXXXX)" - echo "Root integration tests directory: $root" -fi +# The -t is required on macOS. It provides a template file path for +# the kernel to use. +root=${root:-$(mktemp -d -t leitXXXX)} +echo "Root integration tests directory: $root" store_flags="--config-dir $root/conf --work-dir $root/work" store_flags="$store_flags --logs-dir $root/logs" tls_sni_01_port=5001 diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh index f7a74821b..b46080eff 100755 --- a/tests/letstest/scripts/test_leauto_upgrades.sh +++ b/tests/letstest/scripts/test_leauto_upgrades.sh @@ -4,13 +4,6 @@ # are dynamically set at execution cd letsencrypt -#git checkout v0.1.0 use --branch instead -SAVE="$PIP_EXTRA_INDEX_URL" -unset PIP_EXTRA_INDEX_URL -export PIP_INDEX_URL="https://isnot.org/pip/0.1.0/" - -#OLD_LEAUTO="https://raw.githubusercontent.com/letsencrypt/letsencrypt/5747ab7fd9641986833bad474d71b46a8c589247/letsencrypt-auto" - if ! command -v git ; then if [ "$OS_TYPE" = "ubuntu" ] ; then @@ -22,15 +15,18 @@ if ! command -v git ; then fi fi BRANCH=`git rev-parse --abbrev-ref HEAD` -git checkout -f v0.1.0 -./letsencrypt-auto -v --debug --version -unset PIP_INDEX_URL - -export PIP_EXTRA_INDEX_URL="$SAVE" +# 0.4.1 is the oldest version of letsencrypt-auto that can be used because +# it's the first version that both pins package versions and properly supports +# --no-self-upgrade. +git checkout -f v0.4.1 +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.4.1 ; then + echo initial installation appeared to fail + exit 1 +fi git checkout -f "$BRANCH" EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2) -if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade | grep $EXPECTED_VERSION ; then +if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then echo upgrade appeared to fail exit 1 fi diff --git a/tests/manual-http-auth.sh b/tests/manual-http-auth.sh index c4730392b..48c33f04b 100755 --- a/tests/manual-http-auth.sh +++ b/tests/manual-http-auth.sh @@ -1,10 +1,12 @@ #!/bin/sh uri_path=".well-known/acme-challenge/$CERTBOT_TOKEN" +# This script should be run from the top level. e.g. ./tests/manual-http-auth.sh +source_dir="$(pwd)" cd $(mktemp -d) mkdir -p $(dirname $uri_path) echo $CERTBOT_VALIDATION > $uri_path -python -m SimpleHTTPServer $http_01_port >/dev/null 2>&1 & +python "$source_dir/tests/run_http_server.py" $http_01_port >/dev/null 2>&1 & server_pid=$! while ! curl "http://localhost:$http_01_port/$uri_path" >/dev/null 2>&1; do sleep 1s diff --git a/tests/modification-check.sh b/tests/modification-check.sh index 5168f5ab5..6f412ba47 100755 --- a/tests/modification-check.sh +++ b/tests/modification-check.sh @@ -1,46 +1,39 @@ #!/bin/bash temp_dir=`mktemp -d` +trap "rm -rf $temp_dir" EXIT -# Script should be run from Certbot's root directory - -SCRIPT_PATH=`dirname $0` -SCRIPT_PATH=`readlink -f $SCRIPT_PATH` +# cd to repo root +cd $(dirname $(dirname $(readlink -f $0))) FLAG=false -# Compare root letsencrypt-auto and certbot-auto with published versions - -cp letsencrypt-auto ${temp_dir}/letsencrypt-to-be-checked -cp certbot-auto ${temp_dir}/certbot-to-be-checked - -cp letsencrypt-auto-source/pieces/fetch.py ${temp_dir}/fetch.py -cd ${temp_dir} - -LATEST_VERSION=`python fetch.py --latest-version` -python fetch.py --le-auto-script v${LATEST_VERSION} - -cmp -s letsencrypt-auto letsencrypt-to-be-checked - -if [ $? != 0 ]; then - echo "Root letsencrypt-auto has changed." - FLAG=true +if ! cmp -s certbot-auto letsencrypt-auto; then + echo "Root certbot-auto and letsencrypt-auto differ." + FLAG=true else - echo "Root letsencrypt-auto is unchanged." + cp certbot-auto "$temp_dir/local-auto" + cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" + cd $temp_dir + + # Compare file against current version in the target branch + BRANCH=${TRAVIS_BRANCH:-master} + URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" + curl -sS $URL > certbot-auto + if cmp -s certbot-auto local-auto; then + echo "Root *-auto were unchanged." + else + # Compare file against the latest released version + python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" + if cmp -s letsencrypt-auto local-auto; then + echo "Root *-auto were updated to the latest version." + else + echo "Root *-auto have unexpected changes." + FLAG=true + fi + fi + cd ~- fi -cmp -s letsencrypt-auto certbot-to-be-checked - -if [ $? != 0 ]; then - echo "Root certbot-auto has changed." - FLAG=true -else - echo "Root certbot-auto is unchanged." -fi - -# Cleanup -rm ${temp_dir}/* -cd ${SCRIPT_PATH}/../ - # Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea diff --git a/tests/run_http_server.py b/tests/run_http_server.py new file mode 100644 index 000000000..fd1163816 --- /dev/null +++ b/tests/run_http_server.py @@ -0,0 +1,11 @@ +import runpy +import sys + +# Run Python's built-in HTTP server +# Usage: python ./tests/run_http_server.py port_num +# NOTE: This script should be compatible with 2.6, 2.7, 3.3+ + +# sys.argv (port number) is passed as-is to the HTTP server module +runpy.run_module( + 'http.server' if sys.version_info[0] == 3 else 'SimpleHTTPServer', + run_name='__main__') diff --git a/tools/docker-warning.sh b/tools/docker-warning.sh new file mode 100755 index 000000000..e4f5f40ee --- /dev/null +++ b/tools/docker-warning.sh @@ -0,0 +1,4 @@ +#!/bin/sh -e +echo "Warning: This Docker image will soon be switching to Alpine Linux." >&2 +echo "You can switch now using the certbot/certbot repo on Docker Hub." >&2 +exec /opt/certbot/venv/bin/certbot $@ diff --git a/tools/release.sh b/tools/release.sh index 75a4af29c..1da11fe2c 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -88,9 +88,9 @@ SetVersion() { sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py - + # interactive user input - git add -p certbot $SUBPKGS certbot-compatibility-test + git add -p certbot $SUBPKGS certbot-compatibility-test } @@ -109,7 +109,7 @@ do echo "Signing ($pkg_dir)" for x in dist/*.tar.gz dist/*.whl do - gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign $x + gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 $x done cd - @@ -194,7 +194,7 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ done # This signature is not quite as strong, but easier for people to verify out of band -gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign letsencrypt-auto-source/letsencrypt-auto +gpg2 -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign --digest-algo sha256 letsencrypt-auto-source/letsencrypt-auto # We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, # but we can use the right name for certbot-auto.asc from day one mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc @@ -214,7 +214,7 @@ name=${root_without_le%.*} ext="${root_without_le##*.}" rev="$(git rev-parse --short HEAD)" echo tar cJvf $name.$rev.tar.xz $name.$rev -echo gpg -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz +echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz cd ~- echo "New root: $root" diff --git a/tools/venv3.sh b/tools/venv3.sh index 35ffac749..08358aa48 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -1,8 +1,18 @@ #!/bin/sh -xe # Developer Python3 virtualenv setup for Certbot -export VENV_NAME="${VENV_NAME:-venv3}" -export VENV_ARGS="--python python3" +if command -v python3; then + export VENV_NAME="${VENV_NAME:-venv3}" + export VENV_ARGS="--python python3" +else + echo "Couldn't find python3 in $PATH" + exit 1 +fi ./tools/_venv_common.sh \ -e acme[dev] \ + -e .[dev,docs] \ + -e certbot-apache \ + -e certbot-nginx \ + -e letshelp-certbot \ + -e certbot-compatibility-test diff --git a/tox.ini b/tox.ini index e6317e665..d393bb610 100644 --- a/tox.ini +++ b/tox.ini @@ -40,34 +40,6 @@ deps = py{26,27}-oldest: PyOpenSSL==0.13 py{26,27}-oldest: requests<=2.11.1 -[testenv:py33] -commands = - pip install -e acme[dev] - nosetests -v acme --processes=-1 - pip install -e .[dev] - nosetests -v certbot --processes=-1 --process-timeout=100 - -[testenv:py34] -commands = - pip install -e acme[dev] - nosetests -v acme --processes=-1 - pip install -e .[dev] - nosetests -v certbot --processes=-1 --process-timeout=100 - -[testenv:py35] -commands = - pip install -e acme[dev] - nosetests -v acme --processes=-1 - pip install -e .[dev] - nosetests -v certbot --processes=-1 --process-timeout=100 - -[testenv:py36] -commands = - pip install -e acme[dev] - nosetests -v acme --processes=-1 - pip install -e .[dev] - nosetests -v certbot --processes=-1 --process-timeout=100 - [testenv:py27_install] basepython = python2.7 commands = @@ -89,6 +61,13 @@ commands = pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot pylint --reports=n --rcfile=.pylintrc acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot +[testenv:mypy] +basepython = python3.4 +commands = + pip install mypy + pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot + mypy --py2 --ignore-missing-imports acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot + [testenv:apacheconftest] #basepython = python2.7 commands = @@ -143,7 +122,9 @@ commands = docker run --rm -t -i lea whitelist_externals = docker -passenv = DOCKER_* +passenv = + DOCKER_* + TRAVIS_BRANCH [testenv:le_auto_wheezy] # At the moment, this tests under Python 2.7 only, as only that version is