Merge branch 'master' into master

.gitignore

@@ -6,7 +6,8 @@ dist*/
 /venv*/
 /kgs/
 /.tox/
-/releases/
+/releases*/
+/log*
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
@@ -35,3 +36,11 @@ tests/letstest/*.pem
 tests/letstest/venv/
 
 .venv
+
+# pytest cache
+.cache
+.mypy_cache/
+.pytest_cache/
+
+# docker files
+.docker

.pylintrc

@@ -41,7 +41,7 @@ load-plugins=linter_plugin
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
+disable=fixme,locally-disabled,locally-enabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code
 # abstract-class-not-used cannot be disabled locally (at least in
 # pylint 1.4.1), same for abstract-class-little-used
 

.travis.yml

@@ -5,7 +5,7 @@ cache:
         - $HOME/.cache/pip
 
 before_install:
-  - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)'
+  - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)'
 
 before_script:
   - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi'
@@ -13,7 +13,11 @@ before_script:
 matrix:
   include:
     - python: "2.7"
-      env: TOXENV=py27_install BOULDER_INTEGRATION=1
+      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
+      sudo: required
+      services: docker
+    - python: "2.7"
+      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
       sudo: required
       services: docker
     - python: "2.7"
@@ -25,20 +29,21 @@ matrix:
       addons:
     - python: "2.7"
       env: TOXENV=lint
-    - python: "2.6"
-      env: TOXENV=py26
-      sudo: required
-      services: docker
+    - python: "3.4"
+      env: TOXENV=mypy
+    - python: "3.5"
+      env: TOXENV=mypy
     - python: "2.7"
-      env: TOXENV=py27-oldest
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
       sudo: required
       services: docker
-    - python: "3.3"
-      env: TOXENV=py33
+    - python: "3.4"
+      env: TOXENV=py34
       sudo: required
       services: docker
-    - python: "3.6"
-      env: TOXENV=py36
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
       sudo: required
       services: docker
     - sudo: required
@@ -73,8 +78,6 @@ sudo: false
 
 addons:
   apt:
-    sources:
-    - augeas
     packages:  # Keep in sync with letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh and Boulder.
     - python-dev
     - python-virtualenv
@@ -86,23 +89,20 @@ addons:
     # For certbot-nginx integration testing
     - nginx-light
     - openssl
-    # for apacheconftest
-    - apache2
-    - libapache2-mod-wsgi
-    - libapache2-mod-macro
 
-install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls"
+install: "travis_retry $(command -v pip || command -v pip3) install codecov tox"
 script:
     - travis_retry tox
     - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
-after_success: '[ "$TOXENV" == "cover" ] && coveralls'
+after_success: '[ "$TOXENV" == "cover" ] && codecov'
 
 notifications:
   email: false
   irc:
     channels:
       - secure: "SGWZl3ownKx9xKVV2VnGt7DqkTmutJ89oJV9tjKhSs84kLijU6EYdPnllqISpfHMTxXflNZuxtGo0wTDYHXBuZL47w1O32W6nzuXdra5zC+i4sYQwYULUsyfOv9gJX8zWAULiK0Z3r0oho45U+FR5ZN6TPCidi8/eGU+EEPwaAw="
+    on_cancel: never
     on_success: never
     on_failure: always
     use_notice: true

CHANGELOG.md

@@ -2,6 +2,493 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.28.0 - master
+
+### Added
+
+* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`.
+* Use the ACMEv2 newNonce endpoint when a new nonce is needed, and newNonce is available in the directory.
+
+### Changed
+
+* Removed documentation mentions of `#letsencrypt` IRC on Freenode.
+* Write README to the base of (config-dir)/live directory
+* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges.
+* Warn when using deprecated acme.challenges.TLSSNI01
+* Log warning about TLS-SNI deprecation in Certbot
+* Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins
+* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies
+
+### Fixed
+
+* Match Nginx parser update in allowing variable names to start with `${`.
+* Fix ranking of vhosts in Nginx so that all port-matching vhosts come first
+* Correct OVH integration tests on machines without internet access.
+* Stop caching the results of ipv6_info in http01.py
+* Test fix for Route53 plugin to prevent boto3 making outgoing connections.
+* The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors.
+
+## 0.27.1 - 2018-09-06
+
+### Fixed
+
+* Fixed parameter name in OpenSUSE overrides for default parameters in the
+  Apache plugin. Certbot on OpenSUSE works again.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot-apache
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/60?closed=1
+
+## 0.27.0 - 2018-09-05
+
+### Added
+
+* The Apache plugin now accepts the parameter --apache-ctl which can be
+  used to configure the path to the Apache control script.
+
+### Changed
+
+* When using `acme.client.ClientV2` (or
+ `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a
+ newer version of the ACME protocol), an `acme.errors.ConflictError` will be
+ raised if you try to create an ACME account with a key that has already been
+ used. Previously, a JSON parsing error was raised in this scenario when using
+ the library with Let's Encrypt's ACMEv2 endpoint.
+
+### Fixed
+
+* When Apache is not installed, Certbot's Apache plugin no longer prints
+  messages about being unable to find apachectl to the terminal when the plugin
+  is not selected.
+* If you're using the Apache plugin with the --apache-vhost-root flag set to a
+  directory containing a disabled virtual host for the domain you're requesting
+  a certificate for, the virtual host will now be temporarily enabled if
+  necessary to pass the HTTP challenge.
+* The documentation for the Certbot package can now be built using Sphinx 1.6+.
+* You can now call `query_registration` without having to first call
+  `new_account` on `acme.client.ClientV2` objects.
+* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`.
+* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura
+  Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/57?closed=1
+
+## 0.26.1 - 2018-07-17
+
+### Fixed
+
+* Fix a bug that was triggered when users who had previously manually set `--server` to get ACMEv2 certs tried to renew ACMEv1 certs.
+
+Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was:
+
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/58?closed=1
+
+## 0.26.0 - 2018-07-11
+
+### Added
+
+* A new security enhancement which we're calling AutoHSTS has been added to
+  Certbot's Apache plugin. This enhancement configures your webserver to send a
+  HTTP Strict Transport Security header with a low max-age value that is slowly
+  increased over time. The max-age value is not increased to a large value
+  until you've successfully managed to renew your certificate. This enhancement
+  can be requested with the --auto-hsts flag.
+* New official DNS plugins have been created for Gehirn Infrastracture Service,
+  Linode, OVH, and Sakura Cloud. These plugins can be found on our Docker Hub
+  page at https://hub.docker.com/u/certbot and on PyPI.
+* The ability to reuse ACME accounts from Let's Encrypt's ACMEv1 endpoint on
+  Let's Encrypt's ACMEv2 endpoint has been added.
+* Certbot and its components now support Python 3.7.
+* Certbot's install subcommand now allows you to interactively choose which
+  certificate to install from the list of certificates managed by Certbot.
+* Certbot now accepts the flag `--no-autorenew` which causes any obtained
+  certificates to not be automatically renewed when it approaches expiration.
+* Support for parsing the TLS-ALPN-01 challenge has been added back to the acme
+  library.
+
+### Changed
+
+* Certbot's default ACME server has been changed to Let's Encrypt's ACMEv2
+  endpoint. By default, this server will now be used for both new certificate
+  lineages and renewals.
+* The Nginx plugin is no longer marked labeled as an "Alpha" version.
+* The `prepare` method of Certbot's plugins is no longer called before running
+  "Updater" enhancements that are run on every invocation of `certbot renew`.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/55?closed=1
+
+## 0.25.1 - 2018-06-13
+
+### Fixed
+
+* TLS-ALPN-01 support has been removed from our acme library. Using our current
+  dependencies, we are unable to provide a correct implementation of this
+  challenge so we decided to remove it from the library until we can provide
+  proper support.
+* Issues causing test failures when running the tests in the acme package with
+  pytest<3.0 has been resolved.
+* certbot-nginx now correctly depends on acme>=0.25.0.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/56?closed=1
+
+## 0.25.0 - 2018-06-06
+
+### Added
+
+* Support for the ready status type was added to acme. Without this change,
+  Certbot and acme users will begin encountering errors when using Let's
+  Encrypt's ACMEv2 API starting on June 19th for the staging environment and
+  July 5th for production. See
+  https://community.letsencrypt.org/t/acmev2-order-ready-status/62866 for more
+  information.
+* Certbot now accepts the flag --reuse-key which will cause the same key to be
+  used in the certificate when the lineage is renewed rather than generating a
+  new key.
+* You can now add multiple email addresses to your ACME account with Certbot by
+  providing a comma separated list of emails to the --email flag.
+* Support for Let's Encrypt's upcoming TLS-ALPN-01 challenge was added to acme.
+  For more information, see
+  https://community.letsencrypt.org/t/tls-alpn-validation-method/63814/1.
+* acme now supports specifying the source address to bind to when sending
+  outgoing connections. You still cannot specify this address using Certbot.
+* If you run Certbot against Let's Encrypt's ACMEv2 staging server but don't
+  already have an account registered at that server URL, Certbot will
+  automatically reuse your staging account from Let's Encrypt's ACMEv1 endpoint
+  if it exists.
+* Interfaces were added to Certbot allowing plugins to be called at additional
+  points. The `GenericUpdater` interface allows plugins to perform actions
+  every time `certbot renew` is run, regardless of whether any certificates are
+  due for renewal, and the `RenewDeployer` interface allows plugins to perform
+  actions when a certificate is renewed. See `certbot.interfaces` for more
+  information.
+
+### Changed
+
+* When running Certbot with --dry-run and you don't already have a staging
+  account, the created account does not contain an email address even if one
+  was provided to avoid expiration emails from Let's Encrypt's staging server.
+* certbot-nginx does a better job of automatically detecting the location of
+  Nginx's configuration files when run on BSD based systems.
+* acme now requires and uses pytest when running tests with setuptools with
+  `python setup.py test`.
+* `certbot config_changes` no longer waits for user input before exiting.
+
+### Fixed
+
+* Misleading log output that caused users to think that Certbot's standalone
+  plugin failed to bind to a port when performing a challenge has been
+  corrected.
+* An issue where certbot-nginx would fail to enable HSTS if the server block
+  already had an `add_header` directive has been resolved.
+* certbot-nginx now does a better job detecting the server block to base the
+  configuration for TLS-SNI challenges on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/54?closed=1
+
+## 0.24.0 - 2018-05-02
+
+### Added
+
+* certbot now has an enhance subcommand which allows you to configure security
+  enhancements like HTTP to HTTPS redirects, OCSP stapling, and HSTS without
+  reinstalling a certificate.
+* certbot-dns-rfc2136 now allows the user to specify the port to use to reach
+  the DNS server in its credentials file.
+* acme now parses the wildcard field included in authorizations so it can be
+  used by users of the library.
+
+### Changed
+
+* certbot-dns-route53 used to wait for each DNS update to propagate before
+  sending the next one, but now it sends all updates before waiting which
+  speeds up issuance for multiple domains dramatically.
+* Certbot's official Docker images are now based on Alpine Linux 3.7 rather
+  than 3.4 because 3.4 has reached its end-of-life.
+* We've doubled the time Certbot will spend polling authorizations before
+  timing out.
+* The level of the message logged when Certbot is being used with
+  non-standard paths warning that crontabs for renewal included in Certbot
+  packages from OS package managers may not work has been reduced. This stops
+  the message from being written to stderr every time `certbot renew` runs.
+
+### Fixed
+
+* certbot-auto now works with Python 3.6.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-digitalocean (only style improvements to tests)
+* certbot-dns-rfc2136
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/52?closed=1
+
+## 0.23.0 - 2018-04-04
+
+### Added
+
+* Support for OpenResty was added to the Nginx plugin.
+
+### Changed
+
+* The timestamps in Certbot's logfiles now use the system's local time zone
+  rather than UTC.
+* Certbot's DNS plugins that use Lexicon now rely on Lexicon>=2.2.1 to be able
+  to create and delete multiple TXT records on a single domain.
+* certbot-dns-google's test suite now works without an internet connection.
+
+### Fixed
+
+* Removed a small window that if during which an error occurred, Certbot
+  wouldn't clean up performed challenges.
+* The parameters `default` and `ipv6only` are now removed from `listen`
+  directives when creating a new server block in the Nginx plugin.
+* `server_name` directives enclosed in quotation marks in Nginx are now properly
+  supported.
+* Resolved an issue preventing the Apache plugin from starting Apache when it's
+  not currently running on RHEL and Gentoo based systems.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* certbot
+* certbot-apache
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-google
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-rfc2136
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/50?closed=1
+
+## 0.22.2 - 2018-03-19
+
+### Fixed
+
+* A type error introduced in 0.22.1 that would occur during challenge cleanup
+  when a Certbot plugin raises an exception while trying to complete the
+  challenge was fixed.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/53?closed=1
+
+## 0.22.1 - 2018-03-19
+
+### Changed
+
+* The ACME server used with Certbot's --dry-run and --staging flags is now
+  Let's Encrypt's ACMEv2 staging server which allows people to also test ACMEv2
+  features with these flags.
+
+### Fixed
+
+* The HTTP Content-Type header is now set to the correct value during
+  certificate revocation with new versions of the ACME protocol.
+* When using Certbot with Let's Encrypt's ACMEv2 server, it would add a blank
+  line to the top of chain.pem and between the certificates in fullchain.pem
+  for each lineage. These blank lines have been removed.
+* Resolved a bug that caused Certbot's --allow-subset-of-names flag not to
+  work.
+* Fixed a regression in acme.client.Client that caused the class to not work
+  when it was initialized without a ClientNetwork which is done by some of the
+  other projects using our ACME library.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with changes other than their version number were:
+
+* acme
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/51?closed=1
+
+## 0.22.0 - 2018-03-07
+
+### Added
+
+* Support for obtaining wildcard certificates and a newer version of the ACME
+  protocol such as the one implemented by Let's Encrypt's upcoming ACMEv2
+  endpoint was added to Certbot and its ACME library. Certbot still works with
+  older ACME versions and will automatically change the version of the protocol
+  used based on the version the ACME CA implements.
+* The Apache and Nginx plugins are now able to automatically install a wildcard
+  certificate to multiple virtual hosts that you select from your server
+  configuration.
+* The `certbot install` command now accepts the `--cert-name` flag for
+  selecting a certificate.
+* `acme.client.BackwardsCompatibleClientV2` was added to Certbot's ACME library
+  which automatically handles most of the differences between new and old ACME
+  versions. `acme.client.ClientV2` is also available for people who only want
+  to support one version of the protocol or want to handle the differences
+  between versions themselves.
+* certbot-auto now supports the flag --install-only which has the script
+  install Certbot and its dependencies and exit without invoking Certbot.
+* Support for issuing a single certificate for a wildcard and base domain was
+  added to our Google Cloud DNS plugin. To do this, we now require your API
+  credentials have additional permissions, however, your credentials will
+  already have these permissions unless you defined a custom role with fewer
+  permissions than the standard DNS administrator role provided by Google.
+  These permissions are also only needed for the case described above so it
+  will continue to work for existing users. For more information about the
+  permissions changes, see the documentation in the plugin.
+
+### Changed
+
+* We have broken lockstep between our ACME library, Certbot, and its plugins.
+  This means that the different components do not need to be the same version
+  to work together like they did previously. This makes packaging easier
+  because not every piece of Certbot needs to be repackaged to ship a change to
+  a subset of its components.
+* Support for Python 2.6 and Python 3.3 has been removed from ACME, Certbot,
+  Certbot's plugins, and certbot-auto. If you are using certbot-auto on a RHEL
+  6 based system, it will walk you through the process of installing Certbot
+  with Python 3 and refuse to upgrade to a newer version of Certbot until you
+  have done so.
+* Certbot's components now work with older versions of setuptools to simplify
+  packaging for EPEL 7.
+
+### Fixed
+
+* Issues caused by Certbot's Nginx plugin adding multiple ipv6only directives
+  has been resolved.
+* A problem where Certbot's Apache plugin would add redundant include
+  directives for the TLS configuration managed by Certbot has been fixed.
+* Certbot's webroot plugin now properly deletes any directories it creates.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/48?closed=1
+
+## 0.21.1 - 2018-01-25
+
+### Fixed
+
+* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host
+  header of the request is set to an expected value before redirecting users to
+  the domain found in the header. The previous way Certbot configured Nginx
+  redirects was a potential security issue which you can read more about at
+  https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493.
+* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges
+  if basic authentication is configured for the domain you request a
+  certificate for.
+* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6
+  based systems rather than Python 2.6.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/49?closed=1
+
+## 0.21.0 - 2018-01-17
+
+### Added
+
+* Support for the HTTP-01 challenge type was added to our Apache and Nginx
+  plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge
+  type which was what was previously being used by our Apache and Nginx plugins
+  last week due to a security issue. For more information about Let's Encrypt's
+  change, click
+  [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188).
+  Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no
+  changes need to be made to your Certbot configuration, however, you should
+  make sure your server is accessible on port 80 and isn't behind an external
+  proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to
+  HTTPS redirects inside Apache and Nginx are fine.
+* IPv6 support was added to the Nginx plugin.
+* Support for automatically creating server blocks based on the default server
+  block was added to the Nginx plugin.
+* The flags --delete-after-revoke and --no-delete-after-revoke were added
+  allowing users to control whether the revoke subcommand also deletes the
+  certificates it is revoking.
+
+### Changed
+
+* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME
+  library. Support for these versions of Python will be removed in the next
+  major release of Certbot. If you are using certbot-auto on a RHEL 6 based
+  system, it will guide you through the process of installing Python 3.
+* We split our implementation of JOSE (Javascript Object Signing and
+  Encryption) out of our ACME library and into a separate package named josepy.
+  This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and
+  on [GitHub](https://github.com/certbot/josepy).
+* We updated the ciphersuites used in Apache to the new [values recommended by
+  Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29).
+  The major change here is adding ChaCha20 to the list of supported
+  ciphersuites.
+
+### Fixed
+
+* An issue with our Apache plugin on Gentoo due to differences in their
+  apache2ctl command have been resolved.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/47?closed=1
+
 ## 0.20.0 - 2017-12-06
 
 ### Added
@@ -396,7 +883,7 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac
 
 ### Added
 
-* When using the standalone plugin while running Certbot interactively 
+* When using the standalone plugin while running Certbot interactively
 and a required port is bound by another process, Certbot will give you
 the option to retry to grab the port rather than immediately exiting.
 * You are now able to deactivate your account with the Let's Encrypt

CHANGES.rst

@@ -1,8 +0,0 @@
-ChangeLog
-=========
-
-To see the changes in a given release, view the issues closed in a given
-release's GitHub milestone:
-
- - `Past releases <https://github.com/certbot/certbot/milestones?state=closed>`_
- - `Upcoming releases <https://github.com/certbot/certbot/milestones>`_

Dockerfile

@@ -1,11 +1,11 @@
-FROM python:2-alpine
+FROM python:2-alpine3.7
 
 ENTRYPOINT [ "certbot" ]
 EXPOSE 80 443
 VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
-COPY CHANGES.rst README.rst setup.py src/
+COPY CHANGELOG.md README.rst setup.py src/
 COPY acme src/acme
 COPY certbot src/certbot
 

Dockerfile-dev

@@ -1,70 +1,21 @@
 # This Dockerfile builds an image for development.
-FROM ubuntu:trusty
-MAINTAINER Jakub Warmuz <jakub@warmuz.org>
-MAINTAINER William Budington <bill@eff.org>
-MAINTAINER Yan <yan@eff.org>
+FROM ubuntu:xenial
 
-# Note: this only exposes the port to other docker containers. You
-# still have to bind to 443@host at runtime, as per the ACME spec.
-EXPOSE 443
-
-# TODO: make sure --config-dir and --work-dir cannot be changed
-# through the CLI (certbot-docker wrapper that uses standalone
-# authenticator and text mode only?)
-VOLUME /etc/letsencrypt /var/lib/letsencrypt
+# Note: this only exposes the port to other docker containers.
+EXPOSE 80 443
 
 WORKDIR /opt/certbot/src
 
-# no need to mkdir anything:
-# https://docs.docker.com/reference/builder/#copy
-# If <dest> doesn't exist, it is created along with all missing
-# directories in its path.
-
 # TODO: Install Apache/Nginx for plugin development.
-COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto
-RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \
-    apt-get install python3-dev git -y && \
+COPY . .
+RUN apt-get update && \
+    apt-get install apache2 git nginx-light -y && \
+    letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \
            /tmp/* \
            /var/tmp/*
 
-# the above is not likely to change, so by putting it further up the
-# Dockerfile we make sure we cache as much as possible
-
-COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
-
-# all above files are necessary for setup.py, however, package source
-# code directory has to be copied separately to a subdirectory...
-# https://docs.docker.com/reference/builder/#copy: "If <src> is a
-# directory, the entire contents of the directory are copied,
-# including filesystem metadata. Note: The directory itself is not
-# copied, just its contents." Order again matters, three files are far
-# more likely to be cached than the whole project directory
-
-COPY certbot /opt/certbot/src/certbot/
-COPY acme /opt/certbot/src/acme/
-COPY certbot-apache /opt/certbot/src/certbot-apache/
-COPY certbot-nginx /opt/certbot/src/certbot-nginx/
-COPY letshelp-certbot /opt/certbot/src/letshelp-certbot/
-COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/
-COPY tests /opt/certbot/src/tests/
-
-RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \
-    /opt/certbot/venv/bin/pip install -U pip && \
-    /opt/certbot/venv/bin/pip install -U setuptools && \
-    /opt/certbot/venv/bin/pip install \
-    -e /opt/certbot/src/acme \
-    -e /opt/certbot/src \
-    -e /opt/certbot/src/certbot-apache \
-    -e /opt/certbot/src/certbot-nginx \
-    -e /opt/certbot/src/letshelp-certbot \
-    -e /opt/certbot/src/certbot-compatibility-test \
-    -e /opt/certbot/src[dev,docs]
-
-# install in editable mode (-e) to save space: it's not possible to
-# "rm -rf /opt/certbot/src" (it's stays in the underlaying image);
-# this might also help in debugging: you can "docker run --entrypoint
-# bash" and investigate, apply patches, etc.
+RUN VENV_NAME="../venv" tools/venv.sh
 
 ENV PATH /opt/certbot/venv/bin:$PATH

Dockerfile-old

@@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # Dockerfile we make sure we cache as much as possible
 
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
 
 # all above files are necessary for setup.py and venv setup, however,
 # package source code directory has to be copied separately to a

ISSUE_TEMPLATE.md

@@ -1,3 +1,9 @@
+If you're having trouble using Certbot and aren't sure you've found a bug or
+request for a new feature, please first try asking for help at
+https://community.letsencrypt.org/. There is a much larger community there of
+people familiar with the project who will be able to more quickly answer your
+questions.
+
 ## My operating system is (include version):
 
 

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.rst
-include CHANGES.rst
+include CHANGELOG.md
 include CONTRIBUTING.md
 include LICENSE.txt
 include linter_plugin.py

README.rst

@@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha
 
 How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide <https://certbot.eff.org>`_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access <https://certbot.eff.org/faq/#does-certbot-require-root-administrator-privileges>`_ to your web server to run Certbot.
 
-If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
+Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
 
 Certbot is a fully-featured, extensible client for the Let's
 Encrypt CA (or any other CA that speaks the `ACME
@@ -91,8 +91,6 @@ Main Website: https://certbot.eff.org
 
 Let's Encrypt Website: https://letsencrypt.org
 
-IRC Channel: #letsencrypt on `Freenode`_
-
 Community: https://community.letsencrypt.org
 
 ACME spec: http://ietf-wg-acme.github.io/acme/
@@ -101,14 +99,12 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
 
 |build-status| |coverage| |docs| |container|
 
-.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt
-
 .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master
    :target: https://travis-ci.org/certbot/certbot
    :alt: Travis CI status
 
-.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master
-   :target: https://coveralls.io/r/certbot/certbot
+.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/certbot/certbot
    :alt: Coverage status
 
 .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/

acme/MANIFEST.in

@@ -1,5 +1,6 @@
 include LICENSE.txt
 include README.rst
+include pytest.ini
 recursive-include docs *
 recursive-include examples *
 recursive-include acme/testdata *

acme/acme/__init__.py

@@ -10,12 +10,3 @@ supported version: `draft-ietf-acme-01`_.
   https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01
 
 """
-import sys
-import warnings
-
-if sys.version_info[:2] == (3, 3):
-    warnings.warn(
-            "Python 3.3 support will be dropped in the next release of "
-            "acme. Please upgrade your Python version.",
-            PendingDeprecationWarning,
-    ) #pragma: no cover

acme/acme/challenges.py

@@ -4,11 +4,13 @@ import functools
 import hashlib
 import logging
 import socket
+import warnings
 
 from cryptography.hazmat.primitives import hashes  # type: ignore
 import josepy as jose
 import OpenSSL
 import requests
+import six
 
 from acme import errors
 from acme import crypto_util
@@ -139,16 +141,16 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         return True
 
 
+@six.add_metaclass(abc.ABCMeta)
 class KeyAuthorizationChallenge(_TokenChallenge):
     # pylint: disable=abstract-class-little-used,too-many-ancestors
     """Challenge based on Key Authorization.
 
     :param response_cls: Subclass of `KeyAuthorizationChallengeResponse`
         that will be used to generate `response`.
-
+    :param str typ: type of the challenge
     """
-    __metaclass__ = abc.ABCMeta
-
+    typ = NotImplemented
     response_cls = NotImplemented
     thumbprint_hash_function = (
         KeyAuthorizationChallengeResponse.thumbprint_hash_function)
@@ -477,7 +479,7 @@ class TLSSNI01Response(KeyAuthorizationChallengeResponse):
             try:
                 cert = self.probe_cert(domain=domain, **kwargs)
             except errors.Error as error:
-                logger.debug(error, exc_info=True)
+                logger.debug(str(error), exc_info=True)
                 return False
 
         return self.verify_cert(cert)
@@ -492,6 +494,11 @@ class TLSSNI01(KeyAuthorizationChallenge):
     # boulder#962, ietf-wg-acme#22
     #n = jose.Field("n", encoder=int, decoder=int)
 
+    def __init__(self, *args, **kwargs):
+        warnings.warn("TLS-SNI-01 is deprecated, and will stop working soon.",
+            DeprecationWarning, stacklevel=2)
+        super(TLSSNI01, self).__init__(*args, **kwargs)
+
     def validation(self, account_key, **kwargs):
         """Generate validation.
 
@@ -506,6 +513,21 @@ class TLSSNI01(KeyAuthorizationChallenge):
         return self.response(account_key).gen_cert(key=kwargs.get('cert_key'))
 
 
+@Challenge.register  # pylint: disable=too-many-ancestors
+class TLSALPN01(KeyAuthorizationChallenge):
+    """ACME tls-alpn-01 challenge.
+
+    This class simply allows parsing the TLS-ALPN-01 challenge returned from
+    the CA. Full TLS-ALPN-01 support is not currently provided.
+
+    """
+    typ = "tls-alpn-01"
+
+    def validation(self, account_key, **kwargs):
+        """Generate validation for the challenge."""
+        raise NotImplementedError()
+
+
 @Challenge.register  # pylint: disable=too-many-ancestors
 class DNS(_TokenChallenge):
     """ACME "dns" challenge."""

acme/acme/challenges_test.py

@@ -1,5 +1,6 @@
 """Tests for acme.challenges."""
 import unittest
+import warnings
 
 import josepy as jose
 import mock
@@ -360,20 +361,29 @@ class TLSSNI01ResponseTest(unittest.TestCase):
 class TLSSNI01Test(unittest.TestCase):
 
     def setUp(self):
-        from acme.challenges import TLSSNI01
-        self.msg = TLSSNI01(
-            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
         self.jmsg = {
             'type': 'tls-sni-01',
             'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
         }
 
+    def _msg(self):
+        from acme.challenges import TLSSNI01
+        with warnings.catch_warnings(record=True) as warn:
+            warnings.simplefilter("always")
+            msg = TLSSNI01(
+                token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+            assert warn is not None # using a raw assert for mypy
+            self.assertTrue(len(warn) == 1)
+            self.assertTrue(issubclass(warn[-1].category, DeprecationWarning))
+            self.assertTrue('deprecated' in str(warn[-1].message))
+        return msg
+
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+        self.assertEqual(self.jmsg, self._msg().to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import TLSSNI01
-        self.assertEqual(self.msg, TLSSNI01.from_json(self.jmsg))
+        self.assertEqual(self._msg(), TLSSNI01.from_json(self.jmsg))
 
     def test_from_json_hashable(self):
         from acme.challenges import TLSSNI01
@@ -388,11 +398,43 @@ class TLSSNI01Test(unittest.TestCase):
     @mock.patch('acme.challenges.TLSSNI01Response.gen_cert')
     def test_validation(self, mock_gen_cert):
         mock_gen_cert.return_value = ('cert', 'key')
-        self.assertEqual(('cert', 'key'), self.msg.validation(
+        self.assertEqual(('cert', 'key'), self._msg().validation(
             KEY, cert_key=mock.sentinel.cert_key))
         mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key)
 
 
+class TLSALPN01Test(unittest.TestCase):
+
+    def setUp(self):
+        from acme.challenges import TLSALPN01
+        self.msg = TLSALPN01(
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
+        self.jmsg = {
+            'type': 'tls-alpn-01',
+            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
+        }
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import TLSALPN01
+        self.assertEqual(self.msg, TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import TLSALPN01
+        hash(TLSALPN01.from_json(self.jmsg))
+
+    def test_from_json_invalid_token_length(self):
+        from acme.challenges import TLSALPN01
+        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
+        self.assertRaises(
+            jose.DeserializationError, TLSALPN01.from_json, self.jmsg)
+
+    def test_validation(self):
+        self.assertRaises(NotImplementedError, self.msg.validation, KEY)
+
+
 class DNSTest(unittest.TestCase):
 
     def setUp(self):

acme/acme/client.py

@@ -9,16 +9,20 @@ import time
 
 import six
 from six.moves import http_client  # pylint: disable=import-error
-
 import josepy as jose
 import OpenSSL
 import re
+from requests_toolbelt.adapters.source import SourceAddressAdapter
 import requests
+from requests.adapters import HTTPAdapter
 import sys
 
+from acme import crypto_util
 from acme import errors
 from acme import jws
 from acme import messages
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Dict, List, Set, Text
 
 
 logger = logging.getLogger(__name__)
@@ -39,39 +43,23 @@ DEFAULT_NETWORK_TIMEOUT = 45
 DER_CONTENT_TYPE = 'application/pkix-cert'
 
 
-class Client(object):  # pylint: disable=too-many-instance-attributes
-    """ACME client.
-
-    .. todo::
-       Clean up raised error types hierarchy, document, and handle (wrap)
-       instances of `.DeserializationError` raised in `from_json()`.
+class ClientBase(object):  # pylint: disable=too-many-instance-attributes
+    """ACME client base object.
 
     :ivar messages.Directory directory:
-    :ivar key: `.JWK` (private)
-    :ivar alg: `.JWASignature`
-    :ivar bool verify_ssl: Verify SSL certificates?
-    :ivar .ClientNetwork net: Client network. Useful for testing. If not
-        supplied, it will be initialized using `key`, `alg` and
-        `verify_ssl`.
-
+    :ivar .ClientNetwork net: Client network.
+    :ivar int acme_version: ACME protocol version. 1 or 2.
     """
-
-    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
-                 net=None):
+    def __init__(self, directory, net, acme_version):
         """Initialize.
 
-        :param directory: Directory Resource (`.messages.Directory`) or
-            URI from which the resource will be downloaded.
-
+        :param .messages.Directory directory: Directory Resource
+        :param .ClientNetwork net: Client network.
+        :param int acme_version: ACME protocol version. 1 or 2.
         """
-        self.key = key
-        self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net
-
-        if isinstance(directory, six.string_types):
-            self.directory = messages.Directory.from_json(
-                self.net.get(directory).json())
-        else:
-            self.directory = directory
+        self.directory = directory
+        self.net = net
+        self.acme_version = acme_version
 
     @classmethod
     def _regr_from_response(cls, response, uri=None, terms_of_service=None):
@@ -83,28 +71,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             uri=response.headers.get('Location', uri),
             terms_of_service=terms_of_service)
 
-    def register(self, new_reg=None):
-        """Register.
-
-        :param .NewRegistration new_reg:
-
-        :returns: Registration Resource.
-        :rtype: `.RegistrationResource`
-
-        """
-        new_reg = messages.NewRegistration() if new_reg is None else new_reg
-        assert isinstance(new_reg, messages.NewRegistration)
-
-        response = self.net.post(self.directory[new_reg], new_reg)
-        # TODO: handle errors
-        assert response.status_code == http_client.CREATED
-
-        # "Instance of 'Field' has no key/contact member" bug:
-        # pylint: disable=no-member
-        return self._regr_from_response(response)
-
     def _send_recv_regr(self, regr, body):
-        response = self.net.post(regr.uri, body)
+        response = self._post(regr.uri, body)
 
         # TODO: Boulder returns httplib.ACCEPTED
         #assert response.status_code == httplib.OK
@@ -116,6 +84,15 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             response, uri=regr.uri,
             terms_of_service=regr.terms_of_service)
 
+    def _post(self, *args, **kwargs):
+        """Wrapper around self.net.post that adds the acme_version.
+
+        """
+        kwargs.setdefault('acme_version', self.acme_version)
+        if hasattr(self.directory, 'newNonce'):
+            kwargs.setdefault('new_nonce_url', getattr(self.directory, 'newNonce'))
+        return self.net.post(*args, **kwargs)
+
     def update_registration(self, regr, update=None):
         """Update registration.
 
@@ -130,6 +107,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         update = regr.body if update is None else update
         body = messages.UpdateRegistration(**dict(update))
         updated_regr = self._send_recv_regr(regr, body=body)
+        self.net.account = updated_regr
         return updated_regr
 
     def deactivate_registration(self, regr):
@@ -153,65 +131,14 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         """
         return self._send_recv_regr(regr, messages.UpdateRegistration())
 
-    def agree_to_tos(self, regr):
-        """Agree to the terms-of-service.
-
-        Agree to the terms-of-service in a Registration Resource.
-
-        :param regr: Registration Resource.
-        :type regr: `.RegistrationResource`
-
-        :returns: Updated Registration Resource.
-        :rtype: `.RegistrationResource`
-
-        """
-        return self.update_registration(
-            regr.update(body=regr.body.update(agreement=regr.terms_of_service)))
-
-    def _authzr_from_response(self, response, identifier, uri=None):
+    def _authzr_from_response(self, response, identifier=None, uri=None):
         authzr = messages.AuthorizationResource(
             body=messages.Authorization.from_json(response.json()),
             uri=response.headers.get('Location', uri))
-        if authzr.body.identifier != identifier:
+        if identifier is not None and authzr.body.identifier != identifier:
             raise errors.UnexpectedUpdate(authzr)
         return authzr
 
-    def request_challenges(self, identifier, new_authzr_uri=None):
-        """Request challenges.
-
-        :param .messages.Identifier identifier: Identifier to be challenged.
-        :param str new_authzr_uri: Deprecated. Do not use.
-
-        :returns: Authorization Resource.
-        :rtype: `.AuthorizationResource`
-
-        """
-        if new_authzr_uri is not None:
-            logger.debug("request_challenges with new_authzr_uri deprecated.")
-        new_authz = messages.NewAuthorization(identifier=identifier)
-        response = self.net.post(self.directory.new_authz, new_authz)
-        # TODO: handle errors
-        assert response.status_code == http_client.CREATED
-        return self._authzr_from_response(response, identifier)
-
-    def request_domain_challenges(self, domain, new_authzr_uri=None):
-        """Request challenges for domain names.
-
-        This is simply a convenience function that wraps around
-        `request_challenges`, but works with domain names instead of
-        generic identifiers. See ``request_challenges`` for more
-        documentation.
-
-        :param str domain: Domain name to be challenged.
-        :param str new_authzr_uri: Deprecated. Do not use.
-
-        :returns: Authorization Resource.
-        :rtype: `.AuthorizationResource`
-
-        """
-        return self.request_challenges(messages.Identifier(
-            typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri)
-
     def answer_challenge(self, challb, response):
         """Answer challenge.
 
@@ -227,7 +154,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :raises .UnexpectedUpdate:
 
         """
-        response = self.net.post(challb.uri, response)
+        response = self._post(challb.uri, response)
         try:
             authzr_uri = response.links['up']['url']
         except KeyError:
@@ -288,6 +215,142 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             response, authzr.body.identifier, authzr.uri)
         return updated_authzr, response
 
+    def _revoke(self, cert, rsn, url):
+        """Revoke certificate.
+
+        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
+            `.ComparableX509`
+
+        :param int rsn: Reason code for certificate revocation.
+
+        :param str url: ACME URL to post to
+
+        :raises .ClientError: If revocation is unsuccessful.
+
+        """
+        response = self._post(url,
+                              messages.Revocation(
+                                certificate=cert,
+                                reason=rsn))
+        if response.status_code != http_client.OK:
+            raise errors.ClientError(
+                'Successful revocation must return HTTP OK status')
+
+class Client(ClientBase):
+    """ACME client for a v1 API.
+
+    .. todo::
+       Clean up raised error types hierarchy, document, and handle (wrap)
+       instances of `.DeserializationError` raised in `from_json()`.
+
+    :ivar messages.Directory directory:
+    :ivar key: `josepy.JWK` (private)
+    :ivar alg: `josepy.JWASignature`
+    :ivar bool verify_ssl: Verify SSL certificates?
+    :ivar .ClientNetwork net: Client network. Useful for testing. If not
+        supplied, it will be initialized using `key`, `alg` and
+        `verify_ssl`.
+
+    """
+
+    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
+                 net=None):
+        """Initialize.
+
+        :param directory: Directory Resource (`.messages.Directory`) or
+            URI from which the resource will be downloaded.
+
+        """
+        # pylint: disable=too-many-arguments
+        self.key = key
+        if net is None:
+            net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl)
+
+        if isinstance(directory, six.string_types):
+            directory = messages.Directory.from_json(
+                net.get(directory).json())
+        super(Client, self).__init__(directory=directory,
+            net=net, acme_version=1)
+
+    def register(self, new_reg=None):
+        """Register.
+
+        :param .NewRegistration new_reg:
+
+        :returns: Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        new_reg = messages.NewRegistration() if new_reg is None else new_reg
+        response = self._post(self.directory[new_reg], new_reg)
+        # TODO: handle errors
+        assert response.status_code == http_client.CREATED
+
+        # "Instance of 'Field' has no key/contact member" bug:
+        # pylint: disable=no-member
+        return self._regr_from_response(response)
+
+    def agree_to_tos(self, regr):
+        """Agree to the terms-of-service.
+
+        Agree to the terms-of-service in a Registration Resource.
+
+        :param regr: Registration Resource.
+        :type regr: `.RegistrationResource`
+
+        :returns: Updated Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        return self.update_registration(
+            regr.update(body=regr.body.update(agreement=regr.terms_of_service)))
+
+    def request_challenges(self, identifier, new_authzr_uri=None):
+        """Request challenges.
+
+        :param .messages.Identifier identifier: Identifier to be challenged.
+        :param str new_authzr_uri: Deprecated. Do not use.
+
+        :returns: Authorization Resource.
+        :rtype: `.AuthorizationResource`
+
+        :raises errors.WildcardUnsupportedError: if a wildcard is requested
+
+        """
+        if new_authzr_uri is not None:
+            logger.debug("request_challenges with new_authzr_uri deprecated.")
+
+        if identifier.value.startswith("*"):
+            raise errors.WildcardUnsupportedError(
+                "Requesting an authorization for a wildcard name is"
+                " forbidden by this version of the ACME protocol.")
+
+        new_authz = messages.NewAuthorization(identifier=identifier)
+        response = self._post(self.directory.new_authz, new_authz)
+        # TODO: handle errors
+        assert response.status_code == http_client.CREATED
+        return self._authzr_from_response(response, identifier)
+
+    def request_domain_challenges(self, domain, new_authzr_uri=None):
+        """Request challenges for domain names.
+
+        This is simply a convenience function that wraps around
+        `request_challenges`, but works with domain names instead of
+        generic identifiers. See ``request_challenges`` for more
+        documentation.
+
+        :param str domain: Domain name to be challenged.
+        :param str new_authzr_uri: Deprecated. Do not use.
+
+        :returns: Authorization Resource.
+        :rtype: `.AuthorizationResource`
+
+        :raises errors.WildcardUnsupportedError: if a wildcard is requested
+
+        """
+        return self.request_challenges(messages.Identifier(
+            typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri)
+
     def request_issuance(self, csr, authzrs):
         """Request issuance.
 
@@ -307,7 +370,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         req = messages.CertificateRequest(csr=csr)
 
         content_type = DER_CONTENT_TYPE  # TODO: add 'cert_type 'argument
-        response = self.net.post(
+        response = self._post(
             self.directory.new_cert,
             req,
             content_type=content_type,
@@ -356,7 +419,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         """
         # pylint: disable=too-many-locals
         assert max_attempts > 0
-        attempts = collections.defaultdict(int)
+        attempts = collections.defaultdict(int) # type: Dict[messages.AuthorizationResource, int]
         exhausted = set()
 
         # priority queue with datetime.datetime (based on Retry-After) as key,
@@ -470,7 +533,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        chain = []
+        chain = [] # type: List[jose.ComparableX509]
         uri = certr.cert_chain_uri
         while uri is not None and len(chain) < max_length:
             response, cert = self._get_cert(uri)
@@ -492,32 +555,374 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :raises .ClientError: If revocation is unsuccessful.
 
         """
-        response = self.net.post(self.directory[messages.Revocation],
-                                 messages.Revocation(
-                                     certificate=cert,
-                                     reason=rsn),
-                                 content_type=None)
-        if response.status_code != http_client.OK:
-            raise errors.ClientError(
-                'Successful revocation must return HTTP OK status')
+        return self._revoke(cert, rsn, self.directory[messages.Revocation])
+
+
+class ClientV2(ClientBase):
+    """ACME client for a v2 API.
+
+    :ivar messages.Directory directory:
+    :ivar .ClientNetwork net: Client network.
+    """
+
+    def __init__(self, directory, net):
+        """Initialize.
+
+        :param .messages.Directory directory: Directory Resource
+        :param .ClientNetwork net: Client network.
+        """
+        super(ClientV2, self).__init__(directory=directory,
+            net=net, acme_version=2)
+
+    def new_account(self, new_account):
+        """Register.
+
+        :param .NewRegistration new_account:
+
+        :raises .ConflictError: in case the account already exists
+
+        :returns: Registration Resource.
+        :rtype: `.RegistrationResource`
+        """
+        response = self._post(self.directory['newAccount'], new_account)
+        # if account already exists
+        if response.status_code == 200 and 'Location' in response.headers:
+            raise errors.ConflictError(response.headers.get('Location'))
+        # "Instance of 'Field' has no key/contact member" bug:
+        # pylint: disable=no-member
+        regr = self._regr_from_response(response)
+        self.net.account = regr
+        return regr
+
+    def query_registration(self, regr):
+        """Query server about registration.
+
+        :param messages.RegistrationResource: Existing Registration
+            Resource.
+
+        """
+        self.net.account = regr
+        updated_regr = super(ClientV2, self).query_registration(regr)
+        self.net.account = updated_regr
+        return updated_regr
+
+    def update_registration(self, regr, update=None):
+        """Update registration.
+
+        :param messages.RegistrationResource regr: Registration Resource.
+        :param messages.Registration update: Updated body of the
+            resource. If not provided, body will be taken from `regr`.
+
+        :returns: Updated Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        # https://github.com/certbot/certbot/issues/6155
+        new_regr = self._get_v2_account(regr)
+        return super(ClientV2, self).update_registration(new_regr, update)
+
+    def _get_v2_account(self, regr):
+        self.net.account = None
+        only_existing_reg = regr.body.update(only_return_existing=True)
+        response = self._post(self.directory['newAccount'], only_existing_reg)
+        updated_uri = response.headers['Location']
+        new_regr = regr.update(uri=updated_uri)
+        self.net.account = new_regr
+        return new_regr
+
+    def new_order(self, csr_pem):
+        """Request a new Order object from the server.
+
+        :param str csr_pem: A CSR in PEM format.
+
+        :returns: The newly created order.
+        :rtype: OrderResource
+        """
+        csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
+        # pylint: disable=protected-access
+        dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr)
+
+        identifiers = []
+        for name in dnsNames:
+            identifiers.append(messages.Identifier(typ=messages.IDENTIFIER_FQDN,
+                value=name))
+        order = messages.NewOrder(identifiers=identifiers)
+        response = self._post(self.directory['newOrder'], order)
+        body = messages.Order.from_json(response.json())
+        authorizations = []
+        for url in body.authorizations:
+            authorizations.append(self._authzr_from_response(self.net.get(url), uri=url))
+        return messages.OrderResource(
+            body=body,
+            uri=response.headers.get('Location'),
+            authorizations=authorizations,
+            csr_pem=csr_pem)
+
+    def poll_and_finalize(self, orderr, deadline=None):
+        """Poll authorizations and finalize the order.
+
+        If no deadline is provided, this method will timeout after 90
+        seconds.
+
+        :param messages.OrderResource orderr: order to finalize
+        :param datetime.datetime deadline: when to stop polling and timeout
+
+        :returns: finalized order
+        :rtype: messages.OrderResource
+
+        """
+        if deadline is None:
+            deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
+        orderr = self.poll_authorizations(orderr, deadline)
+        return self.finalize_order(orderr, deadline)
+
+    def poll_authorizations(self, orderr, deadline):
+        """Poll Order Resource for status."""
+        responses = []
+        for url in orderr.body.authorizations:
+            while datetime.datetime.now() < deadline:
+                authzr = self._authzr_from_response(self.net.get(url), uri=url)
+                if authzr.body.status != messages.STATUS_PENDING:
+                    responses.append(authzr)
+                    break
+                time.sleep(1)
+        # If we didn't get a response for every authorization, we fell through
+        # the bottom of the loop due to hitting the deadline.
+        if len(responses) < len(orderr.body.authorizations):
+            raise errors.TimeoutError()
+        failed = []
+        for authzr in responses:
+            if authzr.body.status != messages.STATUS_VALID:
+                for chall in authzr.body.challenges:
+                    if chall.error != None:
+                        failed.append(authzr)
+        if len(failed) > 0:
+            raise errors.ValidationError(failed)
+        return orderr.update(authorizations=responses)
+
+    def finalize_order(self, orderr, deadline):
+        """Finalize an order and obtain a certificate.
+
+        :param messages.OrderResource orderr: order to finalize
+        :param datetime.datetime deadline: when to stop polling and timeout
+
+        :returns: finalized order
+        :rtype: messages.OrderResource
+
+        """
+        csr = OpenSSL.crypto.load_certificate_request(
+            OpenSSL.crypto.FILETYPE_PEM, orderr.csr_pem)
+        wrapped_csr = messages.CertificateRequest(csr=jose.ComparableX509(csr))
+        self._post(orderr.body.finalize, wrapped_csr)
+        while datetime.datetime.now() < deadline:
+            time.sleep(1)
+            response = self.net.get(orderr.uri)
+            body = messages.Order.from_json(response.json())
+            if body.error is not None:
+                raise errors.IssuanceError(body.error)
+            if body.certificate is not None:
+                certificate_response = self.net.get(body.certificate,
+                                                    content_type=DER_CONTENT_TYPE).text
+                return orderr.update(body=body, fullchain_pem=certificate_response)
+        raise errors.TimeoutError()
+
+    def revoke(self, cert, rsn):
+        """Revoke certificate.
+
+        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
+            `.ComparableX509`
+
+        :param int rsn: Reason code for certificate revocation.
+
+        :raises .ClientError: If revocation is unsuccessful.
+
+        """
+        return self._revoke(cert, rsn, self.directory['revokeCert'])
+
+
+class BackwardsCompatibleClientV2(object):
+    """ACME client wrapper that tends towards V2-style calls, but
+    supports V1 servers.
+
+    .. note:: While this class handles the majority of the differences
+        between versions of the ACME protocol, if you need to support an
+        ACME server based on version 3 or older of the IETF ACME draft
+        that uses combinations in authorizations (or lack thereof) to
+        signal that the client needs to complete something other than
+        any single challenge in the authorization to make it valid, the
+        user of this class needs to understand and handle these
+        differences themselves.  This does not apply to either of Let's
+        Encrypt's endpoints where successfully completing any challenge
+        in an authorization will make it valid.
+
+    :ivar int acme_version: 1 or 2, corresponding to the Let's Encrypt endpoint
+    :ivar .ClientBase client: either Client or ClientV2
+    """
+
+    def __init__(self, net, key, server):
+        directory = messages.Directory.from_json(net.get(server).json())
+        self.acme_version = self._acme_version_from_directory(directory)
+        if self.acme_version == 1:
+            self.client = Client(directory, key=key, net=net)
+        else:
+            self.client = ClientV2(directory, net=net)
+
+    def __getattr__(self, name):
+        if name in vars(self.client):
+            return getattr(self.client, name)
+        elif name in dir(ClientBase):
+            return getattr(self.client, name)
+        else:
+            raise AttributeError()
+
+    def new_account_and_tos(self, regr, check_tos_cb=None):
+        """Combined register and agree_tos for V1, new_account for V2
+
+        :param .NewRegistration regr:
+        :param callable check_tos_cb: callback that raises an error if
+            the check does not work
+        """
+        def _assess_tos(tos):
+            if check_tos_cb is not None:
+                check_tos_cb(tos)
+        if self.acme_version == 1:
+            regr = self.client.register(regr)
+            if regr.terms_of_service is not None:
+                _assess_tos(regr.terms_of_service)
+                return self.client.agree_to_tos(regr)
+            return regr
+        else:
+            if "terms_of_service" in self.client.directory.meta:
+                _assess_tos(self.client.directory.meta.terms_of_service)
+                regr = regr.update(terms_of_service_agreed=True)
+            return self.client.new_account(regr)
+
+    def new_order(self, csr_pem):
+        """Request a new Order object from the server.
+
+        If using ACMEv1, returns a dummy OrderResource with only
+        the authorizations field filled in.
+
+        :param str csr_pem: A CSR in PEM format.
+
+        :returns: The newly created order.
+        :rtype: OrderResource
+
+        :raises errors.WildcardUnsupportedError: if a wildcard domain is
+            requested but unsupported by the ACME version
+
+        """
+        if self.acme_version == 1:
+            csr = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)
+            # pylint: disable=protected-access
+            dnsNames = crypto_util._pyopenssl_cert_or_req_all_names(csr)
+            authorizations = []
+            for domain in dnsNames:
+                authorizations.append(self.client.request_domain_challenges(domain))
+            return messages.OrderResource(authorizations=authorizations, csr_pem=csr_pem)
+        else:
+            return self.client.new_order(csr_pem)
+
+    def finalize_order(self, orderr, deadline):
+        """Finalize an order and obtain a certificate.
+
+        :param messages.OrderResource orderr: order to finalize
+        :param datetime.datetime deadline: when to stop polling and timeout
+
+        :returns: finalized order
+        :rtype: messages.OrderResource
+
+        """
+        if self.acme_version == 1:
+            csr_pem = orderr.csr_pem
+            certr = self.client.request_issuance(
+                jose.ComparableX509(
+                    OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, csr_pem)),
+                    orderr.authorizations)
+
+            chain = None
+            while datetime.datetime.now() < deadline:
+                try:
+                    chain = self.client.fetch_chain(certr)
+                    break
+                except errors.Error:
+                    time.sleep(1)
+
+            if chain is None:
+                raise errors.TimeoutError(
+                    'Failed to fetch chain. You should not deploy the generated '
+                    'certificate, please rerun the command for a new one.')
+
+            cert = OpenSSL.crypto.dump_certificate(
+                    OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped).decode()
+            chain = crypto_util.dump_pyopenssl_chain(chain).decode()
+
+            return orderr.update(fullchain_pem=(cert + chain))
+        else:
+            return self.client.finalize_order(orderr, deadline)
+
+    def revoke(self, cert, rsn):
+        """Revoke certificate.
+
+        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
+            `.ComparableX509`
+
+        :param int rsn: Reason code for certificate revocation.
+
+        :raises .ClientError: If revocation is unsuccessful.
+
+        """
+        return self.client.revoke(cert, rsn)
+
+    def _acme_version_from_directory(self, directory):
+        if hasattr(directory, 'newNonce'):
+            return 2
+        else:
+            return 1
 
 
 class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
-    """Client network."""
+    """Wrapper around requests that signs POSTs for authentication.
+
+    Also adds user agent, and handles Content-Type.
+    """
     JSON_CONTENT_TYPE = 'application/json'
     JOSE_CONTENT_TYPE = 'application/jose+json'
     JSON_ERROR_CONTENT_TYPE = 'application/problem+json'
     REPLAY_NONCE_HEADER = 'Replay-Nonce'
 
-    def __init__(self, key, alg=jose.RS256, verify_ssl=True,
-                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+    """Initialize.
+
+    :param josepy.JWK key: Account private key
+    :param messages.RegistrationResource account: Account object. Required if you are
+            planning to use .post() with acme_version=2 for anything other than
+            creating a new account; may be set later after registering.
+    :param josepy.JWASignature alg: Algoritm to use in signing JWS.
+    :param bool verify_ssl: Whether to verify certificates on SSL connections.
+    :param str user_agent: String to send as User-Agent header.
+    :param float timeout: Timeout for requests.
+    :param source_address: Optional source address to bind to when making requests.
+    :type source_address: str or tuple(str, int)
+    """
+    def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True,
+                 user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT,
+                 source_address=None):
+        # pylint: disable=too-many-arguments
         self.key = key
+        self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
-        self._nonces = set()
+        self._nonces = set() # type: Set[Text]
         self.user_agent = user_agent
         self.session = requests.Session()
         self._default_timeout = timeout
+        adapter = HTTPAdapter()
+
+        if source_address is not None:
+            adapter = SourceAddressAdapter(source_address)
+
+        self.session.mount("http://", adapter)
+        self.session.mount("https://", adapter)
 
     def __del__(self):
         # Try to close the session, but don't show exceptions to the
@@ -527,21 +932,32 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         except Exception:  # pylint: disable=broad-except
             pass
 
-    def _wrap_in_jws(self, obj, nonce):
+    def _wrap_in_jws(self, obj, nonce, url, acme_version):
         """Wrap `JSONDeSerializable` object in JWS.
 
         .. todo:: Implement ``acmePath``.
 
-        :param .JSONDeSerializable obj:
+        :param josepy.JSONDeSerializable obj:
+        :param str url: The URL to which this object will be POSTed
         :param bytes nonce:
-        :rtype: `.JWS`
+        :rtype: `josepy.JWS`
 
         """
         jobj = obj.json_dumps(indent=2).encode()
         logger.debug('JWS payload:\n%s', jobj)
-        return jws.JWS.sign(
-            payload=jobj, key=self.key, alg=self.alg,
-            nonce=nonce).json_dumps(indent=2)
+        kwargs = {
+            "alg": self.alg,
+            "nonce": nonce
+        }
+        if acme_version == 2:
+            kwargs["url"] = url
+            # newAccount and revokeCert work without the kid
+            # newAccount must not have kid
+            if self.account is not None:
+                kwargs["kid"] = self.account["uri"]
+        kwargs["key"] = self.key
+        # pylint: disable=star-args
+        return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2)
 
     @classmethod
     def _check_response(cls, response, content_type=None):
@@ -657,7 +1073,7 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         if response.headers.get("Content-Type") == DER_CONTENT_TYPE:
             debug_content = base64.b64encode(response.content)
         else:
-            debug_content = response.content
+            debug_content = response.content.decode("utf-8")
         logger.debug('Received response:\nHTTP %d\n%s\n\n%s',
                      response.status_code,
                      "\n".join(["{0}: {1}".format(k, v)
@@ -692,10 +1108,15 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         else:
             raise errors.MissingNonce(response)
 
-    def _get_nonce(self, url):
+    def _get_nonce(self, url, new_nonce_url):
         if not self._nonces:
             logger.debug('Requesting fresh nonce')
-            self._add_nonce(self.head(url))
+            if new_nonce_url is None:
+                response = self.head(url)
+            else:
+                # request a new nonce from the acme newNonce endpoint
+                response = self._check_response(self.head(new_nonce_url), content_type=None)
+            self._add_nonce(response)
         return self._nonces.pop()
 
     def post(self, *args, **kwargs):
@@ -714,9 +1135,15 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
             else:
                 raise
 
-    def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs):
-        data = self._wrap_in_jws(obj, self._get_nonce(url))
+    def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE,
+            acme_version=1, **kwargs):
+        try:
+            new_nonce_url = kwargs.pop('new_nonce_url')
+        except KeyError:
+            new_nonce_url = None
+        data = self._wrap_in_jws(obj, self._get_nonce(url, new_nonce_url), url, acme_version)
         kwargs.setdefault('headers', {'Content-Type': content_type})
         response = self._send_request('POST', url, data=data, **kwargs)
+        response = self._check_response(response, content_type=content_type)
         self._add_nonce(response)
-        return self._check_response(response, content_type=content_type)
+        return response

acme/acme/client_test.py

@@ -1,4 +1,5 @@
 """Tests for acme.client."""
+import copy
 import datetime
 import json
 import unittest
@@ -7,6 +8,7 @@ from six.moves import http_client  # pylint: disable=import-error
 
 import josepy as jose
 import mock
+import OpenSSL
 import requests
 
 from acme import challenges
@@ -15,16 +17,36 @@ from acme import jws as acme_jws
 from acme import messages
 from acme import messages_test
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT_DER = test_util.load_vector('cert.der')
+CERT_SAN_PEM = test_util.load_vector('cert-san.pem')
+CSR_SAN_PEM = test_util.load_vector('csr-san.pem')
 KEY = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
 KEY2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem'))
 
+DIRECTORY_V1 = messages.Directory({
+    messages.NewRegistration:
+        'https://www.letsencrypt-demo.org/acme/new-reg',
+    messages.Revocation:
+        'https://www.letsencrypt-demo.org/acme/revoke-cert',
+    messages.NewAuthorization:
+        'https://www.letsencrypt-demo.org/acme/new-authz',
+    messages.CertificateRequest:
+        'https://www.letsencrypt-demo.org/acme/new-cert',
+})
 
-class ClientTest(unittest.TestCase):
-    """Tests for  acme.client.Client."""
-    # pylint: disable=too-many-instance-attributes,too-many-public-methods
+DIRECTORY_V2 = messages.Directory({
+    'newAccount': 'https://www.letsencrypt-demo.org/acme/new-account',
+    'newNonce': 'https://www.letsencrypt-demo.org/acme/new-nonce',
+    'newOrder': 'https://www.letsencrypt-demo.org/acme/new-order',
+    'revokeCert': 'https://www.letsencrypt-demo.org/acme/revoke-cert',
+})
+
+
+class ClientTestBase(unittest.TestCase):
+    """Base for tests in acme.client."""
 
     def setUp(self):
         self.response = mock.MagicMock(
@@ -33,21 +55,6 @@ class ClientTest(unittest.TestCase):
         self.net.post.return_value = self.response
         self.net.get.return_value = self.response
 
-        self.directory = messages.Directory({
-            messages.NewRegistration:
-                'https://www.letsencrypt-demo.org/acme/new-reg',
-            messages.Revocation:
-                'https://www.letsencrypt-demo.org/acme/revoke-cert',
-            messages.NewAuthorization:
-                'https://www.letsencrypt-demo.org/acme/new-authz',
-            messages.CertificateRequest:
-                'https://www.letsencrypt-demo.org/acme/new-cert',
-        })
-
-        from acme.client import Client
-        self.client = Client(
-            directory=self.directory, key=KEY, alg=jose.RS256, net=self.net)
-
         self.identifier = messages.Identifier(
             typ=messages.IDENTIFIER_FQDN, value='example.com')
 
@@ -55,10 +62,10 @@ class ClientTest(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        self.new_reg = messages.NewRegistration(**dict(reg))
+        the_arg = dict(reg) # type: Dict
+        self.new_reg = messages.NewRegistration(**the_arg) # pylint: disable=star-args
         self.regr = messages.RegistrationResource(
-            body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1',
-            terms_of_service='https://www.letsencrypt-demo.org/tos')
+            body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
 
         # Authorization
         authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1'
@@ -75,14 +82,230 @@ class ClientTest(unittest.TestCase):
         self.authzr = messages.AuthorizationResource(
             body=self.authz, uri=authzr_uri)
 
+        # Reason code for revocation
+        self.rsn = 1
+
+
+class BackwardsCompatibleClientV2Test(ClientTestBase):
+    """Tests for  acme.client.BackwardsCompatibleClientV2."""
+
+    def setUp(self):
+        super(BackwardsCompatibleClientV2Test, self).setUp()
+        # contains a loaded cert
+        self.certr = messages.CertificateResource(
+            body=messages_test.CERT)
+
+        loaded = OpenSSL.crypto.load_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, CERT_SAN_PEM)
+        wrapped = jose.ComparableX509(loaded)
+        self.chain = [wrapped, wrapped]
+
+        self.cert_pem = OpenSSL.crypto.dump_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, messages_test.CERT.wrapped).decode()
+
+        single_chain = OpenSSL.crypto.dump_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, loaded).decode()
+        self.chain_pem = single_chain + single_chain
+
+        self.fullchain_pem = self.cert_pem + self.chain_pem
+
+        self.orderr = messages.OrderResource(
+            csr_pem=CSR_SAN_PEM)
+
+    def _init(self):
+        uri = 'http://www.letsencrypt-demo.org/directory'
+        from acme.client import BackwardsCompatibleClientV2
+        return BackwardsCompatibleClientV2(net=self.net,
+            key=KEY, server=uri)
+
+    def test_init_downloads_directory(self):
+        uri = 'http://www.letsencrypt-demo.org/directory'
+        from acme.client import BackwardsCompatibleClientV2
+        BackwardsCompatibleClientV2(net=self.net,
+            key=KEY, server=uri)
+        self.net.get.assert_called_once_with(uri)
+
+    def test_init_acme_version(self):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        client = self._init()
+        self.assertEqual(client.acme_version, 1)
+
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        client = self._init()
+        self.assertEqual(client.acme_version, 2)
+
+    def test_query_registration_client_v2(self):
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        client = self._init()
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, client.query_registration(self.regr))
+
+    def test_forwarding(self):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        client = self._init()
+        self.assertEqual(client.directory, client.client.directory)
+        self.assertEqual(client.key, KEY)
+        self.assertEqual(client.deactivate_registration, client.client.deactivate_registration)
+        self.assertRaises(AttributeError, client.__getattr__, 'nonexistent')
+        self.assertRaises(AttributeError, client.__getattr__, 'new_account_and_tos')
+        self.assertRaises(AttributeError, client.__getattr__, 'new_account')
+
+    def test_new_account_and_tos(self):
+        # v2 no tos
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            client = self._init()
+            client.new_account_and_tos(self.new_reg)
+            mock_client().new_account.assert_called_with(self.new_reg)
+
+        # v2 tos good
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            mock_client().directory.meta.__contains__.return_value = True
+            client = self._init()
+            client.new_account_and_tos(self.new_reg, lambda x: True)
+            mock_client().new_account.assert_called_with(
+                self.new_reg.update(terms_of_service_agreed=True))
+
+        # v2 tos bad
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            mock_client().directory.meta.__contains__.return_value = True
+            client = self._init()
+            def _tos_cb(tos):
+                raise errors.Error
+            self.assertRaises(errors.Error, client.new_account_and_tos,
+                self.new_reg, _tos_cb)
+            mock_client().new_account.assert_not_called()
+
+        # v1 yes tos
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        with mock.patch('acme.client.Client') as mock_client:
+            regr = mock.MagicMock(terms_of_service="TOS")
+            mock_client().register.return_value = regr
+            client = self._init()
+            client.new_account_and_tos(self.new_reg)
+            mock_client().register.assert_called_once_with(self.new_reg)
+            mock_client().agree_to_tos.assert_called_once_with(regr)
+
+        # v1 no tos
+        with mock.patch('acme.client.Client') as mock_client:
+            regr = mock.MagicMock(terms_of_service=None)
+            mock_client().register.return_value = regr
+            client = self._init()
+            client.new_account_and_tos(self.new_reg)
+            mock_client().register.assert_called_once_with(self.new_reg)
+            mock_client().agree_to_tos.assert_not_called()
+
+    @mock.patch('OpenSSL.crypto.load_certificate_request')
+    @mock.patch('acme.crypto_util._pyopenssl_cert_or_req_all_names')
+    def test_new_order_v1(self, mock__pyopenssl_cert_or_req_all_names,
+        unused_mock_load_certificate_request):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        mock__pyopenssl_cert_or_req_all_names.return_value = ['example.com', 'www.example.com']
+        mock_csr_pem = mock.MagicMock()
+        with mock.patch('acme.client.Client') as mock_client:
+            mock_client().request_domain_challenges.return_value = mock.sentinel.auth
+            client = self._init()
+            orderr = client.new_order(mock_csr_pem)
+            self.assertEqual(orderr.authorizations, [mock.sentinel.auth, mock.sentinel.auth])
+
+    def test_new_order_v2(self):
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        mock_csr_pem = mock.MagicMock()
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            client = self._init()
+            client.new_order(mock_csr_pem)
+            mock_client().new_order.assert_called_once_with(mock_csr_pem)
+
+    @mock.patch('acme.client.Client')
+    def test_finalize_order_v1_success(self, mock_client):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+
+        mock_client().request_issuance.return_value = self.certr
+        mock_client().fetch_chain.return_value = self.chain
+
+        deadline = datetime.datetime(9999, 9, 9)
+        client = self._init()
+        result = client.finalize_order(self.orderr, deadline)
+        self.assertEqual(result.fullchain_pem, self.fullchain_pem)
+        mock_client().fetch_chain.assert_called_once_with(self.certr)
+
+    @mock.patch('acme.client.Client')
+    def test_finalize_order_v1_fetch_chain_error(self, mock_client):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+
+        mock_client().request_issuance.return_value = self.certr
+        mock_client().fetch_chain.return_value = self.chain
+        mock_client().fetch_chain.side_effect = [errors.Error, self.chain]
+
+        deadline = datetime.datetime(9999, 9, 9)
+        client = self._init()
+        result = client.finalize_order(self.orderr, deadline)
+        self.assertEqual(result.fullchain_pem, self.fullchain_pem)
+        self.assertEqual(mock_client().fetch_chain.call_count, 2)
+
+    @mock.patch('acme.client.Client')
+    def test_finalize_order_v1_timeout(self, mock_client):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+
+        mock_client().request_issuance.return_value = self.certr
+
+        deadline = deadline = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        client = self._init()
+        self.assertRaises(errors.TimeoutError, client.finalize_order,
+            self.orderr, deadline)
+
+    def test_finalize_order_v2(self):
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        mock_orderr = mock.MagicMock()
+        mock_deadline = mock.MagicMock()
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            client = self._init()
+            client.finalize_order(mock_orderr, mock_deadline)
+            mock_client().finalize_order.assert_called_once_with(mock_orderr, mock_deadline)
+
+    def test_revoke(self):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        with mock.patch('acme.client.Client') as mock_client:
+            client = self._init()
+            client.revoke(messages_test.CERT, self.rsn)
+        mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn)
+
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        with mock.patch('acme.client.ClientV2') as mock_client:
+            client = self._init()
+            client.revoke(messages_test.CERT, self.rsn)
+        mock_client().revoke.assert_called_once_with(messages_test.CERT, self.rsn)
+
+    def test_update_registration(self):
+        self.response.json.return_value = DIRECTORY_V1.to_json()
+        with mock.patch('acme.client.Client') as mock_client:
+            client = self._init()
+            client.update_registration(mock.sentinel.regr, None)
+        mock_client().update_registration.assert_called_once_with(mock.sentinel.regr, None)
+
+
+class ClientTest(ClientTestBase):
+    """Tests for acme.client.Client."""
+    # pylint: disable=too-many-instance-attributes,too-many-public-methods
+
+    def setUp(self):
+        super(ClientTest, self).setUp()
+
+        self.directory = DIRECTORY_V1
+
+        # Registration
+        self.regr = self.regr.update(
+            terms_of_service='https://www.letsencrypt-demo.org/tos')
+
         # Request issuance
         self.certr = messages.CertificateResource(
             body=messages_test.CERT, authzrs=(self.authzr,),
             uri='https://www.letsencrypt-demo.org/acme/cert/1',
             cert_chain_uri='https://www.letsencrypt-demo.org/ca')
 
-        # Reason code for revocation
-        self.rsn = 1
+        from acme.client import Client
+        self.client = Client(
+            directory=self.directory, key=KEY, alg=jose.RS256, net=self.net)
 
     def test_init_downloads_directory(self):
         uri = 'http://www.letsencrypt-demo.org/directory'
@@ -91,6 +314,16 @@ class ClientTest(unittest.TestCase):
             directory=uri, key=KEY, alg=jose.RS256, net=self.net)
         self.net.get.assert_called_once_with(uri)
 
+    @mock.patch('acme.client.ClientNetwork')
+    def test_init_without_net(self, mock_net):
+        mock_net.return_value = mock.sentinel.net
+        alg = jose.RS256
+        from acme.client import Client
+        self.client = Client(
+            directory=self.directory, key=KEY, alg=alg)
+        mock_net.called_once_with(KEY, alg=alg, verify_ssl=True)
+        self.assertEqual(self.client.net, mock.sentinel.net)
+
     def test_register(self):
         # "Instance of 'Field' has no to_json/update member" bug:
         # pylint: disable=no-member
@@ -142,20 +375,23 @@ class ClientTest(unittest.TestCase):
         self.client.request_challenges(self.identifier)
         self.net.post.assert_called_once_with(
             self.directory.new_authz,
-            messages.NewAuthorization(identifier=self.identifier))
+            messages.NewAuthorization(identifier=self.identifier),
+            acme_version=1)
 
     def test_request_challenges_deprecated_arg(self):
         self._prepare_response_for_request_challenges()
         self.client.request_challenges(self.identifier, new_authzr_uri="hi")
         self.net.post.assert_called_once_with(
             self.directory.new_authz,
-            messages.NewAuthorization(identifier=self.identifier))
+            messages.NewAuthorization(identifier=self.identifier),
+            acme_version=1)
 
     def test_request_challenges_custom_uri(self):
         self._prepare_response_for_request_challenges()
         self.client.request_challenges(self.identifier)
         self.net.post.assert_called_once_with(
-            'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY)
+            'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY,
+            acme_version=1)
 
     def test_request_challenges_unexpected_update(self):
         self._prepare_response_for_request_challenges()
@@ -165,6 +401,13 @@ class ClientTest(unittest.TestCase):
             errors.UnexpectedUpdate, self.client.request_challenges,
             self.identifier)
 
+    def test_request_challenges_wildcard(self):
+        wildcard_identifier = messages.Identifier(
+            typ=messages.IDENTIFIER_FQDN, value='*.example.org')
+        self.assertRaises(
+            errors.WildcardUnsupportedError, self.client.request_challenges,
+            wildcard_identifier)
+
     def test_request_domain_challenges(self):
         self.client.request_challenges = mock.MagicMock()
         self.assertEqual(
@@ -417,7 +660,7 @@ class ClientTest(unittest.TestCase):
     def test_revoke(self):
         self.client.revoke(self.certr.body, self.rsn)
         self.net.post.assert_called_once_with(
-            self.directory[messages.Revocation], mock.ANY, content_type=None)
+            self.directory[messages.Revocation], mock.ANY, acme_version=1)
 
     def test_revocation_payload(self):
         obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn)
@@ -432,9 +675,169 @@ class ClientTest(unittest.TestCase):
             self.certr,
             self.rsn)
 
+class ClientV2Test(ClientTestBase):
+    """Tests for acme.client.ClientV2."""
+
+    def setUp(self):
+        super(ClientV2Test, self).setUp()
+
+        self.directory = DIRECTORY_V2
+
+        from acme.client import ClientV2
+        self.client = ClientV2(self.directory, self.net)
+
+        self.new_reg = self.new_reg.update(terms_of_service_agreed=True)
+
+        self.authzr_uri2 = 'https://www.letsencrypt-demo.org/acme/authz/2'
+        self.authz2 = self.authz.update(identifier=messages.Identifier(
+            typ=messages.IDENTIFIER_FQDN, value='www.example.com'),
+            status=messages.STATUS_PENDING)
+        self.authzr2 = messages.AuthorizationResource(
+            body=self.authz2, uri=self.authzr_uri2)
+
+        self.order = messages.Order(
+            identifiers=(self.authz.identifier, self.authz2.identifier),
+            status=messages.STATUS_PENDING,
+            authorizations=(self.authzr.uri, self.authzr_uri2),
+            finalize='https://www.letsencrypt-demo.org/acme/acct/1/order/1/finalize')
+        self.orderr = messages.OrderResource(
+            body=self.order,
+            uri='https://www.letsencrypt-demo.org/acme/acct/1/order/1',
+            authorizations=[self.authzr, self.authzr2], csr_pem=CSR_SAN_PEM)
+
+    def test_new_account(self):
+        self.response.status_code = http_client.CREATED
+        self.response.json.return_value = self.regr.body.to_json()
+        self.response.headers['Location'] = self.regr.uri
+
+        self.assertEqual(self.regr, self.client.new_account(self.new_reg))
+
+    def test_new_account_conflict(self):
+        self.response.status_code = http_client.OK
+        self.response.headers['Location'] = self.regr.uri
+        self.assertRaises(errors.ConflictError, self.client.new_account, self.new_reg)
+
+    def test_new_order(self):
+        order_response = copy.deepcopy(self.response)
+        order_response.status_code = http_client.CREATED
+        order_response.json.return_value = self.order.to_json()
+        order_response.headers['Location'] = self.orderr.uri
+        self.net.post.return_value = order_response
+
+        authz_response = copy.deepcopy(self.response)
+        authz_response.json.return_value = self.authz.to_json()
+        authz_response.headers['Location'] = self.authzr.uri
+        authz_response2 = self.response
+        authz_response2.json.return_value = self.authz2.to_json()
+        authz_response2.headers['Location'] = self.authzr2.uri
+        self.net.get.side_effect = (authz_response, authz_response2)
+
+        self.assertEqual(self.client.new_order(CSR_SAN_PEM), self.orderr)
+
+    @mock.patch('acme.client.datetime')
+    def test_poll_and_finalize(self, mock_datetime):
+        mock_datetime.datetime.now.return_value = datetime.datetime(2018, 2, 15)
+        mock_datetime.timedelta = datetime.timedelta
+        expected_deadline = mock_datetime.datetime.now() + datetime.timedelta(seconds=90)
+
+        self.client.poll_authorizations = mock.Mock(return_value=self.orderr)
+        self.client.finalize_order = mock.Mock(return_value=self.orderr)
+
+        self.assertEqual(self.client.poll_and_finalize(self.orderr), self.orderr)
+        self.client.poll_authorizations.assert_called_once_with(self.orderr, expected_deadline)
+        self.client.finalize_order.assert_called_once_with(self.orderr, expected_deadline)
+
+    @mock.patch('acme.client.datetime')
+    def test_poll_authorizations_timeout(self, mock_datetime):
+        now_side_effect = [datetime.datetime(2018, 2, 15),
+                           datetime.datetime(2018, 2, 16),
+                           datetime.datetime(2018, 2, 17)]
+        mock_datetime.datetime.now.side_effect = now_side_effect
+        self.response.json.side_effect = [
+            self.authz.to_json(), self.authz2.to_json(), self.authz2.to_json()]
+
+        self.assertRaises(
+            errors.TimeoutError, self.client.poll_authorizations, self.orderr, now_side_effect[1])
+
+    def test_poll_authorizations_failure(self):
+        deadline = datetime.datetime(9999, 9, 9)
+        challb = self.challr.body.update(status=messages.STATUS_INVALID,
+                                         error=messages.Error.with_code('unauthorized'))
+        authz = self.authz.update(status=messages.STATUS_INVALID, challenges=(challb,))
+        self.response.json.return_value = authz.to_json()
+
+        self.assertRaises(
+            errors.ValidationError, self.client.poll_authorizations, self.orderr, deadline)
+
+    def test_poll_authorizations_success(self):
+        deadline = datetime.datetime(9999, 9, 9)
+        updated_authz2 = self.authz2.update(status=messages.STATUS_VALID)
+        updated_authzr2 = messages.AuthorizationResource(
+            body=updated_authz2, uri=self.authzr_uri2)
+        updated_orderr = self.orderr.update(authorizations=[self.authzr, updated_authzr2])
+
+        self.response.json.side_effect = (
+            self.authz.to_json(), self.authz2.to_json(), updated_authz2.to_json())
+        self.assertEqual(self.client.poll_authorizations(self.orderr, deadline), updated_orderr)
+
+    def test_finalize_order_success(self):
+        updated_order = self.order.update(
+            certificate='https://www.letsencrypt-demo.org/acme/cert/')
+        updated_orderr = self.orderr.update(body=updated_order, fullchain_pem=CERT_SAN_PEM)
+
+        self.response.json.return_value = updated_order.to_json()
+        self.response.text = CERT_SAN_PEM
+
+        deadline = datetime.datetime(9999, 9, 9)
+        self.assertEqual(self.client.finalize_order(self.orderr, deadline), updated_orderr)
+
+    def test_finalize_order_error(self):
+        updated_order = self.order.update(error=messages.Error.with_code('unauthorized'))
+        self.response.json.return_value = updated_order.to_json()
+
+        deadline = datetime.datetime(9999, 9, 9)
+        self.assertRaises(errors.IssuanceError, self.client.finalize_order, self.orderr, deadline)
+
+    def test_finalize_order_timeout(self):
+        deadline = datetime.datetime.now() - datetime.timedelta(seconds=60)
+        self.assertRaises(errors.TimeoutError, self.client.finalize_order, self.orderr, deadline)
+
+    def test_revoke(self):
+        self.client.revoke(messages_test.CERT, self.rsn)
+        self.net.post.assert_called_once_with(
+            self.directory["revokeCert"], mock.ANY, acme_version=2,
+            new_nonce_url=DIRECTORY_V2['newNonce'])
+
+    def test_update_registration(self):
+        # "Instance of 'Field' has no to_json/update member" bug:
+        # pylint: disable=no-member
+        self.response.headers['Location'] = self.regr.uri
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, self.client.update_registration(self.regr))
+        self.assertNotEqual(self.client.net.account, None)
+        self.assertEqual(self.client.net.post.call_count, 2)
+        self.assertTrue(DIRECTORY_V2.newAccount in self.net.post.call_args_list[0][0])
+
+        self.response.json.return_value = self.regr.body.update(
+            contact=()).to_json()
+
+
+class MockJSONDeSerializable(jose.JSONDeSerializable):
+    # pylint: disable=missing-docstring
+    def __init__(self, value):
+        self.value = value
+
+    def to_partial_json(self):
+        return {'foo': self.value}
+
+    @classmethod
+    def from_json(cls, value):
+        pass  # pragma: no cover
+
 
 class ClientNetworkTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork."""
+    # pylint: disable=too-many-public-methods
 
     def setUp(self):
         self.verify_ssl = mock.MagicMock()
@@ -453,25 +856,27 @@ class ClientNetworkTest(unittest.TestCase):
         self.assertTrue(self.net.verify_ssl is self.verify_ssl)
 
     def test_wrap_in_jws(self):
-        class MockJSONDeSerializable(jose.JSONDeSerializable):
-            # pylint: disable=missing-docstring
-            def __init__(self, value):
-                self.value = value
-
-            def to_partial_json(self):
-                return {'foo': self.value}
-
-            @classmethod
-            def from_json(cls, value):
-                pass  # pragma: no cover
-
         # pylint: disable=protected-access
         jws_dump = self.net._wrap_in_jws(
-            MockJSONDeSerializable('foo'), nonce=b'Tg')
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url",
+            acme_version=1)
         jws = acme_jws.JWS.json_loads(jws_dump)
         self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
         self.assertEqual(jws.signature.combined.nonce, b'Tg')
 
+    def test_wrap_in_jws_v2(self):
+        self.net.account = {'uri': 'acct-uri'}
+        # pylint: disable=protected-access
+        jws_dump = self.net._wrap_in_jws(
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url",
+            acme_version=2)
+        jws = acme_jws.JWS.json_loads(jws_dump)
+        self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
+        self.assertEqual(jws.signature.combined.nonce, b'Tg')
+        self.assertEqual(jws.signature.combined.kid, u'acct-uri')
+        self.assertEqual(jws.signature.combined.url, u'url')
+
+
     def test_check_response_not_ok_jobj_no_error(self):
         self.response.ok = False
         self.response.json.return_value = {}
@@ -634,8 +1039,8 @@ class ClientNetworkTest(unittest.TestCase):
 
         # Requests Library Exceptions
         except requests.exceptions.ConnectionError as z: #pragma: no cover
-            self.assertEqual("('Connection aborted.', "
-                             "error(111, 'Connection refused'))", str(z))
+            self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))"
+                              == str(z) or "[WinError 10061]" in str(z))
 
 class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork which mock out response."""
@@ -648,7 +1053,10 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.response = mock.MagicMock(ok=True, status_code=http_client.OK)
         self.response.headers = {}
         self.response.links = {}
-        self.checked_response = mock.MagicMock()
+        self.response.checked = False
+        self.acmev1_nonce_response = mock.MagicMock(ok=False,
+            status_code=http_client.METHOD_NOT_ALLOWED)
+        self.acmev1_nonce_response.headers = {}
         self.obj = mock.MagicMock()
         self.wrapped_obj = mock.MagicMock()
         self.content_type = mock.sentinel.content_type
@@ -660,13 +1068,21 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
 
         def send_request(*args, **kwargs):
             # pylint: disable=unused-argument,missing-docstring
+            self.assertFalse("new_nonce_url" in kwargs)
+            method = args[0]
+            uri = args[1]
+            if method == 'HEAD' and uri != "new_nonce_uri":
+                response = self.acmev1_nonce_response
+            else:
+                response = self.response
+
             if self.available_nonces:
-                self.response.headers = {
+                response.headers = {
                     self.net.REPLAY_NONCE_HEADER:
                     self.available_nonces.pop().decode()}
             else:
-                self.response.headers = {}
-            return self.response
+                response.headers = {}
+            return response
 
         # pylint: disable=protected-access
         self.net._send_request = self.send_request = mock.MagicMock(
@@ -678,36 +1094,47 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         # pylint: disable=missing-docstring
         self.assertEqual(self.response, response)
         self.assertEqual(self.content_type, content_type)
-        return self.checked_response
+        self.assertTrue(self.response.ok)
+        self.response.checked = True
+        return self.response
 
     def test_head(self):
-        self.assertEqual(self.response, self.net.head(
+        self.assertEqual(self.acmev1_nonce_response, self.net.head(
             'http://example.com/', 'foo', bar='baz'))
         self.send_request.assert_called_once_with(
             'HEAD', 'http://example.com/', 'foo', bar='baz')
 
+    def test_head_v2(self):
+        self.assertEqual(self.response, self.net.head(
+            'new_nonce_uri', 'foo', bar='baz'))
+        self.send_request.assert_called_once_with(
+            'HEAD', 'new_nonce_uri', 'foo', bar='baz')
+
     def test_get(self):
-        self.assertEqual(self.checked_response, self.net.get(
+        self.assertEqual(self.response, self.net.get(
             'http://example.com/', content_type=self.content_type, bar='baz'))
+        self.assertTrue(self.response.checked)
         self.send_request.assert_called_once_with(
             'GET', 'http://example.com/', bar='baz')
 
     def test_post_no_content_type(self):
         self.content_type = self.net.JOSE_CONTENT_TYPE
-        self.assertEqual(self.checked_response, self.net.post('uri', self.obj))
+        self.assertEqual(self.response, self.net.post('uri', self.obj))
+        self.assertTrue(self.response.checked)
 
     def test_post(self):
         # pylint: disable=protected-access
-        self.assertEqual(self.checked_response, self.net.post(
+        self.assertEqual(self.response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
+        self.assertTrue(self.response.checked)
         self.net._wrap_in_jws.assert_called_once_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1)
 
         self.available_nonces = []
         self.assertRaises(errors.MissingNonce, self.net.post,
                           'uri', self.obj, content_type=self.content_type)
         self.net._wrap_in_jws.assert_called_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1)
 
     def test_post_wrong_initial_nonce(self):  # HEAD
         self.available_nonces = [b'f', jose.b64encode(b'good')]
@@ -731,7 +1158,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     def test_post_not_retried(self):
         check_response = mock.MagicMock()
         check_response.side_effect = [messages.Error.with_code('malformed'),
-                                      self.checked_response]
+                                      self.response]
 
         # pylint: disable=protected-access
         self.net._check_response = check_response
@@ -739,13 +1166,12 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
                           self.obj, content_type=self.content_type)
 
     def test_post_successful_retry(self):
-        check_response = mock.MagicMock()
-        check_response.side_effect = [messages.Error.with_code('badNonce'),
-                                      self.checked_response]
+        post_once = mock.MagicMock()
+        post_once.side_effect = [messages.Error.with_code('badNonce'),
+                                      self.response]
 
         # pylint: disable=protected-access
-        self.net._check_response = check_response
-        self.assertEqual(self.checked_response, self.net.post(
+        self.assertEqual(self.response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
 
     def test_head_get_post_error_passthrough(self):
@@ -756,6 +1182,51 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertRaises(requests.exceptions.RequestException,
                           self.net.post, 'uri', obj=self.obj)
 
+    def test_post_bad_nonce_head(self):
+        # pylint: disable=protected-access
+        # regression test for https://github.com/certbot/certbot/issues/6092
+        bad_response = mock.MagicMock(ok=False, status_code=http_client.SERVICE_UNAVAILABLE)
+        self.net._send_request = mock.MagicMock()
+        self.net._send_request.return_value = bad_response
+        self.content_type = None
+        check_response = mock.MagicMock()
+        self.net._check_response = check_response
+        self.assertRaises(errors.ClientError, self.net.post, 'uri',
+                          self.obj, content_type=self.content_type, acme_version=2,
+                          new_nonce_url='new_nonce_uri')
+        self.assertEqual(check_response.call_count, 1)
+
+    def test_new_nonce_uri_removed(self):
+        self.content_type = None
+        self.net.post('uri', self.obj, content_type=None,
+            acme_version=2, new_nonce_url='new_nonce_uri')
+
+
+class ClientNetworkSourceAddressBindingTest(unittest.TestCase):
+    """Tests that if ClientNetwork has a source IP set manually, the underlying library has
+    used the provided source address."""
+
+    def setUp(self):
+        self.source_address = "8.8.8.8"
+
+    def test_source_address_set(self):
+        from acme.client import ClientNetwork
+        net = ClientNetwork(key=None, alg=None, source_address=self.source_address)
+        for adapter in net.session.adapters.values():
+            self.assertTrue(self.source_address in adapter.source_address)
+
+    def test_behavior_assumption(self):
+        """This is a test that guardrails the HTTPAdapter behavior so that if the default for
+        a Session() changes, the assumptions here aren't violated silently."""
+        from acme.client import ClientNetwork
+        # Source address not specified, so the default adapter type should be bound -- this
+        # test should fail if the default adapter type is changed by requests
+        net = ClientNetwork(key=None, alg=None)
+        session = requests.Session()
+        for scheme in session.adapters.keys():
+            client_network_adapter = net.session.adapters.get(scheme)
+            default_adapter = session.adapters.get(scheme)
+            self.assertEqual(client_network_adapter.__class__, default_adapter.__class__)
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/crypto_util.py

@@ -5,11 +5,15 @@ import logging
 import os
 import re
 import socket
-import sys
 
-import OpenSSL
+from OpenSSL import crypto
+from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052
+import josepy as jose
 
 from acme import errors
+# pylint: disable=unused-import, no-name-in-module
+from acme.magic_typing import Callable, Union, Tuple, Optional
+# pylint: enable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -24,7 +28,7 @@ logger = logging.getLogger(__name__)
 # https://www.openssl.org/docs/ssl/SSLv23_method.html). _serve_sni
 # should be changed to use "set_options" to disable SSLv2 and SSLv3,
 # in case it's used for things other than probing/serving!
-_DEFAULT_TLSSNI01_SSL_METHOD = OpenSSL.SSL.SSLv23_METHOD  # type: ignore
+_DEFAULT_TLSSNI01_SSL_METHOD = SSL.SSLv23_METHOD  # type: ignore
 
 
 class SSLSocket(object):  # pylint: disable=too-few-public-methods
@@ -63,9 +67,9 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
             logger.debug("Server name (%s) not recognized, dropping SSL",
                          server_name)
             return
-        new_context = OpenSSL.SSL.Context(self.method)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        new_context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        new_context = SSL.Context(self.method)
+        new_context.set_options(SSL.OP_NO_SSLv2)
+        new_context.set_options(SSL.OP_NO_SSLv3)
         new_context.use_privatekey(key)
         new_context.use_certificate(cert)
         connection.set_context(new_context)
@@ -88,18 +92,18 @@ class SSLSocket(object):  # pylint: disable=too-few-public-methods
     def accept(self):  # pylint: disable=missing-docstring
         sock, addr = self.sock.accept()
 
-        context = OpenSSL.SSL.Context(self.method)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv2)
-        context.set_options(OpenSSL.SSL.OP_NO_SSLv3)
+        context = SSL.Context(self.method)
+        context.set_options(SSL.OP_NO_SSLv2)
+        context.set_options(SSL.OP_NO_SSLv3)
         context.set_tlsext_servername_callback(self._pick_certificate_cb)
 
-        ssl_sock = self.FakeConnection(OpenSSL.SSL.Connection(context, sock))
+        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
         ssl_sock.set_accept_state()
 
         logger.debug("Performing handshake with %s", addr)
         try:
             ssl_sock.do_handshake()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             # _pick_certificate_cb might have returned without
             # creating SSL context (wrong server name)
             raise socket.error(error)
@@ -127,31 +131,33 @@ def probe_sni(name, host, port=443, timeout=300,
     :rtype: OpenSSL.crypto.X509
 
     """
-    context = OpenSSL.SSL.Context(method)
+    context = SSL.Context(method)
     context.set_timeout(timeout)
 
-    socket_kwargs = {} if sys.version_info < (2, 7) else {
-        'source_address': source_address}
-
-    host_protocol_agnostic = None if host == '::' or host == '0' else host
+    socket_kwargs = {'source_address': source_address}
 
     try:
         # pylint: disable=star-args
-        logger.debug("Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
-            " from {0}:{1}".format(source_address[0], source_address[1]) if \
-            socket_kwargs else "")
-        sock = socket.create_connection((host_protocol_agnostic, port), **socket_kwargs)
+        logger.debug(
+            "Attempting to connect to %s:%d%s.", host, port,
+            " from {0}:{1}".format(
+                source_address[0],
+                source_address[1]
+            ) if socket_kwargs else ""
+        )
+        socket_tuple = (host, port)  # type: Tuple[str, int]
+        sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)
 
     with contextlib.closing(sock) as client:
-        client_ssl = OpenSSL.SSL.Connection(context, client)
+        client_ssl = SSL.Connection(context, client)
         client_ssl.set_connect_state()
         client_ssl.set_tlsext_host_name(name)  # pyOpenSSL>=0.13
         try:
             client_ssl.do_handshake()
             client_ssl.shutdown()
-        except OpenSSL.SSL.Error as error:
+        except SSL.Error as error:
             raise errors.Error(error)
     return client_ssl.get_peer_certificate()
 
@@ -164,18 +170,18 @@ def make_csr(private_key_pem, domains, must_staple=False):
         OCSP Must Staple: https://tools.ietf.org/html/rfc7633).
     :returns: buffer PEM-encoded Certificate Signing Request.
     """
-    private_key = OpenSSL.crypto.load_privatekey(
-        OpenSSL.crypto.FILETYPE_PEM, private_key_pem)
-    csr = OpenSSL.crypto.X509Req()
+    private_key = crypto.load_privatekey(
+        crypto.FILETYPE_PEM, private_key_pem)
+    csr = crypto.X509Req()
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b'subjectAltName',
             critical=False,
             value=', '.join('DNS:' + d for d in domains).encode('ascii')
         ),
     ]
     if must_staple:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"1.3.6.1.5.5.7.1.24",
             critical=False,
             value=b"DER:30:03:02:01:05"))
@@ -183,8 +189,17 @@ def make_csr(private_key_pem, domains, must_staple=False):
     csr.set_pubkey(private_key)
     csr.set_version(2)
     csr.sign(private_key, 'sha256')
-    return OpenSSL.crypto.dump_certificate_request(
-        OpenSSL.crypto.FILETYPE_PEM, csr)
+    return crypto.dump_certificate_request(
+        crypto.FILETYPE_PEM, csr)
+
+def _pyopenssl_cert_or_req_all_names(loaded_cert_or_req):
+    common_name = loaded_cert_or_req.get_subject().CN
+    sans = _pyopenssl_cert_or_req_san(loaded_cert_or_req)
+
+    if common_name is None:
+        return sans
+    else:
+        return [common_name] + [d for d in sans if d != common_name]
 
 def _pyopenssl_cert_or_req_san(cert_or_req):
     """Get Subject Alternative Names from certificate or CSR using pyOpenSSL.
@@ -212,11 +227,12 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
     parts_separator = ", "
     prefix = "DNS" + part_separator
 
-    if isinstance(cert_or_req, OpenSSL.crypto.X509):
-        func = OpenSSL.crypto.dump_certificate
+    if isinstance(cert_or_req, crypto.X509):
+        # pylint: disable=line-too-long
+        func = crypto.dump_certificate # type: Union[Callable[[int, crypto.X509Req], bytes], Callable[[int, crypto.X509], bytes]]
     else:
-        func = OpenSSL.crypto.dump_certificate_request
-    text = func(OpenSSL.crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
+        func = crypto.dump_certificate_request
+    text = func(crypto.FILETYPE_TEXT, cert_or_req).decode("utf-8")
     # WARNING: this function does not support multiple SANs extensions.
     # Multiple X509v3 extensions of the same type is disallowed by RFC 5280.
     match = re.search(r"X509v3 Subject Alternative Name:(?: critical)?\s*(.*)", text)
@@ -243,12 +259,12 @@ def gen_ss_cert(key, domains, not_before=None,
 
     """
     assert domains, "Must provide one or more hostnames for the cert."
-    cert = OpenSSL.crypto.X509()
+    cert = crypto.X509()
     cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16))
     cert.set_version(2)
 
     extensions = [
-        OpenSSL.crypto.X509Extension(
+        crypto.X509Extension(
             b"basicConstraints", True, b"CA:TRUE, pathlen:0"),
     ]
 
@@ -257,7 +273,7 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_issuer(cert.get_subject())
 
     if force_san or len(domains) > 1:
-        extensions.append(OpenSSL.crypto.X509Extension(
+        extensions.append(crypto.X509Extension(
             b"subjectAltName",
             critical=False,
             value=b", ".join(b"DNS:" + d.encode() for d in domains)
@@ -271,3 +287,26 @@ def gen_ss_cert(key, domains, not_before=None,
     cert.set_pubkey(key)
     cert.sign(key, "sha256")
     return cert
+
+def dump_pyopenssl_chain(chain, filetype=crypto.FILETYPE_PEM):
+    """Dump certificate chain into a bundle.
+
+    :param list chain: List of `OpenSSL.crypto.X509` (or wrapped in
+        :class:`josepy.util.ComparableX509`).
+
+    :returns: certificate chain bundle
+    :rtype: bytes
+
+    """
+    # XXX: returns empty string when no chain is available, which
+    # shuts up RenewableCert, but might not be the best solution...
+
+    def _dump_cert(cert):
+        if isinstance(cert, jose.ComparableX509):
+            # pylint: disable=protected-access
+            cert = cert.wrapped
+        return crypto.dump_certificate(filetype, cert)
+
+    # assumes that OpenSSL.crypto.dump_certificate includes ending
+    # newline character
+    return b"".join(_dump_cert(cert) for cert in chain)

acme/acme/crypto_util_test.py

@@ -13,6 +13,7 @@ import OpenSSL
 
 from acme import errors
 from acme import test_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 class SSLSocketAndProbeSNITest(unittest.TestCase):
@@ -41,28 +42,62 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.server_thread = threading.Thread(
             # pylint: disable=no-member
             target=self.server.handle_request)
-        self.server_thread.start()
-        time.sleep(1)  # TODO: avoid race conditions in other way
 
     def tearDown(self):
-        self.server_thread.join()
+        if self.server_thread.is_alive():
+            # The thread may have already terminated.
+            self.server_thread.join()  # pragma: no cover
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni
         return jose.ComparableX509(probe_sni(
             name, host='127.0.0.1', port=self.port))
 
+    def _start_server(self):
+        self.server_thread.start()
+        time.sleep(1)  # TODO: avoid race conditions in other way
+
     def test_probe_ok(self):
+        self._start_server()
         self.assertEqual(self.cert, self._probe(b'foo'))
 
     def test_probe_not_recognized_name(self):
+        self._start_server()
         self.assertRaises(errors.Error, self._probe, b'bar')
 
-    # TODO: py33/py34 tox hangs forever on do_handshake in second probe
-    #def probe_connection_error(self):
-    #    self._probe(b'foo')
-    #    #time.sleep(1)  # TODO: avoid race conditions in other way
-    #    self.assertRaises(errors.Error, self._probe, b'bar')
+    def test_probe_connection_error(self):
+        # pylint has a hard time with six
+        self.server.server_close()  # pylint: disable=no-member
+        original_timeout = socket.getdefaulttimeout()
+        try:
+            socket.setdefaulttimeout(1)
+            self.assertRaises(errors.Error, self._probe, b'bar')
+        finally:
+            socket.setdefaulttimeout(original_timeout)
+
+
+class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase):
+    """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names."""
+
+    @classmethod
+    def _call(cls, loader, name):
+        # pylint: disable=protected-access
+        from acme.crypto_util import _pyopenssl_cert_or_req_all_names
+        return _pyopenssl_cert_or_req_all_names(loader(name))
+
+    def _call_cert(self, name):
+        return self._call(test_util.load_cert, name)
+
+    def test_cert_one_san_no_common(self):
+        self.assertEqual(self._call_cert('cert-nocn.der'),
+                         ['no-common-name.badssl.com'])
+
+    def test_cert_no_sans_yes_common(self):
+        self.assertEqual(self._call_cert('cert.pem'), ['example.com'])
+
+    def test_cert_two_sans_yes_common(self):
+        self.assertEqual(self._call_cert('cert-san.pem'),
+                         ['example.com', 'www.example.com'])
 
 
 class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
@@ -141,7 +176,7 @@ class RandomSnTest(unittest.TestCase):
 
     def setUp(self):
         self.cert_count = 5
-        self.serial_num = []
+        self.serial_num = [] # type: List[int]
         self.key = OpenSSL.crypto.PKey()
         self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
 
@@ -170,9 +205,9 @@ class MakeCSRTest(unittest.TestCase):
         self.assertTrue(b'--END CERTIFICATE REQUEST--' in csr_pem)
         csr = OpenSSL.crypto.load_certificate_request(
             OpenSSL.crypto.FILETYPE_PEM, csr_pem)
-        # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr
-        # objects don't have a get_extensions() method, so we skip this test if
-        # the method isn't available.
+        # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't
+        # have a get_extensions() method, so we skip this test if the method
+        # isn't available.
         if hasattr(csr, 'get_extensions'):
             self.assertEquals(len(csr.get_extensions()), 1)
             self.assertEquals(csr.get_extensions()[0].get_data(),
@@ -188,9 +223,9 @@ class MakeCSRTest(unittest.TestCase):
         csr = OpenSSL.crypto.load_certificate_request(
             OpenSSL.crypto.FILETYPE_PEM, csr_pem)
 
-        # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr
-        # objects don't have a get_extensions() method, so we skip this test if
-        # the method isn't available.
+        # In pyopenssl 0.13 (used with TOXENV=py27-oldest), csr objects don't
+        # have a get_extensions() method, so we skip this test if the method
+        # isn't available.
         if hasattr(csr, 'get_extensions'):
             self.assertEquals(len(csr.get_extensions()), 2)
             # NOTE: Ideally we would filter by the TLS Feature OID, but
@@ -201,5 +236,33 @@ class MakeCSRTest(unittest.TestCase):
             self.assertEqual(len(must_staple_exts), 1,
                 "Expected exactly one Must Staple extension")
 
+
+class DumpPyopensslChainTest(unittest.TestCase):
+    """Test for dump_pyopenssl_chain."""
+
+    @classmethod
+    def _call(cls, loaded):
+        # pylint: disable=protected-access
+        from acme.crypto_util import dump_pyopenssl_chain
+        return dump_pyopenssl_chain(loaded)
+
+    def test_dump_pyopenssl_chain(self):
+        names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem']
+        loaded = [test_util.load_cert(name) for name in names]
+        length = sum(
+            len(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert))
+            for cert in loaded)
+        self.assertEqual(len(self._call(loaded)), length)
+
+    def test_dump_pyopenssl_chain_wrapped(self):
+        names = ['cert.pem', 'cert-san.pem', 'cert-idnsans.pem']
+        loaded = [test_util.load_cert(name) for name in names]
+        wrap_func = jose.ComparableX509
+        wrapped = [wrap_func(cert) for cert in loaded]
+        dump_func = OpenSSL.crypto.dump_certificate
+        length = sum(len(dump_func(OpenSSL.crypto.FILETYPE_PEM, cert)) for cert in loaded)
+        self.assertEqual(len(self._call(wrapped)), length)
+
+
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/errors.py

@@ -83,13 +83,40 @@ class PollError(ClientError):
         return '{0}(exhausted={1!r}, updated={2!r})'.format(
             self.__class__.__name__, self.exhausted, self.updated)
 
+class ValidationError(Error):
+    """Error for authorization failures. Contains a list of authorization
+    resources, each of which is invalid and should have an error field.
+    """
+    def __init__(self, failed_authzrs):
+        self.failed_authzrs = failed_authzrs
+        super(ValidationError, self).__init__()
+
+class TimeoutError(Error):
+    """Error for when polling an authorization or an order times out."""
+
+class IssuanceError(Error):
+    """Error sent by the server after requesting issuance of a certificate."""
+
+    def __init__(self, error):
+        """Initialize.
+
+        :param messages.Error error: The error provided by the server.
+        """
+        self.error = error
+        super(IssuanceError, self).__init__()
+
 class ConflictError(ClientError):
     """Error for when the server returns a 409 (Conflict) HTTP status.
 
     In the version of ACME implemented by Boulder, this is used to find an
     account if you only have the private key, but don't know the account URL.
+
+    Also used in V2 of the ACME client for the same purpose.
     """
     def __init__(self, location):
         self.location = location
         super(ConflictError, self).__init__()
 
+
+class WildcardUnsupportedError(Error):
+    """Error for when a wildcard is requested but is unsupported by ACME CA."""

acme/acme/magic_typing.py

@@ -0,0 +1,16 @@
+"""Shim class to not have to depend on typing module in prod."""
+import sys
+
+class TypingClass(object):
+    """Ignore import errors by getting anything"""
+    def __getattr__(self, name):
+        return None
+
+try:
+    # mypy doesn't respect modifying sys.modules
+    from typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+    # pylint: disable=unused-import
+    from typing import Collection, IO  # type: ignore
+    # pylint: enable=unused-import
+except ImportError:
+    sys.modules[__name__] = TypingClass()

acme/acme/magic_typing_test.py

@@ -0,0 +1,41 @@
+"""Tests for acme.magic_typing."""
+import sys
+import unittest
+
+import mock
+
+
+class MagicTypingTest(unittest.TestCase):
+    """Tests for acme.magic_typing."""
+    def test_import_success(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        typing_class_mock = mock.MagicMock()
+        text_mock = mock.MagicMock()
+        typing_class_mock.Text = text_mock
+        sys.modules['typing'] = typing_class_mock
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertEqual(Text, text_mock)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+    def test_import_failure(self):
+        try:
+            import typing as temp_typing
+        except ImportError: # pragma: no cover
+            temp_typing = None # pragma: no cover
+        sys.modules['typing'] = None
+        if 'acme.magic_typing' in sys.modules:
+            del sys.modules['acme.magic_typing'] # pragma: no cover
+        from acme.magic_typing import Text # pylint: disable=no-name-in-module
+        self.assertTrue(Text is None)
+        del sys.modules['acme.magic_typing']
+        sys.modules['typing'] = temp_typing
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/acme/messages.py

@@ -145,6 +145,7 @@ STATUS_PROCESSING = Status('processing')
 STATUS_VALID = Status('valid')
 STATUS_INVALID = Status('invalid')
 STATUS_REVOKED = Status('revoked')
+STATUS_READY = Status('ready')
 
 
 class IdentifierType(_Constant):
@@ -171,9 +172,30 @@ class Directory(jose.JSONDeSerializable):
 
     class Meta(jose.JSONObjectWithFields):
         """Directory Meta."""
-        terms_of_service = jose.Field('terms-of-service', omitempty=True)
+        _terms_of_service = jose.Field('terms-of-service', omitempty=True)
+        _terms_of_service_v2 = jose.Field('termsOfService', omitempty=True)
         website = jose.Field('website', omitempty=True)
-        caa_identities = jose.Field('caa-identities', omitempty=True)
+        caa_identities = jose.Field('caaIdentities', omitempty=True)
+
+        def __init__(self, **kwargs):
+            kwargs = dict((self._internal_name(k), v) for k, v in kwargs.items())
+            # pylint: disable=star-args
+            super(Directory.Meta, self).__init__(**kwargs)
+
+        @property
+        def terms_of_service(self):
+            """URL for the CA TOS"""
+            return self._terms_of_service or self._terms_of_service_v2
+
+        def __iter__(self):
+            # When iterating over fields, use the external name 'terms_of_service' instead of
+            # the internal '_terms_of_service'.
+            for name in super(Directory.Meta, self).__iter__():
+                yield name[1:] if name == '_terms_of_service' else name
+
+        def _internal_name(self, name):
+            return '_' + name if name == 'terms_of_service' else name
+
 
     @classmethod
     def _canon_key(cls, key):
@@ -251,6 +273,8 @@ class Registration(ResourceBody):
     contact = jose.Field('contact', omitempty=True, default=())
     agreement = jose.Field('agreement', omitempty=True)
     status = jose.Field('status', omitempty=True)
+    terms_of_service_agreed = jose.Field('termsOfServiceAgreed', omitempty=True)
+    only_return_existing = jose.Field('onlyReturnExisting', omitempty=True)
 
     phone_prefix = 'tel:'
     email_prefix = 'mailto:'
@@ -262,7 +286,7 @@ class Registration(ResourceBody):
         if phone is not None:
             details.append(cls.phone_prefix + phone)
         if email is not None:
-            details.append(cls.email_prefix + email)
+            details.extend([cls.email_prefix + mail for mail in email.split(',')])
         kwargs['contact'] = tuple(details)
         return cls(**kwargs)
 
@@ -413,6 +437,7 @@ class Authorization(ResourceBody):
     # be absent'... then acme-spec gives example with 'expires'
     # present... That's confusing!
     expires = fields.RFC3339Field('expires', omitempty=True)
+    wildcard = jose.Field('wildcard', omitempty=True)
 
     @challenges.decoder
     def challenges(value):  # pylint: disable=missing-docstring,no-self-argument
@@ -482,3 +507,49 @@ class Revocation(jose.JSONObjectWithFields):
     certificate = jose.Field(
         'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert)
     reason = jose.Field('reason')
+
+
+class Order(ResourceBody):
+    """Order Resource Body.
+
+    :ivar list of .Identifier: List of identifiers for the certificate.
+    :ivar acme.messages.Status status:
+    :ivar list of str authorizations: URLs of authorizations.
+    :ivar str certificate: URL to download certificate as a fullchain PEM.
+    :ivar str finalize: URL to POST to to request issuance once all
+        authorizations have "valid" status.
+    :ivar datetime.datetime expires: When the order expires.
+    :ivar .Error error: Any error that occurred during finalization, if applicable.
+    """
+    identifiers = jose.Field('identifiers', omitempty=True)
+    status = jose.Field('status', decoder=Status.from_json,
+                        omitempty=True)
+    authorizations = jose.Field('authorizations', omitempty=True)
+    certificate = jose.Field('certificate', omitempty=True)
+    finalize = jose.Field('finalize', omitempty=True)
+    expires = fields.RFC3339Field('expires', omitempty=True)
+    error = jose.Field('error', omitempty=True, decoder=Error.from_json)
+
+    @identifiers.decoder
+    def identifiers(value):  # pylint: disable=missing-docstring,no-self-argument
+        return tuple(Identifier.from_json(identifier) for identifier in value)
+
+class OrderResource(ResourceWithURI):
+    """Order Resource.
+
+    :ivar acme.messages.Order body:
+    :ivar str csr_pem: The CSR this Order will be finalized with.
+    :ivar list of acme.messages.AuthorizationResource authorizations:
+        Fully-fetched AuthorizationResource objects.
+    :ivar str fullchain_pem: The fetched contents of the certificate URL
+        produced once the order was finalized, if it's present.
+    """
+    body = jose.Field('body', decoder=Order.from_json)
+    csr_pem = jose.Field('csr_pem', omitempty=True)
+    authorizations = jose.Field('authorizations')
+    fullchain_pem = jose.Field('fullchain_pem', omitempty=True)
+
+@Directory.register
+class NewOrder(Order):
+    """New order."""
+    resource_type = 'new-order'

acme/acme/messages_test.py

@@ -6,6 +6,7 @@ import mock
 
 from acme import challenges
 from acme import test_util
+from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module
 
 
 CERT = test_util.load_comparable_cert('cert.der')
@@ -85,7 +86,7 @@ class ConstantTest(unittest.TestCase):
         from acme.messages import _Constant
 
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
-            POSSIBLE_NAMES = {}
+            POSSIBLE_NAMES = {} # type: Dict
 
         self.MockConstant = MockConstant  # pylint: disable=invalid-name
         self.const_a = MockConstant('a')
@@ -157,7 +158,7 @@ class DirectoryTest(unittest.TestCase):
             'meta': {
                 'terms-of-service': 'https://example.com/acme/terms',
                 'website': 'https://www.example.com/',
-                'caa-identities': ['example.com'],
+                'caaIdentities': ['example.com'],
             },
         })
 
@@ -165,6 +166,13 @@ class DirectoryTest(unittest.TestCase):
         from acme.messages import Directory
         Directory.from_json({'foo': 'bar'})
 
+    def test_iter_meta(self):
+        result = False
+        for k in self.dir.meta:
+            if k == 'terms_of_service':
+                result = self.dir.meta[k] == 'https://example.com/acme/terms'
+        self.assertTrue(result)
+
 
 class RegistrationTest(unittest.TestCase):
     """Tests for acme.messages.Registration."""
@@ -401,5 +409,34 @@ class RevocationTest(unittest.TestCase):
         hash(Revocation.from_json(self.rev.to_json()))
 
 
+class OrderResourceTest(unittest.TestCase):
+    """Tests for acme.messages.OrderResource."""
+
+    def setUp(self):
+        from acme.messages import OrderResource
+        self.regr = OrderResource(
+            body=mock.sentinel.body, uri=mock.sentinel.uri)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.regr.to_json(), {
+            'body': mock.sentinel.body,
+            'uri': mock.sentinel.uri,
+            'authorizations': None,
+        })
+
+class NewOrderTest(unittest.TestCase):
+    """Tests for acme.messages.NewOrder."""
+
+    def setUp(self):
+        from acme.messages import NewOrder
+        self.reg = NewOrder(
+            identifiers=mock.sentinel.identifiers)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.reg.to_json(), {
+            'identifiers': mock.sentinel.identifiers,
+        })
+
+
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/standalone.py

@@ -16,6 +16,7 @@ import OpenSSL
 
 from acme import challenges
 from acme import crypto_util
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
 
 
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class BaseDualNetworkedServers(object):
 
     def __init__(self, ServerClass, server_address, *remaining_args, **kwargs):
         port = server_address[1]
-        self.threads = []
-        self.servers = []
+        self.threads = [] # type: List[threading.Thread]
+        self.servers = [] # type: List[ACMEServerMixin]
 
         # Must try True first.
         # Ubuntu, for example, will fail to bind to IPv4 if we've already bound
@@ -82,9 +83,22 @@ class BaseDualNetworkedServers(object):
                 new_address = (server_address[0],) + (port,) + server_address[2:]
                 new_args = (new_address,) + remaining_args
                 server = ServerClass(*new_args, **kwargs) # pylint: disable=star-args
-            except socket.error:
-                logger.debug("Failed to bind to %s:%s using %s", new_address[0],
+                logger.debug(
+                    "Successfully bound to %s:%s using %s", new_address[0],
                     new_address[1], "IPv6" if ip_version else "IPv4")
+            except socket.error:
+                if self.servers:
+                    # Already bound using IPv6.
+                    logger.debug(
+                        "Certbot wasn't able to bind to %s:%s using %s, this " +
+                        "is often expected due to the dual stack nature of " +
+                        "IPv6 socket implementations.",
+                        new_address[0], new_address[1],
+                        "IPv6" if ip_version else "IPv4")
+                else:
+                    logger.debug(
+                        "Failed to bind to %s:%s using %s", new_address[0],
+                        new_address[1], "IPv6" if ip_version else "IPv4")
             else:
                 self.servers.append(server)
                 # If two servers are set up and port 0 was passed in, ensure we always
@@ -189,7 +203,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def __init__(self, *args, **kwargs):
         self.simple_http_resources = kwargs.pop("simple_http_resources", set())
-        socketserver.BaseRequestHandler.__init__(self, *args, **kwargs)
+        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def log_message(self, format, *args):  # pylint: disable=redefined-builtin
         """Log arbitrary message."""
@@ -262,7 +276,7 @@ def simple_tls_sni_01_server(cli_args, forever=True):
 
     certs = {}
 
-    _, hosts, _ = next(os.walk('.'))
+    _, hosts, _ = next(os.walk('.')) # type: ignore # https://github.com/python/mypy/issues/465
     for host in hosts:
         with open(os.path.join(host, "cert.pem")) as cert_file:
             cert_contents = cert_file.read()

acme/acme/standalone_test.py

@@ -4,10 +4,10 @@ import shutil
 import socket
 import threading
 import tempfile
-import time
 import unittest
 
 from six.moves import http_client  # pylint: disable=import-error
+from six.moves import queue  # pylint: disable=import-error
 from six.moves import socketserver  # type: ignore  # pylint: disable=import-error
 
 import josepy as jose
@@ -16,8 +16,8 @@ import requests
 
 from acme import challenges
 from acme import crypto_util
-from acme import errors
 from acme import test_util
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 
 
 class TLSServerTest(unittest.TestCase):
@@ -48,7 +48,7 @@ class TLSSNI01ServerTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01Server
-        self.server = TLSSNI01Server(("", 0), certs=self.certs)
+        self.server = TLSSNI01Server(('localhost', 0), certs=self.certs)
         # pylint: disable=no-member
         self.thread = threading.Thread(target=self.server.serve_forever)
         self.thread.start()
@@ -72,7 +72,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01Server
         self.server = HTTP01Server(('', 0), resources=self.resources)
@@ -133,8 +133,11 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
                 self.address_family = socket.AF_INET
             socketserver.TCPServer.__init__(self, *args, **kwargs)
             if ipv6:
+                # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing.
+                # We use the corresponding value (41) instead.
+                level = getattr(socket, "IPPROTO_IPV6", 41)
                 # pylint: disable=no-member
-                self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+                self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1)
                 try:
                     self.server_bind()
                     self.server_activate()
@@ -147,15 +150,15 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
         mock_bind.side_effect = socket.error
         from acme.standalone import BaseDualNetworkedServers
         self.assertRaises(socket.error, BaseDualNetworkedServers,
-            BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
-            socketserver.BaseRequestHandler)
+                          BaseDualNetworkedServersTest.SingleProtocolServer,
+                          ('', 0),
+                          socketserver.BaseRequestHandler)
 
     def test_ports_equal(self):
         from acme.standalone import BaseDualNetworkedServers
         servers = BaseDualNetworkedServers(
             BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
+            ('', 0),
             socketserver.BaseRequestHandler)
         socknames = servers.getsocknames()
         prev_port = None
@@ -177,7 +180,7 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01DualNetworkedServers
-        self.servers = TLSSNI01DualNetworkedServers(("", 0), certs=self.certs)
+        self.servers = TLSSNI01DualNetworkedServers(('localhost', 0), certs=self.certs)
         self.servers.serve_forever()
 
     def tearDown(self):
@@ -201,7 +204,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
-        self.resources = set()
+        self.resources = set() # type: Set
 
         from acme.standalone import HTTP01DualNetworkedServers
         self.servers = HTTP01DualNetworkedServers(('', 0), resources=self.resources)
@@ -245,6 +248,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
         self.assertFalse(self._test_http01(add=False))
 
 
+@test_util.broken_on_windows
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 
@@ -260,10 +264,9 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
                     os.path.join(localhost_dir, 'key.pem'))
 
         from acme.standalone import simple_tls_sni_01_server
-        self.port = 1234
         self.thread = threading.Thread(
             target=simple_tls_sni_01_server, kwargs={
-                'cli_args': ('xxx', '--port', str(self.port)),
+                'cli_args': ('filename',),
                 'forever': False,
             },
         )
@@ -275,25 +278,20 @@ class TestSimpleTLSSNI01Server(unittest.TestCase):
         self.thread.join()
         shutil.rmtree(self.test_cwd)
 
-    def test_it(self):
-        max_attempts = 5
-        for attempt in range(max_attempts):
-            try:
-                cert = crypto_util.probe_sni(
-                    b'localhost', b'0.0.0.0', self.port)
-            except errors.Error:
-                self.assertTrue(attempt + 1 < max_attempts, "Timeout!")
-                time.sleep(1)  # wait until thread starts
-            else:
-                self.assertEqual(jose.ComparableX509(cert),
-                                 test_util.load_comparable_cert(
-                                     'rsa2048_cert.pem'))
-                break
+    @mock.patch('acme.standalone.logger')
+    def test_it(self, mock_logger):
+        # Use a Queue because mock objects aren't thread safe.
+        q = queue.Queue()  # type: queue.Queue[int]
+        # Add port number to the queue.
+        mock_logger.info.side_effect = lambda *args: q.put(args[-1])
+        self.thread.start()
 
-            if attempt == 0:
-                # the first attempt is always meant to fail, so we can test
-                # the socket failure code-path for probe_sni, as well
-                self.thread.start()
+        # After the timeout, an exception is raised if the queue is empty.
+        port = q.get(timeout=5)
+        cert = crypto_util.probe_sni(b'localhost', b'0.0.0.0', port)
+        self.assertEqual(jose.ComparableX509(cert),
+                         test_util.load_comparable_cert(
+                             'rsa2048_cert.pem'))
 
 
 if __name__ == "__main__":

acme/acme/test_util.py

@@ -4,13 +4,14 @@
 
 """
 import os
+import sys
 import pkg_resources
 import unittest
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import josepy as jose
-import OpenSSL
+from OpenSSL import crypto
 
 
 def vector_path(*names):
@@ -39,8 +40,8 @@ def _guess_loader(filename, loader_pem, loader_der):
 def load_cert(*names):
     """Load certificate."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate(loader, load_vector(*names))
 
 
 def load_comparable_cert(*names):
@@ -51,8 +52,8 @@ def load_comparable_cert(*names):
 def load_csr(*names):
     """Load certificate request."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_certificate_request(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_certificate_request(loader, load_vector(*names))
 
 
 def load_comparable_csr(*names):
@@ -71,8 +72,8 @@ def load_rsa_private_key(*names):
 def load_pyopenssl_private_key(*names):
     """Load pyOpenSSL private key."""
     loader = _guess_loader(
-        names[-1], OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.FILETYPE_ASN1)
-    return OpenSSL.crypto.load_privatekey(loader, load_vector(*names))
+        names[-1], crypto.FILETYPE_PEM, crypto.FILETYPE_ASN1)
+    return crypto.load_privatekey(loader, load_vector(*names))
 
 
 def skip_unless(condition, reason):  # pragma: no cover
@@ -94,3 +95,11 @@ def skip_unless(condition, reason):  # pragma: no cover
         return lambda cls: cls
     else:
         return lambda cls: None
+
+def broken_on_windows(function):
+    """Decorator to skip temporarily a broken test on Windows."""
+    reason = 'Test is broken and ignored on windows but should be fixed.'
+    return unittest.skipIf(
+        sys.platform == 'win32'
+        and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true',
+        reason)(function)

acme/docs/api/other.rst

@@ -1,5 +0,0 @@
-Other ACME objects
-------------------
-
-.. automodule:: acme.other
-   :members:

acme/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = .* build dist CVS _darcs {arch} *.egg

acme/setup.py

@@ -1,10 +1,9 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
-
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
@@ -19,19 +18,11 @@ install_requires = [
     'pyrfc3339',
     'pytz',
     'requests[security]>=2.4.1',  # security extras added in 2.4.1
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'requests-toolbelt>=0.3.0',
+    'setuptools',
     'six>=1.9.0',  # needed for python_2_unicode_compatible
 ]
 
-# env markers cause problems with older pip and setuptools
-if sys.version_info < (2, 7):
-    install_requires.extend([
-        'argparse',
-        'ordereddict',
-    ])
-
 dev_extras = [
     'pytest',
     'pytest-xdist',
@@ -43,6 +34,19 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
 
 setup(
     name='acme',
@@ -52,19 +56,19 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],
@@ -76,5 +80,7 @@ setup(
         'dev': dev_extras,
         'docs': docs_extras,
     },
+    tests_require=["pytest"],
     test_suite='acme',
+    cmdclass={"test": PyTest},
 )

appveyor.yml

@@ -0,0 +1,29 @@
+image:
+  # => Windows Server 2012 R2
+  - Visual Studio 2015
+  # => Windows Server 2016
+  - Visual Studio 2017
+
+branches:
+  only:
+    - master
+    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
+    - /^test-.*$/
+
+install:
+  # Use Python 3.7 by default
+  - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
+  # Check env
+  - "python --version"
+  # Upgrade pip to avoid warnings
+  - "python -m pip install --upgrade pip"
+  # Ready to install tox and coverage
+  - "pip install tox codecov"
+
+build: off
+
+test_script:
+  - tox -c tox-win.ini -e py34,py35,py36,py37-cover
+
+on_success:
+  - codecov

certbot-apache/certbot_apache/apache_util.py

@@ -1,4 +1,5 @@
 """ Utility functions for certbot-apache plugin """
+import binascii
 import os
 
 from certbot import util
@@ -98,3 +99,8 @@ def parse_define_file(filepath, varname):
             var_parts = v[2:].partition("=")
             return_vars[var_parts[0]] = var_parts[2]
     return return_vars
+
+
+def unique_id():
+    """ Returns an unique id to be used as a VirtualHost identifier"""
+    return binascii.hexlify(os.urandom(16)).decode("utf-8")

certbot-apache/certbot_apache/augeas_lens/httpd.aug

@@ -44,67 +44,134 @@ autoload xfm
  *****************************************************************)
 let dels (s:string)     = del s s
 
+(* The continuation sequence that indicates that we should consider the
+ * next line part of the current line *)
+let cont = /\\\\\r?\n/
+
+(* Whitespace within a line: space, tab, and the continuation sequence *)
+let ws = /[ \t]/ | cont
+
+(* Any possible character - '.' does not match \n *)
+let any = /(.|\n)/
+
+(* Any character preceded by a backslash *)
+let esc_any = /\\\\(.|\n)/
+
+(* Newline sequence - both for Unix and DOS newlines *)
+let nl = /\r?\n/
+
+(* Whitespace at the end of a line *)
+let eol = del (ws* . nl) "\n"
+
 (* deal with continuation lines *)
-let sep_spc             = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " "
-let sep_osp             = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ ""
-let sep_eq              = del /[ \t]*=[ \t]*/ "="
+let sep_spc             = del ws+ " "
+let sep_osp             = del ws* ""
+let sep_eq              = del (ws* . "=" . ws*) "="
 
 let nmtoken             = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/
 let word                = /[a-z][a-z0-9._-]*/i
 
-let eol                 = Util.doseol
-let empty               = Util.empty_dos
+(* A complete line that is either just whitespace or a comment that only
+ * contains whitespace *)
+let empty = [ del (ws* . /#?/ . ws* . nl) "\n" ]
+
 let indent              = Util.indent
 
-let comment_val_re      = /([^ \t\r\n](.|\\\\\r?\n)*[^ \\\t\r\n]|[^ \t\r\n])/
-let comment             = [ label "#comment" . del /[ \t]*#[ \t]*/ "# "
-                          . store comment_val_re . eol ]
+(* A comment that is not just whitespace. We define it in terms of the
+ * things that are not allowed as part of such a comment:
+ *   1) Starts with whitespace
+ *   2) Ends with whitespace, a backslash or \r
+ *   3) Unescaped newlines
+ *)
+let comment =
+  let comment_start = del (ws* . "#" . ws* ) "# " in
+  let unesc_eol = /[^\]?/ . nl in
+  let w = /[^\t\n\r \\]/ in
+  let r = /[\r\\]/ in
+  let s = /[\t\r ]/ in
+  (*
+   * we'd like to write
+   * let b = /\\\\/ in
+   * let t = /[\t\n\r ]/ in
+   * let x = b . (t? . (s|w)* ) in
+   * but the definition of b depends on commit 244c0edd in 1.9.0 and
+   * would make the lens unusable with versions before 1.9.0. So we write
+   * x out which works in older versions, too
+   *)
+  let x = /\\\\[\t\n\r ]?[^\n\\]*/ in
+  let line = ((r . s* . w|w|r) . (s|w)* . x*|(r.s* )?).w.(s*.w)* in
+  [ label "#comment" . comment_start . store line . eol ]
 
 (* borrowed from shellvars.aug *)
-let char_arg_dir  = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ /
 let char_arg_sec  = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ /
 let char_arg_wl   = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/
 
-let cdot = /\\\\./
-let cl = /\\\\\n/
 let dquot =
      let no_dquot = /[^"\\\r\n]/
-  in /"/ . (no_dquot|cdot|cl)* . /"/
+  in /"/ . (no_dquot|esc_any)* . /"/
 let dquot_msg =
      let no_dquot = /([^ \t"\\\r\n]|[^"\\\r\n]+[^ \t"\\\r\n])/
-  in /"/ . (no_dquot|cdot|cl)*
+  in /"/ . (no_dquot|esc_any)* . no_dquot
+
 let squot =
      let no_squot = /[^'\\\r\n]/
-  in /'/ . (no_squot|cdot|cl)* . /'/
+  in /'/ . (no_squot|esc_any)* . /'/
 let comp = /[<>=]?=/
 
 (******************************************************************
  *                            Attributes
  *****************************************************************)
 
-let arg_dir = [ label "arg" . store (char_arg_dir+|dquot|squot) ]
+(* The arguments for a directive come in two flavors: quoted with single or
+ * double quotes, or bare. Bare arguments may not start with a single or
+ * double quote; since we also treat "word lists" special, i.e. lists
+ * enclosed in curly braces, bare arguments may not start with those,
+ * either.
+ *
+ * Bare arguments may not contain unescaped spaces, but we allow escaping
+ * with '\\'. Quoted arguments can contain anything, though the quote must
+ * be escaped with '\\'.
+ *)
+let bare = /([^{"' \t\n\r]|\\\\.)([^ \t\n\r]|\\\\.)*[^ \t\n\r\\]|[^{"' \t\n\r\\]/
+
+let arg_quoted = [ label "arg" . store (dquot|squot) ]
+let arg_bare = [ label "arg" . store bare ]
+
 (* message argument starts with " but ends at EOL *)
 let arg_dir_msg = [ label "arg" . store dquot_msg ]
-let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ]
 let arg_wl  = [ label "arg" . store (char_arg_wl+|dquot|squot) ]
 
 (* comma-separated wordlist as permitted in the SSLRequire directive *)
 let arg_wordlist =
-     let wl_start = Util.del_str "{" in
-     let wl_end   = Util.del_str "}" in
+     let wl_start = dels "{" in
+     let wl_end   = dels "}" in
      let wl_sep   = del /[ \t]*,[ \t]*/ ", "
   in [ label "wordlist" . wl_start . arg_wl . (wl_sep . arg_wl)* . wl_end ]
 
 let argv (l:lens) = l . (sep_spc . l)*
 
+(* the arguments of a directive. We use this once we have parsed the name
+ * of the directive, and the space right after it. When dir_args is used,
+ * we also know that we have at least one argument. We need to be careful
+ * with the spacing between arguments: quoted arguments and word lists do
+ * not need to have space between them, but bare arguments do.
+ *
+ * Apache apparently is also happy if the last argument starts with a double
+ * quote, but has no corresponding closing duoble quote, which is what
+ * arg_dir_msg handles
+ *)
+let dir_args =
+  let arg_nospc = arg_quoted|arg_wordlist in
+  (arg_bare . sep_spc | arg_nospc . sep_osp)* . (arg_bare|arg_nospc|arg_dir_msg)
+
 let directive =
-    (* arg_dir_msg may be the last or only argument *)
-     let dir_args = (argv (arg_dir|arg_wordlist) . (sep_spc . arg_dir_msg)?) | arg_dir_msg
-  in [ indent . label "directive" . store word .  (sep_spc . dir_args)? . eol ]
+  [ indent . label "directive" . store word .  (sep_spc . dir_args)? . eol ]
+
+let arg_sec = [ label "arg" . store (char_arg_sec+|comp|dquot|squot) ]
 
 let section (body:lens) =
     (* opt_eol includes empty lines *)
-    let opt_eol = del /([ \t]*#?\r?\n)*/ "\n" in
+    let opt_eol = del /([ \t]*#?[ \t]*\r?\n)*/ "\n" in
     let inner = (sep_spc . argv arg_sec)? . sep_osp .
              dels ">" . opt_eol . ((body|comment) . (body|empty|comment)*)? .
              indent . dels "</" in
@@ -133,6 +200,7 @@ let filter = (incl "/etc/apache2/apache2.conf") .
              (incl "/etc/httpd/conf.d/*.conf") .
              (incl "/etc/httpd/httpd.conf") .
              (incl "/etc/httpd/conf/httpd.conf") .
+             (incl "/etc/httpd/conf.modules.d/*.conf") .
              Util.stdexcl
 
 let xfm = transform lns filter

certbot-apache/certbot_apache/centos-options-ssl-apache.conf

@@ -8,7 +8,7 @@ SSLEngine on
 
 # Intermediate configuration, tweak to your needs
 SSLProtocol             all -SSLv2 -SSLv3
-SSLCipherSuite          ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
 
 SSLOptions +StrictRequire

certbot-apache/certbot_apache/constants.py

@@ -16,6 +16,8 @@ ALL_SSL_OPTIONS_HASHES = [
     '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27',
     'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88',
     'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b',
+    '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791',
+    'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 
@@ -46,3 +48,16 @@ UIR_ARGS = ["always", "set", "Content-Security-Policy",
 
 HEADER_ARGS = {"Strict-Transport-Security": HSTS_ARGS,
                "Upgrade-Insecure-Requests": UIR_ARGS}
+
+AUTOHSTS_STEPS = [60, 300, 900, 3600, 21600, 43200, 86400]
+"""AutoHSTS increase steps: 1min, 5min, 15min, 1h, 6h, 12h, 24h"""
+
+AUTOHSTS_PERMANENT = 31536000
+"""Value for the last max-age of HSTS"""
+
+AUTOHSTS_FREQ = 172800
+"""Minimum time since last increase to perform a new one: 48h"""
+
+MANAGED_COMMENT = "DO NOT REMOVE - Managed by Certbot"
+MANAGED_COMMENT_ID = MANAGED_COMMENT+", VirtualHost id: {0}"
+"""Managed by Certbot comments and the VirtualHost identification template"""

certbot-apache/certbot_apache/display_ops.py

@@ -13,10 +13,44 @@ import certbot.display.util as display_util
 logger = logging.getLogger(__name__)
 
 
+def select_vhost_multiple(vhosts):
+    """Select multiple Vhosts to install the certificate for
+
+    :param vhosts: Available Apache VirtualHosts
+    :type vhosts: :class:`list` of type `~obj.Vhost`
+
+    :returns: List of VirtualHosts
+    :rtype: :class:`list`of type `~obj.Vhost`
+    """
+    if not vhosts:
+        return list()
+    tags_list = [vhost.display_repr()+"\n" for vhost in vhosts]
+    # Remove the extra newline from the last entry
+    if len(tags_list):
+        tags_list[-1] = tags_list[-1][:-1]
+    code, names = zope.component.getUtility(interfaces.IDisplay).checklist(
+        "Which VirtualHosts would you like to install the wildcard certificate for?",
+        tags=tags_list, force_interactive=True)
+    if code == display_util.OK:
+        return_vhosts = _reversemap_vhosts(names, vhosts)
+        return return_vhosts
+    return []
+
+def _reversemap_vhosts(names, vhosts):
+    """Helper function for select_vhost_multiple for mapping string
+    representations back to actual vhost objects"""
+    return_vhosts = list()
+
+    for selection in names:
+        for vhost in vhosts:
+            if vhost.display_repr().strip() == selection.strip():
+                return_vhosts.append(vhost)
+    return return_vhosts
+
 def select_vhost(domain, vhosts):
     """Select an appropriate Apache Vhost.
 
-    :param vhosts: Available Apache Virtual Hosts
+    :param vhosts: Available Apache VirtualHosts
     :type vhosts: :class:`list` of type `~obj.Vhost`
 
     :returns: VirtualHost or `None`
@@ -25,13 +59,11 @@ def select_vhost(domain, vhosts):
     """
     if not vhosts:
         return None
-    while True:
-        code, tag = _vhost_menu(domain, vhosts)
-        if code == display_util.OK:
-            return vhosts[tag]
-        else:
-            return None
-
+    code, tag = _vhost_menu(domain, vhosts)
+    if code == display_util.OK:
+        return vhosts[tag]
+    else:
+        return None
 
 def _vhost_menu(domain, vhosts):
     """Select an appropriate Apache Vhost.
@@ -81,8 +113,7 @@ def _vhost_menu(domain, vhosts):
         code, tag = zope.component.getUtility(interfaces.IDisplay).menu(
             "We were unable to find a vhost with a ServerName "
             "or Address of {0}.{1}Which virtual host would you "
-            "like to choose?\n(note: conf files with multiple "
-            "vhosts are not yet supported)".format(domain, os.linesep),
+            "like to choose?".format(domain, os.linesep),
             choices, force_interactive=True)
     except errors.MissingCommandlineFlag:
         msg = (

certbot-apache/certbot_apache/entrypoint.py

@@ -17,6 +17,7 @@ OVERRIDE_CLASSES = {
     "centos": override_centos.CentOSConfigurator,
     "centos linux": override_centos.CentOSConfigurator,
     "fedora": override_centos.CentOSConfigurator,
+    "ol": override_centos.CentOSConfigurator,
     "red hat enterprise linux server": override_centos.CentOSConfigurator,
     "rhel": override_centos.CentOSConfigurator,
     "amazon": override_centos.CentOSConfigurator,

certbot-apache/certbot_apache/http_01.py

@@ -0,0 +1,181 @@
+"""A class that performs HTTP-01 challenges for Apache"""
+import logging
+import os
+
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
+from certbot import errors
+from certbot.plugins import common
+from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
+from certbot_apache.parser import get_aug_path
+
+logger = logging.getLogger(__name__)
+
+class ApacheHttp01(common.TLSSNI01):
+    """Class that performs HTTP-01 challenges within the Apache configurator."""
+
+    CONFIG_TEMPLATE22_PRE = """\
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L]
+
+    """
+    CONFIG_TEMPLATE22_POST = """\
+        <Directory {0}>
+            Order Allow,Deny
+            Allow from all
+        </Directory>
+        <Location /.well-known/acme-challenge>
+            Order Allow,Deny
+            Allow from all
+        </Location>
+    """
+
+    CONFIG_TEMPLATE24_PRE = """\
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END]
+    """
+    CONFIG_TEMPLATE24_POST = """\
+        <Directory {0}>
+            Require all granted
+        </Directory>
+        <Location /.well-known/acme-challenge>
+            Require all granted
+        </Location>
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(ApacheHttp01, self).__init__(*args, **kwargs)
+        self.challenge_conf_pre = os.path.join(
+            self.configurator.conf("challenge-location"),
+            "le_http_01_challenge_pre.conf")
+        self.challenge_conf_post = os.path.join(
+            self.configurator.conf("challenge-location"),
+            "le_http_01_challenge_post.conf")
+        self.challenge_dir = os.path.join(
+            self.configurator.config.work_dir,
+            "http_challenges")
+        self.moded_vhosts = set()  # type: Set[VirtualHost]
+
+    def perform(self):
+        """Perform all HTTP-01 challenges."""
+        if not self.achalls:
+            return []
+        # Save any changes to the configuration as a precaution
+        # About to make temporary changes to the config
+        self.configurator.save("Changes before challenge setup", True)
+
+        self.configurator.ensure_listen(str(
+            self.configurator.config.http01_port))
+        self.prepare_http01_modules()
+
+        responses = self._set_up_challenges()
+
+        self._mod_config()
+        # Save reversible changes
+        self.configurator.save("HTTP Challenge", True)
+
+        return responses
+
+    def prepare_http01_modules(self):
+        """Make sure that we have the needed modules available for http01"""
+
+        if self.configurator.conf("handle-modules"):
+            needed_modules = ["rewrite"]
+            if self.configurator.version < (2, 4):
+                needed_modules.append("authz_host")
+            else:
+                needed_modules.append("authz_core")
+            for mod in needed_modules:
+                if mod + "_module" not in self.configurator.parser.modules:
+                    self.configurator.enable_mod(mod, temp=True)
+
+    def _mod_config(self):
+        for chall in self.achalls:
+            vh = self.configurator.find_best_http_vhost(
+                chall.domain, filter_defaults=False,
+                port=str(self.configurator.config.http01_port))
+            if vh:
+                self._set_up_include_directives(vh)
+            else:
+                for vh in self._relevant_vhosts():
+                    self._set_up_include_directives(vh)
+
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf_pre)
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf_post)
+
+        if self.configurator.version < (2, 4):
+            config_template_pre = self.CONFIG_TEMPLATE22_PRE
+            config_template_post = self.CONFIG_TEMPLATE22_POST
+        else:
+            config_template_pre = self.CONFIG_TEMPLATE24_PRE
+            config_template_post = self.CONFIG_TEMPLATE24_POST
+
+        config_text_pre = config_template_pre.format(self.challenge_dir)
+        config_text_post = config_template_post.format(self.challenge_dir)
+
+        logger.debug("writing a pre config file with text:\n %s", config_text_pre)
+        with open(self.challenge_conf_pre, "w") as new_conf:
+            new_conf.write(config_text_pre)
+        logger.debug("writing a post config file with text:\n %s", config_text_post)
+        with open(self.challenge_conf_post, "w") as new_conf:
+            new_conf.write(config_text_post)
+
+    def _relevant_vhosts(self):
+        http01_port = str(self.configurator.config.http01_port)
+        relevant_vhosts = []
+        for vhost in self.configurator.vhosts:
+            if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs):
+                if not vhost.ssl:
+                    relevant_vhosts.append(vhost)
+        if not relevant_vhosts:
+            raise errors.PluginError(
+                "Unable to find a virtual host listening on port {0} which is"
+                " currently needed for Certbot to prove to the CA that you"
+                " control your domain. Please add a virtual host for port"
+                " {0}.".format(http01_port))
+
+        return relevant_vhosts
+
+    def _set_up_challenges(self):
+        if not os.path.isdir(self.challenge_dir):
+            os.makedirs(self.challenge_dir)
+            os.chmod(self.challenge_dir, 0o755)
+
+        responses = []
+        for achall in self.achalls:
+            responses.append(self._set_up_challenge(achall))
+
+        return responses
+
+    def _set_up_challenge(self, achall):
+        response, validation = achall.response_and_validation()
+
+        name = os.path.join(self.challenge_dir, achall.chall.encode("token"))
+
+        self.configurator.reverter.register_file_creation(True, name)
+        with open(name, 'wb') as f:
+            f.write(validation.encode())
+        os.chmod(name, 0o644)
+
+        return response
+
+    def _set_up_include_directives(self, vhost):
+        """Includes override configuration to the beginning and to the end of
+        VirtualHost. Note that this include isn't added to Augeas search tree"""
+
+        if vhost not in self.moded_vhosts:
+            logger.debug(
+                "Adding a temporary challenge validation Include for name: %s " +
+                "in: %s", vhost.name, vhost.filep)
+            self.configurator.parser.add_dir_beginning(
+                vhost.path, "Include", self.challenge_conf_pre)
+            self.configurator.parser.add_dir(
+                vhost.path, "Include", self.challenge_conf_post)
+
+            if not vhost.enabled:
+                self.configurator.parser.add_dir(
+                    get_aug_path(self.configurator.parser.loc["default"]),
+                    "Include", vhost.filep)
+
+            self.moded_vhosts.add(vhost)

certbot-apache/certbot_apache/obj.py

@@ -1,6 +1,7 @@
 """Module contains classes used by the Apache Configurator."""
 import re
 
+from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 
 
@@ -140,7 +141,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
     def get_names(self):
         """Return a set of all names."""
-        all_names = set()
+        all_names = set()  # type: Set[str]
         all_names.update(self.aliases)
         # Strip out any scheme:// and <port> field from servername
         if self.name is not None:
@@ -167,6 +168,19 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
                 active="Yes" if self.enabled else "No",
                 modmacro="Yes" if self.modmacro else "No"))
 
+    def display_repr(self):
+        """Return a representation of VHost to be used in dialog"""
+        return (
+            "File: {filename}\n"
+            "Addresses: {addrs}\n"
+            "Names: {names}\n"
+            "HTTPS: {https}\n".format(
+                filename=self.filep,
+                addrs=", ".join(str(addr) for addr in self.addrs),
+                names=", ".join(self.get_names()),
+                https="Yes" if self.ssl else "No"))
+
+
     def __eq__(self, other):
         if isinstance(other, self.__class__):
             return (self.filep == other.filep and self.path == other.path and
@@ -238,7 +252,7 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
         # already_found acts to keep everything very conservative.
         # Don't allow multiple ip:ports in same set.
-        already_found = set()
+        already_found = set()  # type: Set[str]
 
         for addr in vhost.addrs:
             for local_addr in self.addrs:

certbot-apache/certbot_apache/options-ssl-apache.conf

@@ -8,7 +8,7 @@ SSLEngine on
 
 # Intermediate configuration, tweak to your needs
 SSLProtocol             all -SSLv2 -SSLv3
-SSLCipherSuite          ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
 SSLCompression          off
 

certbot-apache/certbot_apache/override_arch.py

@@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_centos.py

@@ -18,24 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf.d",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
+        restart_cmd_alt=['apachectl', 'restart'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "centos-options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in CentOS.
+        """
+        super(CentOSConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return CentOSParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -46,10 +55,10 @@ class CentOSParser(parser.ApacheParser):
         self.sysconfig_filep = "/etc/sysconfig/httpd"
         super(CentOSParser, self).__init__(*args, **kwargs)
 
-    def update_runtime_variables(self, *args, **kwargs):
+    def update_runtime_variables(self):
         """ Override for update_runtime_variables for custom parsing """
         # Opportunistic, works if SELinux not enforced
-        super(CentOSParser, self).update_runtime_variables(*args, **kwargs)
+        super(CentOSParser, self).update_runtime_variables()
         self.parse_sysconfig_var()
 
     def parse_sysconfig_var(self):

certbot-apache/certbot_apache/override_darwin.py

@@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/other",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/httpd', '-v'],
-        apache_cmd="/usr/sbin/httpd",
+        ctl="apachectl",
+        version_cmd=['apachectl', '-v'],
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_debian.py

@@ -23,14 +23,14 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/sites-available",
         vhost_files="*",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=True,
+        handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
@@ -134,11 +134,11 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         # Generate reversal command.
         # Try to be safe here... check that we can probably reverse before
         # applying enmod command
-        if not util.exe_exists(self.conf("dismod")):
+        if not util.exe_exists(self.option("dismod")):
             raise errors.MisconfigurationError(
                 "Unable to find a2dismod, please make sure a2enmod and "
                 "a2dismod are configured correctly for certbot.")
 
         self.reverter.register_undo_command(
-            temp, [self.conf("dismod"), mod_name])
-        util.run_script([self.conf("enmod"), mod_name])
+            temp, [self.option("dismod"), "-f", mod_name])
+        util.run_script([self.option("enmod"), mod_name])

certbot-apache/certbot_apache/override_gentoo.py

@@ -18,24 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/apache2', '-v'],
-        apache_cmd="apache2ctl",
+        ctl="apache2ctl",
+        version_cmd=['apache2ctl', '-v'],
         restart_cmd=['apache2ctl', 'graceful'],
+        restart_cmd_alt=['apache2ctl', 'restart'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in Gentoo.
+        """
+        super(GentooConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return GentooParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -49,6 +58,7 @@ class GentooParser(parser.ApacheParser):
     def update_runtime_variables(self):
         """ Override for update_runtime_variables for custom parsing """
         self.parse_sysconfig_var()
+        self.update_modules()
 
     def parse_sysconfig_var(self):
         """ Parses Apache CLI options from Gentoo configuration file """
@@ -56,3 +66,10 @@ class GentooParser(parser.ApacheParser):
                                                 "APACHE2_OPTS")
         for k in defines.keys():
             self.variables[k] = defines[k]
+
+    def update_modules(self):
+        """Get loaded modules from httpd process, and add them to DOM"""
+        mod_cmd = [self.configurator.option("ctl"), "modules"]
+        matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
+        for mod in matches:
+            self.add_mod(mod.strip())

certbot-apache/certbot_apache/override_suse.py

@@ -16,14 +16,14 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/parser.py

@@ -9,12 +9,14 @@ import sys
 
 import six
 
+from acme.magic_typing import Dict, List, Set  # pylint: disable=unused-import, no-name-in-module
 from certbot import errors
 
 logger = logging.getLogger(__name__)
 
 
 class ApacheParser(object):
+    # pylint: disable=too-many-public-methods
     """Class handles the fine details of parsing the Apache Configuration.
 
     .. todo:: Make parsing general... remove sites-available etc...
@@ -38,9 +40,9 @@ class ApacheParser(object):
         # issues with aug.load() after adding new files / defines to parse tree
         self.configurator = configurator
 
-        self.modules = set()
-        self.parser_paths = {}
-        self.variables = {}
+        self.modules = set()  # type: Set[str]
+        self.parser_paths = {}  # type: Dict[str, List[str]]
+        self.variables = {}  # type: Dict[str, str]
 
         self.aug = aug
         # Find configuration root and make sure augeas can parse it.
@@ -67,7 +69,7 @@ class ApacheParser(object):
         # Must also attempt to parse additional virtual host root
         if vhostroot:
             self.parse_file(os.path.abspath(vhostroot) + "/" +
-                            self.configurator.constant("vhost_files"))
+                            self.configurator.option("vhost_files"))
 
         # check to see if there were unparsed define statements
         if version < (2, 4):
@@ -119,7 +121,7 @@ class ApacheParser(object):
             the iteration issue.  Else... parse and enable mods at same time.
 
         """
-        mods = set()
+        mods = set()  # type: Set[str]
         matches = self.find_dir("LoadModule")
         iterator = iter(matches)
         # Make sure prev_size != cur_size for do: while: iteration
@@ -150,7 +152,7 @@ class ApacheParser(object):
         """Get Defines from httpd process"""
 
         variables = dict()
-        define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        define_cmd = [self.configurator.option("ctl"), "-t", "-D",
                       "DUMP_RUN_CFG"]
         matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)")
         try:
@@ -177,7 +179,7 @@ class ApacheParser(object):
         # configuration files
         _ = self.find_dir("Include")
 
-        inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        inc_cmd = [self.configurator.option("ctl"), "-t", "-D",
                    "DUMP_INCLUDES"]
         matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
         if matches:
@@ -188,7 +190,7 @@ class ApacheParser(object):
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
 
-        mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        mod_cmd = [self.configurator.option("ctl"), "-t", "-D",
                        "DUMP_MODULES"]
         matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
@@ -332,6 +334,54 @@ class ApacheParser(object):
         else:
             self.aug.set(aug_conf_path + "/directive[last()]/arg", args)
 
+    def add_dir_beginning(self, aug_conf_path, dirname, args):
+        """Adds the directive to the beginning of defined aug_conf_path.
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str dirname: Directive to add
+        :param args: Value of the directive. ie. Listen 443, 443 is arg
+        :type args: list or str
+        """
+        first_dir = aug_conf_path + "/directive[1]"
+        self.aug.insert(first_dir, "directive", True)
+        self.aug.set(first_dir, dirname)
+        if isinstance(args, list):
+            for i, value in enumerate(args, 1):
+                self.aug.set(first_dir + "/arg[%d]" % (i), value)
+        else:
+            self.aug.set(first_dir + "/arg", args)
+
+    def add_comment(self, aug_conf_path, comment):
+        """Adds the comment to the augeas path
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str comment: Comment content
+
+        """
+        self.aug.set(aug_conf_path + "/#comment[last() + 1]", comment)
+
+    def find_comments(self, arg, start=None):
+        """Finds a comment with specified content from the provided DOM path
+
+        :param str arg: Comment content to search
+        :param str start: Beginning Augeas path to begin looking
+
+        :returns: List of augeas paths containing the comment content
+        :rtype: list
+
+        """
+        if not start:
+            start = get_aug_path(self.root)
+
+        comments = self.aug.match("%s//*[label() = '#comment']" % start)
+
+        results = []
+        for comment in comments:
+            c_content = self.aug.get(comment)
+            if c_content and arg in c_content:
+                results.append(comment)
+        return results
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
@@ -391,7 +441,7 @@ class ApacheParser(object):
         else:
             arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        ordered_matches = []
+        ordered_matches = []  # type: List[str]
 
         # TODO: Wildcards should be included in alphabetical order
         # https://httpd.apache.org/docs/2.4/mod/core.html#include

certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test

@@ -46,6 +46,7 @@ function Cleanup() {
 
 # if our environment asks us to enable modules, do our best!
 if [ "$1" = --debian-modules ] ; then
+    sudo apt-get install -y apache2
     sudo apt-get install -y libapache2-mod-wsgi
     sudo apt-get install -y libapache2-mod-macro
 

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -0,0 +1,187 @@
+# pylint: disable=too-many-public-methods,too-many-lines
+"""Test for certbot_apache.configurator AutoHSTS functionality"""
+import re
+import unittest
+import mock
+# six is used in mock.patch()
+import six  # pylint: disable=unused-import
+
+from certbot import errors
+from certbot_apache import constants
+from certbot_apache.tests import util
+
+
+class AutoHSTSTest(util.ApacheTest):
+    """Tests for AutoHSTS feature"""
+    # pylint: disable=protected-access
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(AutoHSTSTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
+        self.config.parser.modules.add("headers_module")
+        self.config.parser.modules.add("mod_headers.c")
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+
+    def get_autohsts_value(self, vh_path):
+        """ Get value from Strict-Transport-Security header """
+        header_path = self.config.parser.find_dir("Header", None, vh_path)
+        if header_path:
+            pat = '(?:[ "]|^)(strict-transport-security)(?:[ "]|$)'
+            for head in header_path:
+                if re.search(pat, self.config.parser.aug.get(head).lower()):
+                    return self.config.parser.aug.get(head.replace("arg[3]",
+                                                                   "arg[4]"))
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_autohsts_enable_headers_mod(self, mock_enable, _restart):
+        self.config.parser.modules.discard("headers_module")
+        self.config.parser.modules.discard("mod_header.c")
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertTrue(mock_enable.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_deploy_already_exists(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        self.assertRaises(errors.PluginEnhancementAlreadyPresent,
+                          self.config.enable_autohsts,
+                          mock.MagicMock(), ["ocspvhost.com"])
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.prepare")
+    def test_autohsts_increase(self, mock_prepare, _mock_restart):
+        self.config._prepared = False
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        inc_val = maxage.format(constants.AUTOHSTS_STEPS[1])
+
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+        # Increase
+        self.config.update_autohsts(mock.MagicMock())
+        # Verify increased value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          inc_val)
+        self.assertTrue(mock_prepare.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._autohsts_increase")
+    def test_autohsts_increase_noop(self, mock_increase, _restart):
+        maxage = "\"max-age={0}\""
+        initial_val = maxage.format(constants.AUTOHSTS_STEPS[0])
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Verify initial value
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          initial_val)
+
+        self.config.update_autohsts(mock.MagicMock())
+        # Freq not patched, so value shouldn't increase
+        self.assertFalse(mock_increase.called)
+
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    def test_autohsts_increase_no_header(self, _restart):
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        # Remove the header
+        dir_locs = self.config.parser.find_dir("Header", None,
+                                              self.vh_truth[7].path)
+        dir_loc = "/".join(dir_locs[0].split("/")[:-1])
+        self.config.parser.aug.remove(dir_loc)
+        self.assertRaises(errors.PluginError,
+                          self.config.update_autohsts,
+                          mock.MagicMock())
+
+    @mock.patch("certbot_apache.constants.AUTOHSTS_FREQ", 0)
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    def test_autohsts_increase_and_make_permanent(self, _mock_restart):
+        maxage = "\"max-age={0}\""
+        max_val = maxage.format(constants.AUTOHSTS_PERMANENT)
+        mock_lineage = mock.MagicMock()
+        mock_lineage.key_path = "/etc/apache2/ssl/key-certbot_15.pem"
+        self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"])
+        for i in range(len(constants.AUTOHSTS_STEPS)-1):
+            # Ensure that value is not made permanent prematurely
+            self.config.deploy_autohsts(mock_lineage)
+            self.assertNotEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                                 max_val)
+            self.config.update_autohsts(mock.MagicMock())
+            # Value should match pre-permanent increment step
+            cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1])
+            self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                              cur_val)
+        # Ensure that the value is raised to max
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          maxage.format(constants.AUTOHSTS_STEPS[-1]))
+        # Make permanent
+        self.config.deploy_autohsts(mock_lineage)
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          max_val)
+
+    def test_autohsts_update_noop(self):
+        with mock.patch("time.time") as mock_time:
+            # Time mock is used to make sure that the execution does not
+            # continue when no autohsts entries exist in pluginstorage
+            self.config.update_autohsts(mock.MagicMock())
+            self.assertFalse(mock_time.called)
+
+    def test_autohsts_make_permanent_noop(self):
+        self.config.storage.put = mock.MagicMock()
+        self.config.deploy_autohsts(mock.MagicMock())
+        # Make sure that the execution does not continue when no entries in store
+        self.assertFalse(self.config.storage.put.called)
+
+    @mock.patch("certbot_apache.display_ops.select_vhost")
+    def test_autohsts_no_ssl_vhost(self, mock_select):
+        mock_select.return_value = self.vh_truth[0]
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError,
+                              self.config.enable_autohsts,
+                              mock.MagicMock(), "invalid.example.com")
+            self.assertTrue(
+                "Certbot was not able to find SSL" in mock_log.call_args[0][0])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.add_vhost_id")
+    def test_autohsts_dont_enhance_twice(self, mock_id, _restart):
+        mock_id.return_value = "1234567"
+        self.config.enable_autohsts(mock.MagicMock(),
+                                    ["ocspvhost.com", "ocspvhost.com"])
+        self.assertEquals(mock_id.call_count, 1)
+
+    def test_autohsts_remove_orphaned(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 0, "timestamp": 0}
+
+        self.config._autohsts_save_state()
+        self.config.update_autohsts(mock.MagicMock())
+        self.assertFalse("orphan_id" in self.config._autohsts)
+        # Make sure it's removed from the pluginstorage file as well
+        self.config._autohsts = None
+        self.config._autohsts_fetch_state()
+        self.assertFalse(self.config._autohsts)
+
+    def test_autohsts_make_permanent_vhost_not_found(self):
+        # pylint: disable=protected-access
+        self.config._autohsts_fetch_state()
+        self.config._autohsts["orphan_id"] = {"laststep": 999, "timestamp": 0}
+        self.config._autohsts_save_state()
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.config.deploy_autohsts(mock.MagicMock())
+            self.assertTrue(mock_log.called)
+            self.assertTrue(
+                "VirtualHost with id orphan_id was not" in mock_log.call_args[0][0])
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/centos_test.py

@@ -4,6 +4,8 @@ import unittest
 
 import mock
 
+from certbot import errors
+
 from certbot_apache import obj
 from certbot_apache import override_centos
 from certbot_apache.tests import util
@@ -121,5 +123,19 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
         self.assertTrue("MOCK_NOSEP" in self.config.parser.variables.keys())
         self.assertEqual("NOSEP_VAL", self.config.parser.variables["NOSEP_TWO"])
 
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_errors(self, mock_run_script):
+        mock_run_script.side_effect = [None,
+                                       errors.SubprocessError,
+                                       errors.SubprocessError]
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -115,9 +115,22 @@ class MultipleVhostsTest(util.ApacheTest):
         # Weak test..
         ApacheConfigurator.add_parser_arguments(mock.MagicMock())
 
+    def test_add_parser_arguments_all_configurators(self):  # pylint: disable=no-self-use
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        for cls in OVERRIDE_CLASSES.values():
+            cls.add_parser_arguments(mock.MagicMock())
+
+    def test_all_configurators_defaults_defined(self):
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        from certbot_apache.configurator import ApacheConfigurator
+        parameters = set(ApacheConfigurator.OS_DEFAULTS.keys())
+        for cls in OVERRIDE_CLASSES.values():
+            self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys())))
+
     def test_constant(self):
-        self.assertEqual(self.config.constant("server_root"), "/etc/apache2")
-        self.assertEqual(self.config.constant("nonexistent"), None)
+        self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in
+                        self.config.option("server_root"))
+        self.assertEqual(self.config.option("nonexistent"), None)
 
     @certbot_util.patch_get_utility()
     def test_get_all_names(self, mock_getutility):
@@ -126,7 +139,7 @@ class MultipleVhostsTest(util.ApacheTest):
         names = self.config.get_all_names()
         self.assertEqual(names, set(
             ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
-             "nonsym.link", "vhost.in.rootconf"]
+             "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"]
         ))
 
     @certbot_util.patch_get_utility()
@@ -146,7 +159,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         names = self.config.get_all_names()
         # Names get filtered, only 5 are returned
-        self.assertEqual(len(names), 7)
+        self.assertEqual(len(names), 8)
         self.assertTrue("zombo.com" in names)
         self.assertTrue("google.com" in names)
         self.assertTrue("certbot.demo" in names)
@@ -246,7 +259,7 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.display_ops.select_vhost")
     def test_choose_vhost_select_vhost_with_temp(self, mock_select):
         mock_select.return_value = self.vh_truth[0]
-        chosen_vhost = self.config.choose_vhost("none.com", temp=True)
+        chosen_vhost = self.config.choose_vhost("none.com", create_if_no_ssl=False)
         self.assertEqual(self.vh_truth[0], chosen_vhost)
 
     @mock.patch("certbot_apache.display_ops.select_vhost")
@@ -260,6 +273,20 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertRaises(
             errors.PluginError, self.config.choose_vhost, "none.com")
 
+    def test_find_best_http_vhost_default(self):
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True)
+        self.config.vhosts = [vh]
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh)
+
+    def test_find_best_http_vhost_port(self):
+        port = "8080"
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("*:" + port)]),
+            False, True, "encryption-example.demo")
+        self.config.vhosts.append(vh)
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh)
+
     def test_findbest_continues_on_short_domain(self):
         # pylint: disable=protected-access
         chosen_vhost = self.config._find_best_vhost("purple.com")
@@ -305,7 +332,8 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_non_default_vhosts(self):
         # pylint: disable=protected-access
-        self.assertEqual(len(self.config._non_default_vhosts()), 8)
+        vhosts = self.config._non_default_vhosts(self.config.vhosts)
+        self.assertEqual(len(vhosts), 8)
 
     def test_deploy_cert_enable_new_vhost(self):
         # Create
@@ -320,6 +348,30 @@ class MultipleVhostsTest(util.ApacheTest):
             "example/cert_chain.pem", "example/fullchain.pem")
         self.assertTrue(ssl_vhost.enabled)
 
+    def test_no_duplicate_include(self):
+        def mock_find_dir(directive, argument, _):
+            """Mock method for parser.find_dir"""
+            if directive == "Include" and argument.endswith("options-ssl-apache.conf"):
+                return ["/path/to/whatever"]
+
+        mock_add = mock.MagicMock()
+        self.config.parser.add_dir = mock_add
+        self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
+        tried_to_add = False
+        for a in mock_add.call_args_list:
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                tried_to_add = True
+        # Include should be added, find_dir is not patched, and returns falsy
+        self.assertTrue(tried_to_add)
+
+        self.config.parser.find_dir = mock_find_dir
+        mock_add.reset_mock()
+        self.config._add_dummy_ssl_directives(self.vh_truth[0])  # pylint: disable=protected-access
+        for a in mock_add.call_args_list:
+            if a[0][1] == "Include" and a[0][2] == self.config.mod_ssl_conf:
+                self.fail("Include shouldn't be added, as patched find_dir 'finds' existing one") \
+                    # pragma: no cover
+
     def test_deploy_cert(self):
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")
@@ -399,13 +451,37 @@ class MultipleVhostsTest(util.ApacheTest):
             self.vh_truth[1].path))
 
     def test_deploy_cert_invalid_vhost(self):
+        """For test cases where the `ApacheConfigurator` class' `_deploy_cert`
+        method is called with an invalid vhost parameter. Currently this tests
+        that a PluginError is appropriately raised when important directives
+        are missing in an SSL module."""
         self.config.parser.modules.add("ssl_module")
-        mock_find = mock.MagicMock()
-        mock_find.return_value = []
-        self.config.parser.find_dir = mock_find
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("socache_shmcb_module")
+
+        def side_effect(*args):
+            """Mocks case where an SSLCertificateFile directive can be found
+            but an SSLCertificateKeyFile directive is missing."""
+            if "SSLCertificateFile" in args:
+                return ["example/cert.pem"]
+            else:
+                return []
+
+        mock_find_dir = mock.MagicMock(return_value=[])
+        mock_find_dir.side_effect = side_effect
+
+        self.config.parser.find_dir = mock_find_dir
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
+
+        self.assertRaises(
+            errors.PluginError, self.config.deploy_cert, "random.demo",
+            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
+
+        # Remove side_effect to mock case where both SSLCertificateFile
+        # and SSLCertificateKeyFile directives are missing
+        self.config.parser.find_dir.side_effect = None
         self.assertRaises(
             errors.PluginError, self.config.deploy_cert, "random.demo",
             "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
@@ -424,6 +500,47 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertTrue(self.config.parser.find_dir(
             "NameVirtualHost", "*:80"))
 
+    def test_add_listen_80(self):
+        mock_find = mock.Mock()
+        mock_add_dir = mock.Mock()
+        mock_find.return_value = []
+        self.config.parser.find_dir = mock_find
+        self.config.parser.add_dir = mock_add_dir
+        self.config.ensure_listen("80")
+        self.assertTrue(mock_add_dir.called)
+        self.assertTrue(mock_find.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        self.assertEqual(mock_add_dir.call_args[0][2], "80")
+
+    def test_add_listen_80_named(self):
+        mock_find = mock.Mock()
+        mock_find.return_value = ["test1", "test2", "test3"]
+        mock_get = mock.Mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+        mock_add_dir = mock.Mock()
+
+        self.config.parser.find_dir = mock_find
+        self.config.parser.get_arg = mock_get
+        self.config.parser.add_dir = mock_add_dir
+
+        self.config.ensure_listen("80")
+        self.assertEqual(mock_add_dir.call_count, 0)
+
+        # Reset return lists and inputs
+        mock_add_dir.reset_mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+
+        # Test
+        self.config.ensure_listen("8080")
+        self.assertEqual(mock_add_dir.call_count, 3)
+        self.assertTrue(mock_add_dir.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        call_found = False
+        for mock_call in mock_add_dir.mock_calls:
+            if mock_call[1][2] == ['1.2.3.4:8080']:
+                call_found = True
+        self.assertTrue(call_found)
+
     def test_prepare_server_https(self):
         mock_enable = mock.Mock()
         self.config.enable_mod = mock_enable
@@ -435,7 +552,6 @@ class MultipleVhostsTest(util.ApacheTest):
         # This will test the Add listen
         self.config.parser.find_dir = mock_find
         self.config.parser.add_dir_to_ifmodssl = mock_add_dir
-
         self.config.prepare_server_https("443")
         # Changing the order these modules are enabled breaks the reverter
         self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb")
@@ -548,22 +664,10 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(ssl_vhost_slink.name, "nonsym.link")
 
     def test_make_vhost_ssl_nonexistent_vhost_path(self):
-        def conf_side_effect(arg):
-            """ Mock function for ApacheConfigurator.conf """
-            confvars = {
-                "vhost-root": "/tmp/nonexistent",
-                "le_vhost_ext": "-le-ssl.conf",
-                "handle-sites": True}
-            return confvars[arg]
-
-        with mock.patch(
-                "certbot_apache.configurator.ApacheConfigurator.conf"
-        ) as mock_conf:
-            mock_conf.side_effect = conf_side_effect
-            ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
-            self.assertEqual(os.path.dirname(ssl_vhost.filep),
-                             os.path.dirname(os.path.realpath(
-                                 self.vh_truth[1].filep)))
+        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
+        self.assertEqual(os.path.dirname(ssl_vhost.filep),
+                            os.path.dirname(os.path.realpath(
+                                self.vh_truth[1].filep)))
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -676,23 +780,33 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config._add_name_vhost_if_necessary(self.vh_truth[0])
         self.assertEqual(self.config.add_name_vhost.call_count, 2)
 
+    @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform")
     @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform")
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
-    def test_perform(self, mock_restart, mock_perform):
+    def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
-        account_key, achall1, achall2 = self.get_achalls()
+        account_key, achalls = self.get_key_and_achalls()
 
-        expected = [
-            achall1.response(account_key),
-            achall2.response(account_key),
-        ]
+        all_expected = []
+        http_expected = []
+        tls_expected = []
+        for achall in achalls:
+            response = achall.response(account_key)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_expected.append(response)
+            else:
+                tls_expected.append(response)
+            all_expected.append(response)
 
-        mock_perform.return_value = expected
-        responses = self.config.perform([achall1, achall2])
+        mock_http_perform.return_value = http_expected
+        mock_tls_perform.return_value = tls_expected
 
-        self.assertEqual(mock_perform.call_count, 1)
-        self.assertEqual(responses, expected)
+        responses = self.config.perform(achalls)
+
+        self.assertEqual(mock_http_perform.call_count, 1)
+        self.assertEqual(mock_tls_perform.call_count, 1)
+        self.assertEqual(responses, all_expected)
 
         self.assertEqual(mock_restart.call_count, 1)
 
@@ -700,29 +814,32 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
-        self.config._chall_out.add(achall2)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall1])
-        self.assertFalse(mock_restart.called)
-
-        self.config.cleanup([achall2])
-        self.assertTrue(mock_restart.called)
+        for i, achall in enumerate(achalls):
+            self.config.cleanup([achall])
+            if i == len(achalls) - 1:
+                self.assertTrue(mock_restart.called)
+            else:
+                self.assertFalse(mock_restart.called)
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup_no_errors(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
+        self.config.http_doer = mock.MagicMock()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall2])
+        self.config.cleanup([achalls[-1]])
         self.assertFalse(mock_restart.called)
 
-        self.config.cleanup([achall1, achall2])
+        self.config.cleanup(achalls)
         self.assertTrue(mock_restart.called)
 
     @mock.patch("certbot.util.run_script")
@@ -817,6 +934,22 @@ class MultipleVhostsTest(util.ApacheTest):
             errors.PluginError,
             self.config.enhance, "certbot.demo", "unknown_enhancement")
 
+    def test_enhance_no_ssl_vhost(self):
+        with mock.patch("certbot_apache.configurator.logger.warning") as mock_log:
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "redirect")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("\"redirect\"" in mock_log.call_args[0][0])
+
+            mock_log.reset_mock()
+
+            self.assertRaises(errors.PluginError, self.config.enhance,
+                              "certbot.demo", "ensure-http-header", "Test")
+            # Check that correct logger.warning was printed
+            self.assertTrue("not able to find" in mock_log.call_args[0][0])
+            self.assertTrue("Test" in mock_log.call_args[0][0])
+
     @mock.patch("certbot.util.exe_exists")
     def test_ocsp_stapling(self, mock_exe):
         self.config.parser.update_runtime_variables = mock.Mock()
@@ -826,6 +959,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
 
         # Get the ssl vhost for certbot.demo
@@ -852,6 +986,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # Checking the case with already enabled ocsp stapling configuration
+        self.config.choose_vhost("ocspvhost.com")
         self.config.enhance("ocspvhost.com", "staple-ocsp")
 
         # Get the ssl vhost for letsencrypt.demo
@@ -876,6 +1011,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.parser.modules.add("socache_shmcb_module")
         self.config.get_version = mock.Mock(return_value=(2, 2, 0))
+        self.config.choose_vhost("certbot.demo")
 
         self.assertRaises(errors.PluginError,
                 self.config.enhance, "certbot.demo", "staple-ocsp")
@@ -901,6 +1037,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -920,7 +1057,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Strict-Transport-Security")
 
@@ -939,6 +1077,7 @@ class MultipleVhostsTest(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -960,7 +1099,8 @@ class MultipleVhostsTest(util.ApacheTest):
         # skip the enable mod
         self.config.parser.modules.add("headers_module")
 
-        # This will create an ssl vhost for certbot.demo
+        # This will create an ssl vhost for encryption-example.demo
+        self.config.choose_vhost("encryption-example.demo")
         self.config.enhance("encryption-example.demo", "ensure-http-header",
                             "Upgrade-Insecure-Requests")
 
@@ -978,6 +1118,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.get_version = mock.Mock(return_value=(2, 2))
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1028,6 +1169,7 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.save()
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
 
         # These are not immediately available in find_dir even with save() and
@@ -1094,6 +1236,9 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
 
+        # Creates ssl vhost for the domain
+        self.config.choose_vhost("red.blue.purple.com")
+
         self.config.enhance("red.blue.purple.com", "redirect")
         verify_no_redirect = ("certbot_apache.configurator."
                               "ApacheConfigurator._verify_no_certbot_redirect")
@@ -1105,7 +1250,7 @@ class MultipleVhostsTest(util.ApacheTest):
         # Skip the enable mod
         self.config.parser.modules.add("rewrite_module")
         self.config.get_version = mock.Mock(return_value=(2, 3, 9))
-
+        self.config.choose_vhost("red.blue.purple.com")
         self.config.enhance("red.blue.purple.com", "redirect")
         # Clear state about enabling redirect on this run
         # pylint: disable=protected-access
@@ -1151,7 +1296,7 @@ class MultipleVhostsTest(util.ApacheTest):
         not_rewriterule = "NotRewriteRule ^ ..."
         self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule))
 
-    def get_achalls(self):
+    def get_key_and_achalls(self):
         """Return testing achallenges."""
         account_key = self.rsa512jwk
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
@@ -1166,8 +1311,12 @@ class MultipleVhostsTest(util.ApacheTest):
                     token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
                 "pending"),
             domain="certbot.demo", account_key=account_key)
+        achall3 = achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=(b'x' * 16)), "pending"),
+            domain="example.org", account_key=account_key)
 
-        return account_key, achall1, achall2
+        return account_key, (achall1, achall2, achall3)
 
     def test_make_addrs_sni_ready(self):
         self.config.version = (2, 2)
@@ -1238,6 +1387,123 @@ class MultipleVhostsTest(util.ApacheTest):
                           self.config.enable_mod,
                           "whatever")
 
+    def test_wildcard_domain(self):
+        # pylint: disable=protected-access
+        cases = {u"*.example.org": True, b"*.x.example.org": True,
+                 u"a.example.org": False, b"a.x.example.org": False}
+        for key in cases.keys():
+            self.assertEqual(self.config._wildcard_domain(key), cases[key])
+
+    def test_choose_vhosts_wildcard(self):
+        # pylint: disable=protected-access
+        mock_path = "certbot_apache.display_ops.select_vhost_multiple"
+        with mock.patch(mock_path) as mock_select_vhs:
+            mock_select_vhs.return_value = [self.vh_truth[3]]
+            vhs = self.config._choose_vhosts_wildcard("*.certbot.demo",
+                                                     create_ssl=True)
+            # Check that the dialog was called with one vh: certbot.demo
+            self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[3])
+            self.assertEquals(len(mock_select_vhs.call_args_list), 1)
+
+            # And the actual returned values
+            self.assertEquals(len(vhs), 1)
+            self.assertTrue(vhs[0].name == "certbot.demo")
+            self.assertTrue(vhs[0].ssl)
+
+            self.assertFalse(vhs[0] == self.vh_truth[3])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl")
+    def test_choose_vhosts_wildcard_no_ssl(self, mock_makessl):
+        # pylint: disable=protected-access
+        mock_path = "certbot_apache.display_ops.select_vhost_multiple"
+        with mock.patch(mock_path) as mock_select_vhs:
+            mock_select_vhs.return_value = [self.vh_truth[1]]
+            vhs = self.config._choose_vhosts_wildcard("*.certbot.demo",
+                                                     create_ssl=False)
+            self.assertFalse(mock_makessl.called)
+            self.assertEquals(vhs[0], self.vh_truth[1])
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._vhosts_for_wildcard")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.make_vhost_ssl")
+    def test_choose_vhosts_wildcard_already_ssl(self, mock_makessl, mock_vh_for_w):
+        # pylint: disable=protected-access
+        # Already SSL vhost
+        mock_vh_for_w.return_value = [self.vh_truth[7]]
+        mock_path = "certbot_apache.display_ops.select_vhost_multiple"
+        with mock.patch(mock_path) as mock_select_vhs:
+            mock_select_vhs.return_value = [self.vh_truth[7]]
+            vhs = self.config._choose_vhosts_wildcard("whatever",
+                                                     create_ssl=True)
+            self.assertEquals(mock_select_vhs.call_args[0][0][0], self.vh_truth[7])
+            self.assertEquals(len(mock_select_vhs.call_args_list), 1)
+            # Ensure that make_vhost_ssl was not called, vhost.ssl == true
+            self.assertFalse(mock_makessl.called)
+
+            # And the actual returned values
+            self.assertEquals(len(vhs), 1)
+            self.assertTrue(vhs[0].ssl)
+            self.assertEquals(vhs[0], self.vh_truth[7])
+
+
+    def test_deploy_cert_wildcard(self):
+        # pylint: disable=protected-access
+        mock_choose_vhosts = mock.MagicMock()
+        mock_choose_vhosts.return_value = [self.vh_truth[7]]
+        self.config._choose_vhosts_wildcard = mock_choose_vhosts
+        mock_d = "certbot_apache.configurator.ApacheConfigurator._deploy_cert"
+        with mock.patch(mock_d) as mock_dep:
+            self.config.deploy_cert("*.wildcard.example.org", "/tmp/path",
+                                    "/tmp/path", "/tmp/path", "/tmp/path")
+            self.assertTrue(mock_dep.called)
+            self.assertEquals(len(mock_dep.call_args_list), 1)
+            self.assertEqual(self.vh_truth[7], mock_dep.call_args_list[0][0][0])
+
+    @mock.patch("certbot_apache.display_ops.select_vhost_multiple")
+    def test_deploy_cert_wildcard_no_vhosts(self, mock_dialog):
+        # pylint: disable=protected-access
+        mock_dialog.return_value = []
+        self.assertRaises(errors.PluginError,
+                          self.config.deploy_cert,
+                          "*.wild.cat", "/tmp/path", "/tmp/path",
+                           "/tmp/path", "/tmp/path")
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard")
+    def test_enhance_wildcard_after_install(self, mock_choose):
+        # pylint: disable=protected-access
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
+        self.vh_truth[3].ssl = True
+        self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]]
+        self.config.enhance("*.certbot.demo", "ensure-http-header",
+                            "Upgrade-Insecure-Requests")
+        self.assertFalse(mock_choose.called)
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator._choose_vhosts_wildcard")
+    def test_enhance_wildcard_no_install(self, mock_choose):
+        self.vh_truth[3].ssl = True
+        mock_choose.return_value = [self.vh_truth[3]]
+        self.config.parser.modules.add("mod_ssl.c")
+        self.config.parser.modules.add("headers_module")
+        self.config.enhance("*.certbot.demo", "ensure-http-header",
+                            "Upgrade-Insecure-Requests")
+        self.assertTrue(mock_choose.called)
+
+    def test_add_vhost_id(self):
+        for vh in [self.vh_truth[0], self.vh_truth[1], self.vh_truth[2]]:
+            vh_id = self.config.add_vhost_id(vh)
+            self.assertEqual(vh, self.config.find_vhost_by_id(vh_id))
+
+    def test_find_vhost_by_id_404(self):
+        self.assertRaises(errors.PluginError,
+                          self.config.find_vhost_by_id,
+                          "nonexistent")
+
+    def test_add_vhost_id_already_exists(self):
+        first_id = self.config.add_vhost_id(self.vh_truth[0])
+        second_id = self.config.add_vhost_id(self.vh_truth[0])
+        self.assertEqual(first_id, second_id)
+
+
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""
     # pylint: disable=protected-access
@@ -1318,7 +1584,7 @@ class AugeasVhostsTest(util.ApacheTest):
                           broken_vhost)
 
 class MultiVhostsTest(util.ApacheTest):
-    """Test vhosts with illegal names dependent on augeas version."""
+    """Test configuration with multiple virtualhosts in a single file."""
     # pylint: disable=protected-access
 
     def setUp(self):  # pylint: disable=arguments-differ
@@ -1438,7 +1704,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC"))
+        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1474,7 +1740,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1490,7 +1756,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:

certbot-apache/certbot_apache/tests/debian_test.py

@@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTestDebian, self).setUp()
         self.config = util.get_apache_configurator(
-            self.config_path, None, self.config_dir, self.work_dir,
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
             os_info="debian")
         self.config = self.mock_deploy_cert(self.config)
         self.vh_truth = util.get_vh_truth(self.temp_dir,
@@ -161,6 +161,8 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         self.config.parser.modules.add("mod_ssl.c")
         self.config.get_version = mock.Mock(return_value=(2, 4, 7))
         mock_exe.return_value = True
+        # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "staple-ocsp")
         self.assertTrue("socache_shmcb_module" in self.config.parser.modules)
 
@@ -172,6 +174,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
 
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "ensure-http-header",
                             "Strict-Transport-Security")
         self.assertTrue("headers_module" in self.config.parser.modules)
@@ -183,6 +186,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
         mock_exe.return_value = True
         self.config.get_version = mock.Mock(return_value=(2, 2))
         # This will create an ssl vhost for certbot.demo
+        self.config.choose_vhost("certbot.demo")
         self.config.enhance("certbot.demo", "redirect")
         self.assertTrue("rewrite_module" in self.config.parser.modules)
 

certbot-apache/certbot_apache/tests/display_ops_test.py

@@ -11,9 +11,39 @@ from certbot.tests import util as certbot_util
 
 from certbot_apache import obj
 
+from certbot_apache.display_ops import select_vhost_multiple
 from certbot_apache.tests import util
 
 
+class SelectVhostMultiTest(unittest.TestCase):
+    """Tests for certbot_apache.display_ops.select_vhost_multiple."""
+
+    def setUp(self):
+        self.base_dir = "/example_path"
+        self.vhosts = util.get_vh_truth(
+            self.base_dir, "debian_apache_2_4/multiple_vhosts")
+
+    def test_select_no_input(self):
+        self.assertFalse(select_vhost_multiple([]))
+
+    @certbot_util.patch_get_utility()
+    def test_select_correct(self, mock_util):
+        mock_util().checklist.return_value = (
+            display_util.OK, [self.vhosts[3].display_repr(),
+                              self.vhosts[2].display_repr()])
+        vhs = select_vhost_multiple([self.vhosts[3],
+                                     self.vhosts[2],
+                                     self.vhosts[1]])
+        self.assertTrue(self.vhosts[2] in vhs)
+        self.assertTrue(self.vhosts[3] in vhs)
+        self.assertFalse(self.vhosts[1] in vhs)
+
+    @certbot_util.patch_get_utility()
+    def test_select_cancel(self, mock_util):
+        mock_util().checklist.return_value = (display_util.CANCEL, "whatever")
+        vhs = select_vhost_multiple([self.vhosts[2], self.vhosts[3]])
+        self.assertFalse(vhs)
+
 class SelectVhostTest(unittest.TestCase):
     """Tests for certbot_apache.display_ops.select_vhost."""
 

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -2,6 +2,10 @@
 import os
 import unittest
 
+import mock
+
+from certbot import errors
+
 from certbot_apache import override_gentoo
 from certbot_apache import obj
 from certbot_apache.tests import util
@@ -46,9 +50,10 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
                                                     config_root=config_root,
                                                     vhost_root=vhost_root)
 
-        self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
-            os_info="gentoo")
+        with mock.patch("certbot_apache.override_gentoo.GentooParser.update_runtime_variables"):
+            self.config = util.get_apache_configurator(
+                self.config_path, self.vhost_path, self.config_dir, self.work_dir,
+                os_info="gentoo")
         self.vh_truth = get_vh_truth(
             self.temp_dir, "gentoo_apache/apache")
 
@@ -78,9 +83,53 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.config.parser.apacheconfig_filep = os.path.realpath(
             os.path.join(self.config.parser.root, "../conf.d/apache2"))
         self.config.parser.variables = {}
-        self.config.parser.update_runtime_variables()
+        with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"):
+            self.config.parser.update_runtime_variables()
         for define in defines:
             self.assertTrue(define in self.config.parser.variables.keys())
 
+    @mock.patch("certbot_apache.parser.ApacheParser.parse_from_subprocess")
+    def test_no_binary_configdump(self, mock_subprocess):
+        """Make sure we don't call binary dumps other than modules from Apache
+        as this is not supported in Gentoo currently"""
+
+        with mock.patch("certbot_apache.override_gentoo.GentooParser.update_modules"):
+            self.config.parser.update_runtime_variables()
+            self.config.parser.reset_modules()
+        self.assertFalse(mock_subprocess.called)
+
+        self.config.parser.update_runtime_variables()
+        self.config.parser.reset_modules()
+        self.assertTrue(mock_subprocess.called)
+
+    @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_opportunistic_httpd_runtime_parsing(self, mock_get):
+        mod_val = (
+            'Loaded Modules:\n'
+            ' mock_module (static)\n'
+            ' another_module (static)\n'
+        )
+        def mock_get_cfg(command):
+            """Mock httpd process stdout"""
+            if command == ['apache2ctl', 'modules']:
+                return mod_val
+        mock_get.side_effect = mock_get_cfg
+        self.config.parser.modules = set()
+
+        with mock.patch("certbot.util.get_os_info") as mock_osi:
+            # Make sure we have the have the Gentoo httpd constants
+            mock_osi.return_value = ("gentoo", "123")
+            self.config.parser.update_runtime_variables()
+
+        self.assertEquals(mock_get.call_count, 1)
+        self.assertEquals(len(self.config.parser.modules), 4)
+        self.assertTrue("mod_another.c" in self.config.parser.modules)
+
+    @mock.patch("certbot_apache.configurator.util.run_script")
+    def test_alt_restart_works(self, mock_run_script):
+        mock_run_script.side_effect = [None, errors.SubprocessError, None]
+        self.config.restart()
+        self.assertEquals(mock_run_script.call_count, 3)
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -0,0 +1,222 @@
+"""Test for certbot_apache.http_01."""
+import mock
+import os
+import unittest
+
+from acme import challenges
+from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+
+from certbot import achallenges
+from certbot import errors
+
+from certbot.tests import acme_util
+from certbot_apache.parser import get_aug_path
+from certbot_apache.tests import util
+
+
+NUM_ACHALLS = 3
+
+
+class ApacheHttp01Test(util.ApacheTest):
+    """Test for certbot_apache.http_01.ApacheHttp01."""
+
+    def setUp(self, *args, **kwargs):
+        super(ApacheHttp01Test, self).setUp(*args, **kwargs)
+
+        self.account_key = self.rsa512jwk
+        self.achalls = []  # type: List[achallenges.KeyAuthorizationAnnotatedChallenge]
+        vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+        # Takes the vhosts for encryption-example.demo, certbot.demo, and
+        # vhost.in.rootconf
+        self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]]
+
+        for i in range(NUM_ACHALLS):
+            self.achalls.append(
+                achallenges.KeyAuthorizationAnnotatedChallenge(
+                    challb=acme_util.chall_to_challb(
+                        challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))),
+                        "pending"),
+                    domain=self.vhosts[i].name, account_key=self.account_key))
+
+        modules = ["rewrite", "authz_core", "authz_host"]
+        for mod in modules:
+            self.config.parser.modules.add("mod_{0}.c".format(mod))
+            self.config.parser.modules.add(mod + "_module")
+
+        from certbot_apache.http_01 import ApacheHttp01
+        self.http = ApacheHttp01(self.config)
+
+    def test_empty_perform(self):
+        self.assertFalse(self.http.perform())
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_enable_modules_apache_2_2(self, mock_enmod):
+        self.config.version = (2, 2)
+        self.config.parser.modules.remove("authz_host_module")
+        self.config.parser.modules.remove("mod_authz_host.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_host")
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_enable_modules_apache_2_4(self, mock_enmod):
+        self.config.parser.modules.remove("authz_core_module")
+        self.config.parser.modules.remove("mod_authz_core.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_core")
+
+    def common_enable_modules_test(self, mock_enmod):
+        """Tests enabling mod_rewrite and other modules."""
+        self.config.parser.modules.remove("rewrite_module")
+        self.config.parser.modules.remove("mod_rewrite.c")
+
+        self.http.prepare_http01_modules()
+
+        self.assertTrue(mock_enmod.called)
+        calls = mock_enmod.call_args_list
+        other_calls = []
+        for call in calls:
+            if "rewrite" != call[0][0]:
+                other_calls.append(call)
+
+        # If these lists are equal, we never enabled mod_rewrite
+        self.assertNotEqual(calls, other_calls)
+        return other_calls
+
+    def test_same_vhost(self):
+        vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo")
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain=vhost.name, account_key=self.account_key),
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'b' * 16))),
+                    "pending"),
+                domain=next(iter(vhost.aliases)), account_key=self.account_key)
+        ]
+        self.common_perform_test(achalls, [vhost])
+
+    def test_anonymous_vhost(self):
+        vhosts = [v for v in self.config.vhosts if not v.ssl]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="something.nonexistent", account_key=self.account_key)]
+        self.common_perform_test(achalls, vhosts)
+
+    def test_no_vhost(self):
+        for achall in self.achalls:
+            self.http.add_chall(achall)
+        self.config.config.http01_port = 12345
+        self.assertRaises(errors.PluginError, self.http.perform)
+
+    def test_perform_1_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=2)
+
+    def test_perform_1_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=1, minor_version=4)
+
+    def test_perform_2_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=2)
+
+    def test_perform_2_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=2, minor_version=4)
+
+    def test_perform_3_achall_apache_2_2(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=2)
+
+    def test_perform_3_achall_apache_2_4(self):
+        self.combinations_perform_test(num_achalls=3, minor_version=4)
+
+    def test_activate_disabled_vhost(self):
+        vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="certbot.demo", account_key=self.account_key)]
+        vhosts[0].enabled = False
+        self.common_perform_test(achalls, vhosts)
+        matches = self.config.parser.find_dir(
+            "Include", vhosts[0].filep,
+            get_aug_path(self.config.parser.loc["default"]))
+        self.assertEqual(len(matches), 1)
+
+    def combinations_perform_test(self, num_achalls, minor_version):
+        """Test perform with the given achall count and Apache version."""
+        achalls = self.achalls[:num_achalls]
+        vhosts = self.vhosts[:num_achalls]
+        self.config.version = (2, minor_version)
+        self.common_perform_test(achalls, vhosts)
+
+    def common_perform_test(self, achalls, vhosts):
+        """Tests perform with the given achalls."""
+        challenge_dir = self.http.challenge_dir
+        self.assertFalse(os.path.exists(challenge_dir))
+        for achall in achalls:
+            self.http.add_chall(achall)
+
+        expected_response = [
+            achall.response(self.account_key) for achall in achalls]
+        self.assertEqual(self.http.perform(), expected_response)
+
+        self.assertTrue(os.path.isdir(self.http.challenge_dir))
+        self._has_min_permissions(self.http.challenge_dir, 0o755)
+        self._test_challenge_conf()
+
+        for achall in achalls:
+            self._test_challenge_file(achall)
+
+        for vhost in vhosts:
+            if not vhost.ssl:
+                matches = self.config.parser.find_dir("Include",
+                                                      self.http.challenge_conf_pre,
+                                                      vhost.path)
+                self.assertEqual(len(matches), 1)
+                matches = self.config.parser.find_dir("Include",
+                                                      self.http.challenge_conf_post,
+                                                      vhost.path)
+                self.assertEqual(len(matches), 1)
+
+        self.assertTrue(os.path.exists(challenge_dir))
+
+    def _test_challenge_conf(self):
+        with open(self.http.challenge_conf_pre) as f:
+            pre_conf_contents = f.read()
+
+        with open(self.http.challenge_conf_post) as f:
+            post_conf_contents = f.read()
+
+        self.assertTrue("RewriteEngine on" in pre_conf_contents)
+        self.assertTrue("RewriteRule" in pre_conf_contents)
+
+        self.assertTrue(self.http.challenge_dir in post_conf_contents)
+        if self.config.version < (2, 4):
+            self.assertTrue("Allow from all" in post_conf_contents)
+        else:
+            self.assertTrue("Require all granted" in post_conf_contents)
+
+    def _test_challenge_file(self, achall):
+        name = os.path.join(self.http.challenge_dir, achall.chall.encode("token"))
+        validation = achall.validation(self.account_key)
+
+        self._has_min_permissions(name, 0o644)
+        with open(name, 'rb') as f:
+            self.assertEqual(f.read(), validation.encode())
+
+    def _has_min_permissions(self, path, min_mode):
+        """Tests the given file has at least the permissions in mode."""
+        st_mode = os.stat(path).st_mode
+        self.assertEqual(st_mode, st_mode | min_mode)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/parser_test.py

@@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest):
         for i, match in enumerate(matches):
             self.assertEqual(self.parser.aug.get(match), str(i + 1))
 
+    def test_add_dir_beginning(self):
+        aug_default = "/files" + self.parser.loc["default"]
+        self.parser.add_dir_beginning(aug_default,
+                                      "AddDirectiveBeginning",
+                                      "testBegin")
+
+        self.assertTrue(
+            self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default))
+
+        self.assertEqual(
+            self.parser.aug.get(aug_default+"/directive[1]"),
+                                "AddDirectiveBeginning")
+        self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"])
+        matches = self.parser.find_dir("AddList", None, aug_default)
+        for i, match in enumerate(matches):
+            self.assertEqual(self.parser.aug.get(match), str(i + 1))
+
     def test_empty_arg(self):
         self.assertEquals(None,
                           self.parser.get_arg("/files/whatever/nonexistent"))
@@ -265,11 +282,11 @@ class BasicParserTest(util.ParserTest):
         self.assertRaises(
             errors.PluginError, self.parser.update_runtime_variables)
 
-    @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.option")
     @mock.patch("certbot_apache.parser.subprocess.Popen")
-    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const):
+    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt):
         mock_popen.side_effect = OSError
-        mock_const.return_value = "nonexistent"
+        mock_opt.return_value = "nonexistent"
         self.assertRaises(
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
@@ -282,6 +299,13 @@ class BasicParserTest(util.ParserTest):
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)
 
+    def test_add_comment(self):
+        from certbot_apache.parser import get_aug_path
+        self.parser.add_comment(get_aug_path(self.parser.loc["name"]), "123456")
+        comm = self.parser.find_comments("123456")
+        self.assertEquals(len(comm), 1)
+        self.assertTrue(self.parser.loc["name"] in comm[0])
+
 
 class ParserInitTest(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ

certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf

@@ -1,5 +1,6 @@
 <VirtualHost *:80>
 ServerName certbot.demo
+ServerAlias www.certbot.demo
 ServerAdmin webmaster@localhost
 
 DocumentRoot /var/www-certbot-reworld/static/

certbot-apache/certbot_apache/tests/tls_sni_01_test.py

@@ -1,6 +1,6 @@
 """Test for certbot_apache.tls_sni_01."""
-import unittest
 import shutil
+import unittest
 
 import mock
 
@@ -16,8 +16,8 @@ from six.moves import xrange  # pylint: disable=redefined-builtin, import-error
 class TlsSniPerformTest(util.ApacheTest):
     """Test the ApacheTlsSni01 challenge."""
 
-    auth_key = common_test.TLSSNI01Test.auth_key
-    achalls = common_test.TLSSNI01Test.achalls
+    auth_key = common_test.AUTH_KEY
+    achalls = common_test.ACHALLS
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(TlsSniPerformTest, self).setUp()

certbot-apache/certbot_apache/tests/util.py

@@ -87,7 +87,6 @@ class ParserTest(ApacheTest):
 def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-locals
         config_path, vhost_path,
         config_dir, work_dir, version=(2, 4, 7),
-        conf=None,
         os_info="generic",
         conf_vhost_path=None):
     """Create an Apache Configurator with the specified options.
@@ -98,46 +97,36 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
     backups = os.path.join(work_dir, "backups")
     mock_le_config = mock.MagicMock(
         apache_server_root=config_path,
-        apache_vhost_root=conf_vhost_path,
+        apache_vhost_root=None,
         apache_le_vhost_ext="-le-ssl.conf",
         apache_challenge_location=config_path,
+        apache_enmod=None,
         backup_dir=backups,
         config_dir=config_dir,
+        http01_port=80,
         temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)
 
-    orig_os_constant = configurator.ApacheConfigurator(mock_le_config,
-                                                       name="apache",
-                                                       version=version).constant
-
-    def mock_os_constant(key, vhost_path=vhost_path):
-        """Mock default vhost path"""
-        if key == "vhost_root":
-            return vhost_path
-        else:
-            return orig_os_constant(key)
-
-    with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons:
-        mock_cons.side_effect = mock_os_constant
-        with mock.patch("certbot_apache.configurator.util.run_script"):
-            with mock.patch("certbot_apache.configurator.util."
-                            "exe_exists") as mock_exe_exists:
-                mock_exe_exists.return_value = True
-                with mock.patch("certbot_apache.parser.ApacheParser."
-                                "update_runtime_variables"):
-                    try:
-                        config_class = entrypoint.OVERRIDE_CLASSES[os_info]
-                    except KeyError:
-                        config_class = configurator.ApacheConfigurator
-                    config = config_class(config=mock_le_config, name="apache",
-                        version=version)
-                    # This allows testing scripts to set it a bit more
-                    # quickly
-                    if conf is not None:
-                        config.conf = conf  # pragma: no cover
-
-                    config.prepare()
+    with mock.patch("certbot_apache.configurator.util.run_script"):
+        with mock.patch("certbot_apache.configurator.util."
+                        "exe_exists") as mock_exe_exists:
+            mock_exe_exists.return_value = True
+            with mock.patch("certbot_apache.parser.ApacheParser."
+                            "update_runtime_variables"):
+                try:
+                    config_class = entrypoint.OVERRIDE_CLASSES[os_info]
+                except KeyError:
+                    config_class = configurator.ApacheConfigurator
+                config = config_class(config=mock_le_config, name="apache",
+                    version=version)
+                if not conf_vhost_path:
+                    config_class.OS_DEFAULTS["vhost_root"] = vhost_path
+                else:
+                    # Custom virtualhost path was requested
+                    config.config.apache_vhost_root = conf_vhost_path
+                config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"]
+                config.prepare()
     return config
 
 
@@ -169,7 +158,7 @@ def get_vh_truth(temp_dir, config_name):
                 os.path.join(prefix, "certbot.conf"),
                 os.path.join(aug_pre, "certbot.conf/VirtualHost"),
                 set([obj.Addr.fromstring("*:80")]), False, True,
-                "certbot.demo"),
+                "certbot.demo", aliases=["www.certbot.demo"]),
             obj.VirtualHost(
                 os.path.join(prefix, "mod_macro-example.conf"),
                 os.path.join(aug_pre,

certbot-apache/certbot_apache/tls_sni_01.py

@@ -3,6 +3,7 @@
 import os
 import logging
 
+from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-module
 from certbot.plugins import common
 from certbot.errors import PluginError, MissingCommandlineFlag
 
@@ -93,7 +94,7 @@ class ApacheTlsSni01(common.TLSSNI01):
         :rtype: set
 
         """
-        addrs = set()
+        addrs = set()  # type: Set[obj.Addr]
         config_text = "<IfModule mod_ssl.c>\n"
 
         for achall in self.achalls:
@@ -123,7 +124,8 @@ class ApacheTlsSni01(common.TLSSNI01):
             self.configurator.config.tls_sni_01_port)))
 
         try:
-            vhost = self.configurator.choose_vhost(achall.domain, temp=True)
+            vhost = self.configurator.choose_vhost(achall.domain,
+                                                   create_if_no_ssl=False)
         except (PluginError, MissingCommandlineFlag):
             # We couldn't find the virtualhost for this domain, possibly
             # because it's a new vhost that's not configured yet

certbot-apache/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.25.0
+-e .[dev]

certbot-apache/setup.py

@@ -1,20 +1,17 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.25.0',
+    'certbot>=0.26.0',
     'mock',
     'python-augeas',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'zope.component',
     'zope.interface',
 ]
@@ -32,21 +29,21 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Plugins',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.20.0"
+LE_AUTO_VERSION="0.27.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -47,6 +47,7 @@ Help for certbot itself cannot be provided until it is installed.
   --no-bootstrap                            do not install OS dependencies
   --no-self-upgrade                         do not download updates
   --os-packages-only                        install OS dependencies and exit
+  --install-only                            install certbot, upgrade if needed, and exit
   -v, --verbose                             provide more output
   -q, --quiet                               provide only update/error output;
                                             implies --non-interactive
@@ -60,6 +61,8 @@ for arg in "$@" ; do
       DEBUG=1;;
     --os-packages-only)
       OS_PACKAGES_ONLY=1;;
+    --install-only)
+      INSTALL_ONLY=1;;
     --no-self-upgrade)
       # Do not upgrade this script (also prevents client upgrades, because each
       # copy of the script pins a hash of the python client)
@@ -68,10 +71,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +98,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
@@ -244,23 +249,42 @@ DeprecationBootstrap() {
   fi
 }
 
-
+MIN_PYTHON_VERSION="2.7"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
+  if [ "$USE_PYTHON_3" = 1 ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
+  if [ "$?" != "0" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -384,23 +408,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -408,15 +428,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -425,14 +445,20 @@ BootstrapRpmCommon() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -444,10 +470,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -455,9 +510,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -468,8 +522,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -477,16 +530,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -696,13 +764,8 @@ BootstrapMageiaCommon() {
 # Set Bootstrap to the function that installs OS dependencies on this system
 # and BOOTSTRAP_VERSION to the unique identifier for the current version of
 # that function. If Bootstrap is set to a function that doesn't install any
-# packages (either because --no-bootstrap was included on the command line or
-# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set.
-if [ "$NO_BOOTSTRAP" = 1 ]; then
-  Bootstrap() {
-    :
-  }
-elif [ -f /etc/debian_version ]; then
+# packages BOOTSTRAP_VERSION is not set.
+if [ -f /etc/debian_version ]; then
   Bootstrap() {
     BootstrapMessage "Debian-based OSes"
     BootstrapDebCommon
@@ -715,11 +778,27 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -782,6 +861,17 @@ else
   }
 fi
 
+# We handle this case after determining the normal bootstrap version to allow
+# variables like USE_PYTHON_3 to be properly set. As described above, if the
+# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not
+# be set so we unset it here.
+if [ "$NO_BOOTSTRAP" = 1 ]; then
+  Bootstrap() {
+    :
+  }
+  unset BOOTSTRAP_VERSION
+fi
+
 # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used
 # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set
 # if it is unknown how OS dependencies were installed on this system.
@@ -816,7 +906,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -824,14 +918,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -841,6 +947,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -858,10 +968,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -937,41 +1055,32 @@ cffi==1.10.0 \
     --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \
     --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
 ConfigArgParse==0.12.0 \
-    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339
+    --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \
+    --no-binary ConfigArgParse
 configobj==5.0.6 \
-    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902
-cryptography==2.0.2 \
-    --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \
-    --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \
-    --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \
-    --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \
-    --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \
-    --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \
-    --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \
-    --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \
-    --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \
-    --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \
-    --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \
-    --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \
-    --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \
-    --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \
-    --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \
-    --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \
-    --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \
-    --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \
-    --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \
-    --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \
-    --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \
-    --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \
-    --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \
-    --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \
-    --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \
-    --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \
-    --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \
-    --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \
-    --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \
-    --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab
-enum34==1.1.2 \
+    --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
+    --no-binary configobj
+cryptography==2.2.2 \
+    --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \
+    --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \
+    --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \
+    --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \
+    --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \
+    --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \
+    --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \
+    --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \
+    --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \
+    --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \
+    --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \
+    --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \
+    --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \
+    --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \
+    --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \
+    --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \
+    --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \
+    --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \
+    --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887
+enum34==1.1.2 ; python_version < '3.4' \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
 funcsigs==1.0.2 \
@@ -983,11 +1092,19 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+josepy==1.1.0 \
+    --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \
+    --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
+# Using an older version of mock here prevents regressions of #5276.
+mock==1.3.0 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
-    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f \
+    --no-binary ordereddict
 packaging==16.8 \
     --hash=sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388 \
     --hash=sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e
@@ -1013,7 +1130,8 @@ pyRFC3339==1.0 \
     --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \
     --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535
 python-augeas==0.5.0 \
-    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2
+    --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 \
+    --no-binary python-augeas
 pytz==2015.7 \
     --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \
     --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \
@@ -1041,9 +1159,11 @@ unittest2==1.1.0 \
     --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \
     --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579
 zope.component==4.2.2 \
-    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a
+    --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a \
+    --no-binary zope.component
 zope.event==4.1.0 \
-    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786
+    --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 \
+    --no-binary zope.event
 zope.interface==4.1.3 \
     --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \
     --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \
@@ -1062,10 +1182,9 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-# Using an older version of mock here prevents regressions of #5276.
-mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
+requests-toolbelt==0.8.0 \
+    --hash=sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237 \
+    --hash=sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1078,24 +1197,24 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.27.1 \
+    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
+    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
+acme==0.27.1 \
+    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
+    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
+certbot-apache==0.27.1 \
+    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
+    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
+certbot-nginx==0.27.1 \
+    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
+    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
     cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py"
 #!/usr/bin/env python
-"""A small script that can act as a trust root for installing pip 8
+"""A small script that can act as a trust root for installing pip >=8
 
 Embed this in your project, and your VCS checkout is all you have to trust. In
 a post-peep era, this lets you claw your way to a hash-checking version of pip,
@@ -1119,6 +1238,7 @@ anything goes wrong, it will exit with a non-zero status code.
 from __future__ import print_function
 from distutils.version import StrictVersion
 from hashlib import sha256
+from os import environ
 from os.path import join
 from pipes import quote
 from shutil import rmtree
@@ -1152,14 +1272,14 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 3, 0
+__version__ = 1, 5, 1
 PIP_VERSION = '9.0.1'
+DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
 
 # wheel has a conditional dependency on argparse:
 maybe_argparse = (
-    [('https://pypi.python.org/packages/18/dd/'
-      'e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
+    [('18/dd/e617cfc3f6210ae183374cd9f6a26b20514bbb5a792af97949c5aacddf0f/'
       'argparse-1.4.0.tar.gz',
       '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')]
     if version_info < (2, 7, 0) else [])
@@ -1167,18 +1287,14 @@ maybe_argparse = (
 
 PACKAGES = maybe_argparse + [
     # Pip has no dependencies, as it vendors everything:
-    ('https://pypi.python.org/packages/11/b6/'
-     'abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
-     'pip-{0}.tar.gz'
-     .format(PIP_VERSION),
+    ('11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/'
+     'pip-{0}.tar.gz'.format(PIP_VERSION),
      '09f243e1a7b461f654c26a725fa373211bb7ff17a9300058b205c61658ca940d'),
     # This version of setuptools has only optional dependencies:
-    ('https://pypi.python.org/packages/69/65/'
-     '4c544cde88d4d876cdf5cbc5f3f15d02646477756d89547e9a7ecd6afa76/'
-     'setuptools-20.2.2.tar.gz',
-     '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'),
-    ('https://pypi.python.org/packages/c9/1d/'
-     'bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
+    ('59/88/2f3990916931a5de6fa9706d6d75eb32ee8b78627bb2abaab7ed9e6d0622/'
+     'setuptools-29.0.1.tar.gz',
+     'b539118819a4857378398891fa5366e090690e46b3e41421a1e07d6e9fd8feb0'),
+    ('c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/'
      'wheel-0.29.0.tar.gz',
      '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648')
 ]
@@ -1199,12 +1315,13 @@ def hashed_download(url, temp, digest):
     # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
     # authenticity has only privacy (not arbitrary code execution)
     # implications, since we're checking hashes.
-    def opener():
+    def opener(using_https=True):
         opener = build_opener(HTTPSHandler())
-        # Strip out HTTPHandler to prevent MITM spoof:
-        for handler in opener.handlers:
-            if isinstance(handler, HTTPHandler):
-                opener.handlers.remove(handler)
+        if using_https:
+            # Strip out HTTPHandler to prevent MITM spoof:
+            for handler in opener.handlers:
+                if isinstance(handler, HTTPHandler):
+                    opener.handlers.remove(handler)
         return opener
 
     def read_chunks(response, chunk_size):
@@ -1214,8 +1331,9 @@ def hashed_download(url, temp, digest):
                 break
             yield chunk
 
-    response = opener().open(url)
-    path = join(temp, urlparse(url).path.split('/')[-1])
+    parsed_url = urlparse(url)
+    response = opener(using_https=parsed_url.scheme == 'https').open(url)
+    path = join(temp, parsed_url.path.split('/')[-1])
     actual_hash = sha256()
     with open(path, 'wb') as file:
         for chunk in read_chunks(response, 4096):
@@ -1228,6 +1346,24 @@ def hashed_download(url, temp, digest):
     return path
 
 
+def get_index_base():
+    """Return the URL to the dir containing the "packages" folder.
+
+    Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the
+    end if it's there; that is likely to give us the right dir.
+
+    """
+    env_var = environ.get('PIP_INDEX_URL', '').rstrip('/')
+    if env_var:
+        SIMPLE = '/simple'
+        if env_var.endswith(SIMPLE):
+            return env_var[:-len(SIMPLE)]
+        else:
+            return env_var
+    else:
+        return DEFAULT_INDEX_BASE
+
+
 def main():
     pip_version = StrictVersion(check_output(['pip', '--version'])
                                 .decode('utf-8').split()[1])
@@ -1235,11 +1371,13 @@ def main():
     if pip_version >= min_pip_version:
         return 0
     has_pip_cache = pip_version >= StrictVersion('6.0')
-
+    index_base = get_index_base()
     temp = mkdtemp(prefix='pipstrap-')
     try:
-        downloads = [hashed_download(url, temp, digest)
-                     for url, digest in PACKAGES]
+        downloads = [hashed_download(index_base + '/packages/' + path,
+                                     temp,
+                                     digest)
+                     for path, digest in PACKAGES]
         check_output('pip install --no-index --no-deps -U ' +
                      # Disable cache since we're not using it and it otherwise
                      # sometimes throws permission warnings:
@@ -1310,6 +1448,12 @@ UNLIKELY_EOF
 
     say "Installation succeeded."
   fi
+
+  if [ "$INSTALL_ONLY" = 1 ]; then
+    say "Certbot is installed."
+    exit 0
+  fi
+
   "$VENV_BIN/letsencrypt" "$@"
 
 else
@@ -1319,9 +1463,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1353,17 +1498,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1385,8 +1535,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1408,7 +1561,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1416,13 +1569,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1439,7 +1592,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1454,6 +1607,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def cert_none_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
@@ -1475,8 +1636,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."

certbot-compatibility-test/Dockerfile

@@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
 
 # all above files are necessary for setup.py, however, package source
 # code directory has to be copied separately to a subdirectory...

certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py

@@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy):
             setattr(self.le_config, "apache_" + k,
                     entrypoint.ENTRYPOINT.OS_DEFAULTS[k])
 
-        # An alias
-        self.le_config.apache_handle_modules = self.le_config.apache_handle_mods
-
         self._configurator = entrypoint.ENTRYPOINT(
             config=configuration.NamespaceConfig(self.le_config),
             name="apache")

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -10,9 +10,12 @@ import sys
 
 import OpenSSL
 
+from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
+
 from acme import challenges
 from acme import crypto_util
 from acme import messages
+from acme.magic_typing import List, Tuple  # pylint: disable=unused-import, no-name-in-module
 from certbot import achallenges
 from certbot import errors as le_errors
 from certbot.tests import acme_util
@@ -50,9 +53,8 @@ def test_authenticator(plugin, config, temp_dir):
 
     try:
         responses = plugin.perform(achalls)
-    except le_errors.Error as error:
-        logger.error("Performing challenges on %s caused an error:", config)
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("Performing challenges on %s caused an error:", config, exc_info=True)
         return False
 
     success = True
@@ -80,9 +82,8 @@ def test_authenticator(plugin, config, temp_dir):
     if success:
         try:
             plugin.cleanup(achalls)
-        except le_errors.Error as error:
-            logger.error("Challenge cleanup for %s caused an error:", config)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("Challenge cleanup for %s caused an error:", config, exc_info=True)
             success = False
 
         if _dirs_are_unequal(config, backup):
@@ -145,9 +146,8 @@ def test_deploy_cert(plugin, temp_dir, domains):
         try:
             plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path, cert_path)
             plugin.save()  # Needed by the Apache plugin
-        except le_errors.Error as error:
-            logger.error("**** Plugin failed to deploy certificate for %s:", domain)
-            logger.exception(error)
+        except le_errors.Error:
+            logger.error("**** Plugin failed to deploy certificate for %s:", domain, exc_info=True)
             return False
 
     if not _save_and_restart(plugin, "deployed"):
@@ -177,7 +177,7 @@ def test_enhancements(plugin, domains):
                      "enhancements")
         return False
 
-    domains_and_info = [(domain, []) for domain in domains]
+    domains_and_info = [(domain, []) for domain in domains]  # type: List[Tuple[str, List[bool]]]
 
     for domain, info in domains_and_info:
         try:
@@ -190,10 +190,9 @@ def test_enhancements(plugin, domains):
             # Don't immediately fail because a redirect may already be enabled
             logger.warning("*** Plugin failed to enable redirect for %s:", domain)
             logger.warning("%s", error)
-        except le_errors.Error as error:
+        except le_errors.Error:
             logger.error("*** An error occurred while enabling redirect for %s:",
-                         domain)
-            logger.exception(error)
+                         domain, exc_info=True)
 
     if not _save_and_restart(plugin, "enhanced"):
         return False
@@ -220,9 +219,8 @@ def _save_and_restart(plugin, title=None):
         plugin.save(title)
         plugin.restart()
         return True
-    except le_errors.Error as error:
-        logger.error("*** Plugin failed to save and restart server:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin failed to save and restart server:", exc_info=True)
         return False
 
 
@@ -230,9 +228,8 @@ def test_rollback(plugin, config, backup):
     """Tests the rollback checkpoints function"""
     try:
         plugin.rollback_checkpoints(1337)
-    except le_errors.Error as error:
-        logger.error("*** Plugin raised an exception during rollback:")
-        logger.exception(error)
+    except le_errors.Error:
+        logger.error("*** Plugin raised an exception during rollback:", exc_info=True)
         return False
 
     if _dirs_are_unequal(config, backup):
@@ -261,21 +258,21 @@ def _dirs_are_unequal(dir1, dir2):
             logger.error("The following files and directories are only "
                          "present in one directory")
             if dircmp.left_only:
-                logger.error(dircmp.left_only)
+                logger.error(str(dircmp.left_only))
             else:
-                logger.error(dircmp.right_only)
+                logger.error(str(dircmp.right_only))
             return True
         elif dircmp.common_funny or dircmp.funny_files:
             logger.error("The following files and directories could not be "
                          "compared:")
             if dircmp.common_funny:
-                logger.error(dircmp.common_funny)
+                logger.error(str(dircmp.common_funny))
             else:
-                logger.error(dircmp.funny_files)
+                logger.error(str(dircmp.funny_files))
             return True
         elif dircmp.diff_files:
             logger.error("The following files differ:")
-            logger.error(dircmp.diff_files)
+            logger.error(str(dircmp.diff_files))
             return True
 
         for subdir in dircmp.subdirs.itervalues():
@@ -352,9 +349,8 @@ def main():
                     success = test_authenticator(plugin, config, temp_dir)
                 if success and args.install:
                     success = test_installer(args, plugin, config, temp_dir)
-            except errors.Error as error:
-                logger.error("Tests on %s raised:", config)
-                logger.exception(error)
+            except errors.Error:
+                logger.error("Tests on %s raised:", config, exc_info=True)
                 success = False
 
             if success:

certbot-compatibility-test/certbot_compatibility_test/util.py

@@ -26,12 +26,12 @@ def create_le_config(parent_dir):
     config = copy.deepcopy(constants.CLI_DEFAULTS)
 
     le_dir = os.path.join(parent_dir, "certbot")
-    config["config_dir"] = os.path.join(le_dir, "config")
-    config["work_dir"] = os.path.join(le_dir, "work")
-    config["logs_dir"] = os.path.join(le_dir, "logs_dir")
-    os.makedirs(config["config_dir"])
-    os.mkdir(config["work_dir"])
-    os.mkdir(config["logs_dir"])
+    os.mkdir(le_dir)
+    for dir_name in ("config", "logs", "work"):
+        full_path = os.path.join(le_dir, dir_name)
+        os.mkdir(full_path)
+        full_name = dir_name + "_dir"
+        config[full_name] = full_path
 
     config["domains"] = None
 

certbot-compatibility-test/certbot_compatibility_test/validator.py

@@ -5,6 +5,7 @@ import requests
 import zope.interface
 
 import six
+from six.moves import xrange  # pylint: disable=import-error,redefined-builtin
 
 from acme import crypto_util
 from acme import errors as acme_errors
@@ -32,7 +33,7 @@ class Validator(object):
         try:
             presented_cert = crypto_util.probe_sni(name, host, port)
         except acme_errors.Error as error:
-            logger.exception(error)
+            logger.exception(str(error))
             return False
 
         return presented_cert.digest("sha256") == cert.digest("sha256")
@@ -85,8 +86,7 @@ class Validator(object):
             return False
 
         try:
-            _, max_age_value = max_age[0]
-            max_age_value = int(max_age_value)
+            max_age_value = int(max_age[0][1])
         except ValueError:
             logger.error("Server responded with invalid HSTS header field")
             return False

certbot-compatibility-test/nginx/roundtrip.py

@@ -8,7 +8,7 @@ from certbot_nginx import nginxparser
 def roundtrip(stuff):
     success = True
     for t in stuff:
-        print t
+        print(t)
         if not os.path.isfile(t):
             continue
         with open(t, "r") as f:

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
 install_requires = [
     'certbot',
@@ -34,19 +34,19 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
     ],

certbot-dns-cloudflare/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudflare
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare

certbot-dns-cloudflare/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudflare's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_cloudflare
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_cloudflare
-   :members:
-
 
 Indices and tables
 ==================

certbot-dns-cloudflare/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudflare/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-cloudflare[docs]

certbot-dns-cloudflare/setup.py

@@ -1,20 +1,17 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'cloudflare>=1.5.1',
     'mock',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'zope.interface',
 ]
 
@@ -31,6 +28,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',
@@ -39,13 +37,12 @@ setup(
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-cloudxns/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-cloudxns
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns

certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py

@@ -70,6 +70,7 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient):
         super(_CloudXNSLexiconClient, self).__init__()
 
         self.provider = cloudxns.Provider({
+            'provider_name': 'cloudxns',
             'auth_username': api_key,
             'auth_token': secret_key,
             'ttl': ttl,

certbot-dns-cloudxns/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-cloudxns's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_cloudxns
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_cloudxns
-   :members:
-
 
 
 Indices and tables

certbot-dns-cloudxns/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-cloudxns/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-cloudxns[docs]

certbot-dns-cloudxns/setup.py

@@ -1,20 +1,17 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'zope.interface',
 ]
 
@@ -31,6 +28,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',
@@ -41,10 +39,10 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-digitalocean/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-digitalocean
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean

certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py

@@ -50,7 +50,8 @@ class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthentic
 
 
 class DigitalOceanClientTest(unittest.TestCase):
-    id = 1
+
+    id_num = 1
     record_prefix = "_acme-challenge"
     record_name = record_prefix + "." + DOMAIN
     record_content = "bar"
@@ -70,7 +71,7 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         domain_mock = mock.MagicMock()
         domain_mock.name = DOMAIN
-        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}}
+        domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id_num}}
 
         self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock]
 

certbot-dns-digitalocean/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-digitalocean's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_digitalocean
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_digitalocean
-   :members:
-
 
 
 Indices and tables

certbot-dns-digitalocean/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-digitalocean/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-digitalocean[docs]

certbot-dns-digitalocean/setup.py

@@ -1,20 +1,17 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
     'mock',
     'python-digitalocean>=1.11',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'six',
     'zope.interface',
 ]
@@ -32,6 +29,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',
@@ -40,13 +38,12 @@ setup(
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsimple/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsimple
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple

certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py

@@ -66,6 +66,7 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient):
         super(_DNSimpleLexiconClient, self).__init__()
 
         self.provider = dnsimple.Provider({
+            'provider_name': 'dnssimple',
             'auth_token': token,
             'ttl': ttl,
         })

certbot-dns-dnsimple/docs/index.rst

@@ -10,14 +10,14 @@ Welcome to certbot-dns-dnsimple's documentation!
    :maxdepth: 2
    :caption: Contents:
 
+.. automodule:: certbot_dns_dnsimple
+   :members:
+
 .. toctree::
    :maxdepth: 1
 
    api
 
-.. automodule:: certbot_dns_dnsimple
-   :members:
-
 
 
 Indices and tables

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+acme[dev]==0.21.1
+certbot[dev]==0.21.1

certbot-dns-dnsimple/readthedocs.org.requirements.txt

@@ -0,0 +1,12 @@
+# readthedocs.org gives no way to change the install command to "pip
+# install -e .[docs]" (that would in turn install documentation
+# dependencies), but it allows to specify a requirements.txt file at
+# https://readthedocs.org/dashboard/letsencrypt/advanced/ (c.f. #259)
+
+# Although ReadTheDocs certainly doesn't need to install the project
+# in --editable mode (-e), just "pip install .[docs]" does not work as
+# expected and "pip install -e .[docs]" must be used instead
+
+-e acme
+-e .
+-e certbot-dns-dnsimple[docs]

certbot-dns-dnsimple/setup.py

@@ -1,20 +1,17 @@
-import sys
-
 from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.28.0.dev0'
 
-# Please update tox.ini when modifying dependency version requirements
+# Remember to update local-oldest-requirements.txt when changing the minimum
+# acme/certbot version.
 install_requires = [
-    'acme=={0}'.format(version),
-    'certbot=={0}'.format(version),
-    'dns-lexicon',
+    'acme>=0.21.1',
+    'certbot>=0.21.1',
+    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
     'mock',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'zope.interface',
 ]
 
@@ -31,6 +28,7 @@ setup(
     author="Certbot Project",
     author_email='client-dev@letsencrypt.org',
     license='Apache License 2.0',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Plugins',
@@ -41,10 +39,10 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Security',
         'Topic :: System :: Installation/Setup',

certbot-dns-dnsmadeeasy/Dockerfile

@@ -0,0 +1,5 @@
+FROM certbot/certbot
+
+COPY . src/certbot-dns-dnsmadeeasy
+
+RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy