From 18f6deada8dca329c1a797bcf5b88dbdcbd18cf7 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Fri, 5 Jan 2018 19:27:00 -0800
Subject: [PATCH 01/45] Fix letsencrypt-auto name and long forms of -n (#5375)

---
 letsencrypt-auto-source/letsencrypt-auto          | 8 +++++---
 letsencrypt-auto-source/letsencrypt-auto.template | 8 +++++---
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 93e3e7b83..46cb51822 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 4eef10c80..861ef0a6e 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 

From 8585cdd86190c5da753cae1a691e66e586fd2bde Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Mon, 8 Jan 2018 13:57:04 -0800
Subject: [PATCH 02/45] Deprecate Python2.6 by using Python3 on CentOS/RHEL 6
 (#5329)

* If there's no python or there's only python2.6 on red hat systems, install python3

* Always check for python2.6

* address style, documentation, nits

* factor out all initialization code

* fix up python version return value when no python installed

* add no python error and exit

* document DeterminePythonVersion parameters

* build letsencrypt-auto

* close brace

* build leauto

* fix syntax errors

* set USE_PYTHON_3 for all cases

* rip out NOCRASH

* replace NOCRASH, update LE_PYTHON set logic

* use built-in venv for py3

* switch to LE_PYTHON not affecting bootstrap selection and not overwriting LE_PYTHON

* python3ify fetch.py

* get fetch.py working with python2 and 3

* don't verify server certificates in fetch.py HttpsGetter

* Use SSLContext and an environment variable so that our tests continue to never verify server certificates.

* typo

* build

* remove commented out code

* address review comments

* add documentation for YES_FLAG and QUIET_FLAG

* Add tests to centos6 Dockerfile to make sure we install python3 if and only if appropriate to do so.
---
 letsencrypt-auto-source/Dockerfile.centos6    |   3 +-
 letsencrypt-auto-source/letsencrypt-auto      | 205 +++++++++++++-----
 .../letsencrypt-auto.template                 |  67 ++++--
 .../pieces/bootstrappers/rpm_common.sh        |  75 +------
 .../pieces/bootstrappers/rpm_common_base.sh   |  78 +++++++
 .../pieces/bootstrappers/rpm_python3.sh       |  23 ++
 letsencrypt-auto-source/pieces/fetch.py       |  34 ++-
 letsencrypt-auto-source/tests/auto_test.py    |   2 +
 .../tests/centos6_tests.sh                    |  65 ++++++
 9 files changed, 405 insertions(+), 147 deletions(-)
 create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
 create mode 100644 letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh
 create mode 100644 letsencrypt-auto-source/tests/centos6_tests.sh

diff --git a/letsencrypt-auto-source/Dockerfile.centos6 b/letsencrypt-auto-source/Dockerfile.centos6
index 8c1a4b353..47eb48f50 100644
--- a/letsencrypt-auto-source/Dockerfile.centos6
+++ b/letsencrypt-auto-source/Dockerfile.centos6
@@ -33,4 +33,5 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+RUN sudo chmod +x certbot/letsencrypt-auto-source/tests/centos6_tests.sh
+CMD sudo certbot/letsencrypt-auto-source/tests/centos6_tests.sh
diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 46cb51822..f1361d8ea 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -246,15 +246,29 @@ DeprecationBootstrap() {
   fi
 }
 
-
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  if [ -n "$USE_PYTHON_3" ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
   if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
   export LE_PYTHON
 
@@ -386,23 +400,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -410,15 +420,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -430,11 +440,17 @@ BootstrapRpmCommon() {
       /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -446,10 +462,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -457,9 +502,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -470,8 +514,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -479,16 +522,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -717,11 +775,24 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  export LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -860,10 +931,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -1358,17 +1437,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1390,8 +1474,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1413,7 +1500,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1421,13 +1508,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in iter(metadata['releases'].keys())
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1444,7 +1531,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1459,6 +1546,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def create_CERT_NONE_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 861ef0a6e..f4c1b202f 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -246,15 +246,29 @@ DeprecationBootstrap() {
   fi
 }
 
-
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  if [ -n "$USE_PYTHON_3" ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
   if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
   export LE_PYTHON
 
@@ -267,7 +281,9 @@ DeterminePythonVersion() {
 }
 
 {{ bootstrappers/deb_common.sh }}
+{{ bootstrappers/rpm_common_base.sh }}
 {{ bootstrappers/rpm_common.sh }}
+{{ bootstrappers/rpm_python3.sh }}
 {{ bootstrappers/suse_common.sh }}
 {{ bootstrappers/arch_common.sh }}
 {{ bootstrappers/gentoo_common.sh }}
@@ -298,11 +314,24 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  export LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -441,10 +470,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh
index 5b120a9e6..80d55a393 100755
--- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh
+++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh
@@ -7,61 +7,13 @@ BootstrapRpmCommon() {
   #   - Fedora 20, 21, 22, 23 (x64)
   #   - Centos 7 (x64: on DigitalOcean droplet)
   #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+  #   - CentOS 6
 
-  if type dnf 2>/dev/null
-  then
-    tool=dnf
-  elif type yum 2>/dev/null
-  then
-    tool=yum
-
-  else
-    error "Neither yum nor dnf found. Aborting bootstrap!"
-    exit 1
-  fi
-
-  if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
-  fi
-  if [ "$QUIET" = 1 ]; then
-    QUIET_FLAG='--quiet'
-  fi
-
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
-    echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
-      error "Enable the EPEL repository and try running Certbot again."
-      exit 1
-    fi
-    if [ "$ASSUME_YES" = 1 ]; then
-      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
-      sleep 1s
-      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
-      sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
-      sleep 1s
-    fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
-      error "Could not enable EPEL. Aborting bootstrap!"
-      exit 1
-    fi
-  fi
-
-  pkgs="
-    gcc
-    augeas-libs
-    openssl
-    openssl-devel
-    libffi-devel
-    redhat-rpm-config
-    ca-certificates
-  "
+  InitializeRPMCommonBase
 
   # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -69,9 +21,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -82,8 +33,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -91,14 +41,5 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
-
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
-    exit 1
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
 }
diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
new file mode 100644
index 000000000..d7a9f3133
--- /dev/null
+++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
@@ -0,0 +1,78 @@
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
+
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
+  if type dnf 2>/dev/null
+  then
+    TOOL=dnf
+  elif type yum 2>/dev/null
+  then
+    TOOL=yum
+
+  else
+    error "Neither yum nor dnf found. Aborting bootstrap!"
+    exit 1
+  fi
+
+  if [ "$ASSUME_YES" = 1 ]; then
+    YES_FLAG="-y"
+  fi
+  if [ "$QUIET" = 1 ]; then
+    QUIET_FLAG='--quiet'
+  fi
+
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
+    echo "To use Certbot, packages from the EPEL repository need to be installed."
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
+      error "Enable the EPEL repository and try running Certbot again."
+      exit 1
+    fi
+    if [ "$ASSUME_YES" = 1 ]; then
+      /bin/echo -n "Enabling the EPEL repository in 3 seconds..."
+      sleep 1s
+      /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
+      sleep 1s
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      sleep 1s
+    fi
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
+      error "Could not enable EPEL. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
+
+  pkgs="
+    gcc
+    augeas-libs
+    openssl
+    openssl-devel
+    libffi-devel
+    redhat-rpm-config
+    ca-certificates
+  "
+
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
+    pkgs="$pkgs
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh
new file mode 100644
index 000000000..b011a7235
--- /dev/null
+++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_python3.sh
@@ -0,0 +1,23 @@
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
+    exit 1
+  fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
+}
diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py
index 8f34351c9..ae72a299b 100644
--- a/letsencrypt-auto-source/pieces/fetch.py
+++ b/letsencrypt-auto-source/pieces/fetch.py
@@ -11,17 +11,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -43,8 +48,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -66,7 +74,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -74,13 +82,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in iter(metadata['releases'].keys())
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -97,7 +105,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -112,6 +120,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def create_CERT_NONE_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py
index 156466c82..d187452a1 100644
--- a/letsencrypt-auto-source/tests/auto_test.py
+++ b/letsencrypt-auto-source/tests/auto_test.py
@@ -202,6 +202,7 @@ LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47
 Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68
 iQIDAQAB
 -----END PUBLIC KEY-----""",
+             NO_CERT_VERIFY='1',
              **kwargs)
     env.update(d)
     return out_and_err(
@@ -349,6 +350,7 @@ class AutoTests(TestCase):
                     self.assertTrue("Couldn't verify signature of downloaded "
                                     "certbot-auto." in exc.output)
                 else:
+                    print(out)
                     self.fail('Signature check on certbot-auto erroneously passed.')
 
     def test_pip_failure(self):
diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh
new file mode 100644
index 000000000..e3ebbaec5
--- /dev/null
+++ b/letsencrypt-auto-source/tests/centos6_tests.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Start by making sure your system is up-to-date:
+yum update > /dev/null
+yum install -y centos-release-scl > /dev/null
+yum install -y python27 > /dev/null 2> /dev/null
+
+# we're going to modify env variables, so do this in a subshell
+(
+source /opt/rh/python27/enable
+
+# ensure python 3 isn't installed
+python3 --version 2> /dev/null
+RESULT=$?
+if [ $RESULT -eq 0 ]; then
+  error "Python3 is already installed."
+  exit 1
+fi
+
+# ensure python2.7 is available
+python2.7 --version 2> /dev/null
+RESULT=$?
+if [ $RESULT -ne 0 ]; then
+  error "Python3 is not available."
+  exit 1
+fi
+
+# bootstrap, but don't install python 3.
+certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null
+
+# ensure python 3 isn't installed
+python3 --version 2> /dev/null
+RESULT=$?
+if [ $RESULT -eq 0 ]; then
+  error "letsencrypt-auto installed Python3 even though Python2.7 is present."
+  exit 1
+fi
+
+echo ""
+echo "PASSED: Did not upgrade to Python3 when Python2.7 is present."
+)
+
+# ensure python2.7 isn't available
+python2.7 --version 2> /dev/null
+RESULT=$?
+if [ $RESULT -eq 0 ]; then
+  error "Python2.7 is still available."
+  exit 1
+fi
+
+# bootstrap, this time installing python3
+certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null
+
+# ensure python 3 is installed
+python3 --version > /dev/null
+RESULT=$?
+if [ $RESULT -ne 0 ]; then
+  error "letsencrypt-auto failed to install Python3 when only Python2.6 is present."
+  exit 1
+fi
+
+echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present."
+echo ""
+
+# test using python3
+pytest -v -s certbot/letsencrypt-auto-source/tests

From 24ddc65cd4765e82067ab5fd963ffe81348e1f02 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Mon, 8 Jan 2018 17:02:20 -0800
Subject: [PATCH 03/45] Allow non-interactive revocation without deleting
 certificates (#5386)

* Add --delete-after-revoke flags

* Use delete_after_revoke value

* Add delete_after_revoke unit tests

* Add integration tests for delete-after-revoke.
---
 certbot/cli.py               | 12 ++++++++++
 certbot/constants.py         |  1 +
 certbot/main.py              |  8 ++++---
 certbot/tests/cli_test.py    | 14 +++++++++++
 certbot/tests/main_test.py   | 46 ++++++++++++++++++++++++++++--------
 tests/boulder-integration.sh |  9 +++++--
 6 files changed, 75 insertions(+), 15 deletions(-)

diff --git a/certbot/cli.py b/certbot/cli.py
index 622462278..f0fa7eb7e 100644
--- a/certbot/cli.py
+++ b/certbot/cli.py
@@ -1220,6 +1220,18 @@ def _create_subparsers(helpful):
                                                    key=constants.REVOCATION_REASONS.get)),
                 action=_EncodeReasonAction, default=flag_default("reason"),
                 help="Specify reason for revoking certificate. (default: unspecified)")
+    helpful.add("revoke",
+                "--delete-after-revoke", action="store_true",
+                default=flag_default("delete_after_revoke"),
+                help="Delete certificates after revoking them.")
+    helpful.add("revoke",
+                "--no-delete-after-revoke", action="store_false",
+                dest="delete_after_revoke",
+                default=flag_default("delete_after_revoke"),
+                help="Do not delete certificates after revoking them. This "
+                     "option should be used with caution because the 'renew' "
+                     "subcommand will attempt to renew undeleted revoked "
+                     "certificates.")
     helpful.add("rollback",
                 "--checkpoints", type=int, metavar="N",
                 default=flag_default("rollback_checkpoints"),
diff --git a/certbot/constants.py b/certbot/constants.py
index 0ac82dafe..a6878824b 100644
--- a/certbot/constants.py
+++ b/certbot/constants.py
@@ -71,6 +71,7 @@ CLI_DEFAULTS = dict(
     user_agent_comment=None,
     csr=None,
     reason=0,
+    delete_after_revoke=None,
     rollback_checkpoints=1,
     init=False,
     prepare=False,
diff --git a/certbot/main.py b/certbot/main.py
index 1c6432fd9..e25e030aa 100644
--- a/certbot/main.py
+++ b/certbot/main.py
@@ -536,9 +536,11 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b
     display = zope.component.getUtility(interfaces.IDisplay)
     reporter_util = zope.component.getUtility(interfaces.IReporter)
 
-    msg = ("Would you like to delete the cert(s) you just revoked?")
-    attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No",
-            force_interactive=True, default=True)
+    attempt_deletion = config.delete_after_revoke
+    if attempt_deletion is None:
+        msg = ("Would you like to delete the cert(s) you just revoked?")
+        attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No",
+                force_interactive=True, default=True)
 
     if not attempt_deletion:
         reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY)
diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py
index 2fce412e2..c5935d722 100644
--- a/certbot/tests/cli_test.py
+++ b/certbot/tests/cli_test.py
@@ -164,6 +164,8 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         self.assertTrue("--cert-path" in out)
         self.assertTrue("--key-path" in out)
         self.assertTrue("--reason" in out)
+        self.assertTrue("--delete-after-revoke" in out)
+        self.assertTrue("--no-delete-after-revoke" in out)
 
         out = self._help_output(['-h', 'config_changes'])
         self.assertTrue("--cert-path" not in out)
@@ -412,6 +414,18 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
     def test_no_directory_hooks_unset(self):
         self.assertTrue(self.parse([]).directory_hooks)
 
+    def test_delete_after_revoke(self):
+        namespace = self.parse(["--delete-after-revoke"])
+        self.assertTrue(namespace.delete_after_revoke)
+
+    def test_delete_after_revoke_default(self):
+        namespace = self.parse([])
+        self.assertEqual(namespace.delete_after_revoke, None)
+
+    def test_no_delete_after_revoke(self):
+        namespace = self.parse(["--no-delete-after-revoke"])
+        self.assertFalse(namespace.delete_after_revoke)
+
 
 class DefaultTest(unittest.TestCase):
     """Tests for certbot.cli._Default."""
diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py
index 04b71dcc7..b1d58542f 100644
--- a/certbot/tests/main_test.py
+++ b/certbot/tests/main_test.py
@@ -298,25 +298,29 @@ class RevokeTest(test_util.TempDirTestCase):
         self._call()
         self.assertFalse(mock_delete.called)
 
-class DeleteIfAppropriateTest(unittest.TestCase):
+class DeleteIfAppropriateTest(test_util.ConfigTestCase):
     """Tests for certbot.main._delete_if_appropriate """
 
-    def setUp(self):
-        self.config = mock.Mock()
-        self.config.namespace = mock.Mock()
-        self.config.namespace.noninteractive_mode = False
-
     def _call(self, mock_config):
         from certbot.main import _delete_if_appropriate
         _delete_if_appropriate(mock_config)
 
-    @mock.patch('certbot.cert_manager.delete')
+    def _test_delete_opt_out_common(self, mock_get_utility):
+        with mock.patch('certbot.cert_manager.delete') as mock_delete:
+            self._call(self.config)
+        mock_delete.assert_not_called()
+        self.assertTrue(mock_get_utility().add_message.called)
+
     @test_util.patch_get_utility()
-    def test_delete_opt_out(self, mock_get_utility, mock_delete):
+    def test_delete_flag_opt_out(self, mock_get_utility):
+        self.config.delete_after_revoke = False
+        self._test_delete_opt_out_common(mock_get_utility)
+
+    @test_util.patch_get_utility()
+    def test_delete_prompt_opt_out(self, mock_get_utility):
         util_mock = mock_get_utility()
         util_mock.yesno.return_value = False
-        self._call(self.config)
-        mock_delete.assert_not_called()
+        self._test_delete_opt_out_common(mock_get_utility)
 
     # pylint: disable=too-many-arguments
     @mock.patch('certbot.storage.renewal_file_for_certname')
@@ -397,6 +401,28 @@ class DeleteIfAppropriateTest(unittest.TestCase):
         self._call(config)
         self.assertEqual(mock_delete.call_count, 1)
 
+    # pylint: disable=too-many-arguments
+    @mock.patch('certbot.storage.renewal_file_for_certname')
+    @mock.patch('certbot.cert_manager.match_and_check_overlaps')
+    @mock.patch('certbot.storage.full_archive_path')
+    @mock.patch('certbot.cert_manager.cert_path_to_lineage')
+    @mock.patch('certbot.cert_manager.delete')
+    @test_util.patch_get_utility()
+    def test_opt_in_deletion(self, mock_get_utility, mock_delete,
+            mock_cert_path_to_lineage, mock_full_archive_dir,
+            mock_match_and_check_overlaps, mock_renewal_file_for_certname):
+        # pylint: disable = unused-argument
+        config = self.config
+        config.namespace.delete_after_revoke = True
+        config.cert_path = "/some/reasonable/path"
+        config.certname = ""
+        mock_cert_path_to_lineage.return_value = "example.com"
+        mock_full_archive_dir.return_value = ""
+        mock_match_and_check_overlaps.return_value = ""
+        self._call(config)
+        self.assertEqual(mock_delete.call_count, 1)
+        self.assertFalse(mock_get_utility().yesno.called)
+
     # pylint: disable=too-many-arguments
     @mock.patch('certbot.storage.renewal_file_for_certname')
     @mock.patch('certbot.cert_manager.match_and_check_overlaps')
diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh
index 1e0b7754b..e1aad4336 100755
--- a/tests/boulder-integration.sh
+++ b/tests/boulder-integration.sh
@@ -345,9 +345,14 @@ common auth --must-staple --domains "must-staple.le.wtf"
 openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24'
 
 # revoke by account key
-common revoke --cert-path "$root/conf/live/le.wtf/cert.pem"
+common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" --delete-after-revoke
 # revoke renewed
-common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem"
+common revoke --cert-path "$root/conf/live/le1.wtf/cert.pem" --no-delete-after-revoke
+if [ ! -d "$root/conf/live/le1.wtf" ]; then
+    echo "cert deleted when --no-delete-after-revoke was used!"
+    exit 1
+fi
+common delete --cert-name le1.wtf
 # revoke by cert key
 common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \
     --key-path "$root/conf/live/le2.wtf/privkey.pem"

From e02adec26b9018a3c7fae40fdc13086cd336b05c Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Mon, 8 Jan 2018 17:38:03 -0800
Subject: [PATCH 04/45] Have letsencrypt-auto do a real upgrade in
 leauto-upgrades option 2 (#5390)

* Make leauto_upgrades do a real upgrade

* Cleanup vars and output

* Sleep until the server is ready

* add simple_http_server.py

* Use a randomly assigned port

* s/realpath/readlink

* wait for server before getting port

* s/localhost/all interfaces
---
 .../letstest/scripts/test_leauto_upgrades.sh  | 47 +++++++++++++++++--
 tools/simple_http_server.py                   | 26 ++++++++++
 2 files changed, 68 insertions(+), 5 deletions(-)
 create mode 100755 tools/simple_http_server.py

diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index cb659786e..a83cbd826 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -15,19 +15,56 @@ if ! command -v git ; then
         exit 1
     fi
 fi
-BRANCH=`git rev-parse --abbrev-ref HEAD`
 # 0.5.0 is the oldest version of letsencrypt-auto that can be used because it's
 # the first version that pins package versions, properly supports
 # --no-self-upgrade, and works with newer versions of pip.
-git checkout -f v0.5.0
+git checkout -f v0.5.0 letsencrypt-auto
 if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then
     echo initial installation appeared to fail
     exit 1
 fi
 
-git checkout -f "$BRANCH"
-EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2)
-if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep $EXPECTED_VERSION ; then
+# Now that python and openssl have been installed, we can set up a fake server
+# to provide a new version of letsencrypt-auto. First, we start the server and
+# directory to be served.
+MY_TEMP_DIR=$(mktemp -d)
+PORT_FILE="$MY_TEMP_DIR/port"
+SERVER_PATH=$(tools/readlink.py tools/simple_http_server.py)
+cd "$MY_TEMP_DIR"
+"$SERVER_PATH" 0 > $PORT_FILE &
+SERVER_PID=$!
+trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT
+cd ~-
+
+# Then, we set up the files to be served.
+FAKE_VERSION_NUM="99.99.99"
+echo "{\"releases\": {\"$FAKE_VERSION_NUM\": null}}" > "$MY_TEMP_DIR/json"
+LE_AUTO_SOURCE_DIR="$MY_TEMP_DIR/v$FAKE_VERSION_NUM"
+NEW_LE_AUTO_PATH="$LE_AUTO_SOURCE_DIR/letsencrypt-auto"
+mkdir "$LE_AUTO_SOURCE_DIR"
+cp letsencrypt-auto-source/letsencrypt-auto "$LE_AUTO_SOURCE_DIR/letsencrypt-auto"
+SIGNING_KEY="letsencrypt-auto-source/tests/signing.key"
+openssl dgst -sha256 -sign "$SIGNING_KEY" -out "$NEW_LE_AUTO_PATH.sig" "$NEW_LE_AUTO_PATH"
+
+# Next, we wait for the server to start and get the port number.
+sleep 5s
+SERVER_PORT=$(sed -n 's/.*port \([0-9]\+\).*/\1/p' "$PORT_FILE")
+
+# Finally, we set the necessary certbot-auto environment variables.
+export LE_AUTO_DIR_TEMPLATE="http://localhost:$SERVER_PORT/%s/"
+export LE_AUTO_JSON_URL="http://localhost:$SERVER_PORT/json"
+export LE_AUTO_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsMoSzLYQ7E1sdSOkwelg
+tzKIh2qi3bpXuYtcfFC0XrvWig071NwIj+dZiT0OLZ2hPispEH0B7ISuuWg1ll7G
+hFW0VdbxL6JdGzS2ShNWkX9hE9z+j8VqwDPOBn3ZHm03qwpYkBDwQib3KqOdYbTT
+uUtJmmGcuk3a9Aq/sCT6DdfmTSdP5asdQYwIcaQreDrOosaS84DTWI3IU+UYJVgl
+LsIVPBuy9IcgHidUQ96hJnoPsDCWsHwX62495QKEarauyKQrJzFes0EY95orDM47
+Z5o/NDiQB11m91yNB0MmPYY9QSbnOA9j7IaaC97AwRLuwXY+/R2ablTcxurWou68
+iQIDAQAB
+-----END PUBLIC KEY-----
+"
+
+if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     echo upgrade appeared to fail
     exit 1
 fi
diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py
new file mode 100755
index 000000000..26bf231b7
--- /dev/null
+++ b/tools/simple_http_server.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""A version of Python 2.x's SimpleHTTPServer that flushes its output."""
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+import sys
+
+def serve_forever(port=0):
+    """Spins up an HTTP server on all interfaces and the given port.
+
+    A message is printed to stdout specifying the address and port being used
+    by the server.
+
+    :param int port: port number to use.
+
+    """
+    server = HTTPServer(('', port), SimpleHTTPRequestHandler)
+    print 'Serving HTTP on {0} port {1} ...'.format(*server.server_address)
+    sys.stdout.flush()
+    server.serve_forever()
+
+
+if __name__ == '__main__':
+    kwargs = {}
+    if len(sys.argv) > 1:
+        kwargs['port'] = int(sys.argv[1])
+    serve_forever(**kwargs)

From d557475bb6f921b7bdb0459d77a162d673044dce Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Tue, 9 Jan 2018 07:46:21 -0800
Subject: [PATCH 05/45] update Apache ciphersuites (#5383)

---
 certbot-apache/certbot_apache/centos-options-ssl-apache.conf | 2 +-
 certbot-apache/certbot_apache/constants.py                   | 2 ++
 certbot-apache/certbot_apache/options-ssl-apache.conf        | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf
index 17ae1be76..56c946a4e 100644
--- a/certbot-apache/certbot_apache/centos-options-ssl-apache.conf
+++ b/certbot-apache/certbot_apache/centos-options-ssl-apache.conf
@@ -8,7 +8,7 @@ SSLEngine on
 
 # Intermediate configuration, tweak to your needs
 SSLProtocol             all -SSLv2 -SSLv3
-SSLCipherSuite          ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
 
 SSLOptions +StrictRequire
diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py
index a13ca04a6..fd6a9eb11 100644
--- a/certbot-apache/certbot_apache/constants.py
+++ b/certbot-apache/certbot_apache/constants.py
@@ -16,6 +16,8 @@ ALL_SSL_OPTIONS_HASHES = [
     '4066b90268c03c9ba0201068eaa39abbc02acf9558bb45a788b630eb85dadf27',
     'f175e2e7c673bd88d0aff8220735f385f916142c44aa83b09f1df88dd4767a88',
     'cfdd7c18d2025836ea3307399f509cfb1ebf2612c87dd600a65da2a8e2f2797b',
+    '80720bd171ccdc2e6b917ded340defae66919e4624962396b992b7218a561791',
+    'c0c022ea6b8a51ecc8f1003d0a04af6c3f2bc1c3ce506b3c2dfc1f11ef931082',
 ]
 """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC"""
 
diff --git a/certbot-apache/certbot_apache/options-ssl-apache.conf b/certbot-apache/certbot_apache/options-ssl-apache.conf
index 950a02a8b..8113ee81e 100644
--- a/certbot-apache/certbot_apache/options-ssl-apache.conf
+++ b/certbot-apache/certbot_apache/options-ssl-apache.conf
@@ -8,7 +8,7 @@ SSLEngine on
 
 # Intermediate configuration, tweak to your needs
 SSLProtocol             all -SSLv2 -SSLv3
-SSLCipherSuite          ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+SSLCipherSuite          ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
 SSLHonorCipherOrder     on
 SSLCompression          off
 

From 62ffcf53738760f901d8ad3b31bff3855d1648c6 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joohoi@users.noreply.github.com>
Date: Tue, 9 Jan 2018 17:48:05 +0200
Subject: [PATCH 06/45] Fix macOS builds for Python2.7 in Travis (#5378)

* Add OSX Python2 tests

* Make sure python2 is originating from homebrew on macOS

* Upgrade the already installed python2 instead of trying to reinstall
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3d41bfa4b..35666d8e6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ cache:
         - $HOME/.cache/pip
 
 before_install:
-  - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3)'
+  - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)'
 
 before_script:
   - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi'

From 288c4d956cf59015590bc24e045138335b46a40d Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joohoi@users.noreply.github.com>
Date: Tue, 9 Jan 2018 18:28:52 +0200
Subject: [PATCH 07/45] Automatically install updates in test script (#5394)

---
 letsencrypt-auto-source/tests/centos6_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh
index e3ebbaec5..23b1e16e8 100644
--- a/letsencrypt-auto-source/tests/centos6_tests.sh
+++ b/letsencrypt-auto-source/tests/centos6_tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 # Start by making sure your system is up-to-date:
-yum update > /dev/null
+yum update -y > /dev/null
 yum install -y centos-release-scl > /dev/null
 yum install -y python27 > /dev/null 2> /dev/null
 

From 887a6bcfce73f7b3c556720ccbbc470bf8855606 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Tue, 9 Jan 2018 15:40:26 -0800
Subject: [PATCH 08/45] Handle need to rebootstrap before fetch.py (#5389)

* Fix #5387

* Add test for #5387

* remove LE_PYTHON

* Use environment variable to reduce line length
---
 letsencrypt-auto-source/letsencrypt-auto      | 20 +++++++++++++------
 .../letsencrypt-auto.template                 | 20 +++++++++++++------
 .../tests/centos6_tests.sh                    | 12 +++++++++--
 3 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index f1361d8ea..5f46e3a31 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -246,10 +246,14 @@ DeprecationBootstrap() {
   fi
 }
 
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
 # digits of the python version
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
   if [ -n "$USE_PYTHON_3" ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
@@ -273,10 +277,12 @@ DeterminePythonVersion() {
   export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -1575,8 +1581,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index f4c1b202f..7c3cbac08 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -246,10 +246,14 @@ DeprecationBootstrap() {
   fi
 }
 
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
 # Sets LE_PYTHON to Python version string and PYVER to the first two
 # digits of the python version
 DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
   if [ -n "$USE_PYTHON_3" ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
@@ -273,10 +277,12 @@ DeterminePythonVersion() {
   export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -586,8 +592,10 @@ else
 {{ fetch.py }}
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."
diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh
index 23b1e16e8..a0e96edf8 100644
--- a/letsencrypt-auto-source/tests/centos6_tests.sh
+++ b/letsencrypt-auto-source/tests/centos6_tests.sh
@@ -4,6 +4,8 @@ yum update -y > /dev/null
 yum install -y centos-release-scl > /dev/null
 yum install -y python27 > /dev/null 2> /dev/null
 
+LE_AUTO="certbot/letsencrypt-auto-source/letsencrypt-auto"
+
 # we're going to modify env variables, so do this in a subshell
 (
 source /opt/rh/python27/enable
@@ -25,7 +27,7 @@ if [ $RESULT -ne 0 ]; then
 fi
 
 # bootstrap, but don't install python 3.
-certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null
+"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
 
 # ensure python 3 isn't installed
 python3 --version 2> /dev/null
@@ -47,8 +49,14 @@ if [ $RESULT -eq 0 ]; then
   exit 1
 fi
 
+# Skip self upgrade due to Python 3 not being available.
+if ! "$LE_AUTO" 2>&1 | grep -q "WARNING: couldn't find Python"; then
+  echo "Python upgrade failure warning not printed!"
+  exit 1
+fi
+
 # bootstrap, this time installing python3
-certbot/letsencrypt-auto-source/letsencrypt-auto --no-self-upgrade -n > /dev/null 2> /dev/null
+"$LE_AUTO" --no-self-upgrade -n > /dev/null 2> /dev/null
 
 # ensure python 3 is installed
 python3 --version > /dev/null

From f5a02714cd8b610db52ac04e62685bba9c081a47 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Tue, 9 Jan 2018 16:11:04 -0800
Subject: [PATCH 09/45] Add deprecation warning for Python 2.6 (#5391)

* Add deprecation warning for Python 2.6

* Allow disabling Python 2.6 warning
---
 acme/acme/__init__.py | 13 +++++++------
 certbot/main.py       | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py
index 618dda200..5850fa955 100644
--- a/acme/acme/__init__.py
+++ b/acme/acme/__init__.py
@@ -13,9 +13,10 @@ supported version: `draft-ietf-acme-01`_.
 import sys
 import warnings
 
-if sys.version_info[:2] == (3, 3):
-    warnings.warn(
-            "Python 3.3 support will be dropped in the next release of "
-            "acme. Please upgrade your Python version.",
-            PendingDeprecationWarning,
-    ) #pragma: no cover
+for (major, minor) in [(2, 6), (3, 3)]:
+    if sys.version_info[:2] == (major, minor):
+        warnings.warn(
+                "Python {0}.{1} support will be dropped in the next release of "
+                "acme. Please upgrade your Python version.".format(major, minor),
+                DeprecationWarning,
+        ) #pragma: no cover
diff --git a/certbot/main.py b/certbot/main.py
index e25e030aa..32dd69256 100644
--- a/certbot/main.py
+++ b/certbot/main.py
@@ -4,6 +4,7 @@ import functools
 import logging.handlers
 import os
 import sys
+import warnings
 
 import configobj
 import josepy as jose
@@ -1217,9 +1218,17 @@ def main(cli_args=sys.argv[1:]):
         # Let plugins_cmd be run as un-privileged user.
         if config.func != plugins_cmd:
             raise
-    if sys.version_info[:2] == (3, 3):
-        logger.warning("Python 3.3 support will be dropped in the next release "
-                    "of Certbot - please upgrade your Python version.")
+    deprecation_fmt = (
+        "Python %s.%s support will be dropped in the next "
+        "release of Certbot - please upgrade your Python version.")
+    # We use the warnings system for Python 2.6 and logging for Python 3
+    # because DeprecationWarnings are only reported by default in Python <= 2.6
+    # and warnings can be disabled by the user.
+    if sys.version_info[:2] == (2, 6):
+        warning = deprecation_fmt % sys.version_info[:2]
+        warnings.warn(warning, DeprecationWarning)
+    elif sys.version_info[:2] == (3, 3):
+        logger.warning(deprecation_fmt, *sys.version_info[:2])
 
     set_displayer(config)
 

From 6eb459354fc28433ac2eabc3be62c809df66a4a1 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Tue, 9 Jan 2018 16:48:16 -0800
Subject: [PATCH 10/45] Address erikrose's comments on #5329 (#5400)

---
 letsencrypt-auto-source/letsencrypt-auto            | 13 ++++++++-----
 letsencrypt-auto-source/letsencrypt-auto.template   |  5 ++++-
 .../pieces/bootstrappers/rpm_common_base.sh         |  2 +-
 letsencrypt-auto-source/pieces/fetch.py             |  6 +++---
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 5f46e3a31..712ef6813 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -254,7 +254,7 @@ DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
-  if [ -n "$USE_PYTHON_3" ]; then
+  if [ "$USE_PYTHON_3" = 1 ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -443,7 +443,7 @@ InitializeRPMCommonBase() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
     if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
@@ -781,6 +781,9 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
@@ -1482,7 +1485,7 @@ class HttpsGetter(object):
         # Based on pip 1.4.1's URLOpener
         # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
         if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
-            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
         else:
             self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
@@ -1520,7 +1523,7 @@ def latest_stable_version(get):
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in iter(metadata['releases'].keys())
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1552,7 +1555,7 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
-def create_CERT_NONE_context():
+def cert_none_context():
     """Create a SSLContext object to not check hostname."""
     # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
     context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 7c3cbac08..b06ac9c80 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -254,7 +254,7 @@ DeterminePythonVersion() {
   # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
   #
   # If no Python is found, PYVER is set to 0.
-  if [ -n "$USE_PYTHON_3" ]; then
+  if [ "$USE_PYTHON_3" = 1 ]; then
     for LE_PYTHON in "$LE_PYTHON" python3; do
       # Break (while keeping the LE_PYTHON value) if found.
       $EXISTS "$LE_PYTHON" > /dev/null && break
@@ -320,6 +320,9 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
   prev_le_python="$LE_PYTHON"
   unset LE_PYTHON
   DeterminePythonVersion "NOCRASH"
diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
index d7a9f3133..326ad8b3f 100644
--- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
+++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common_base.sh
@@ -35,7 +35,7 @@ InitializeRPMCommonBase() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
     if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py
index ae72a299b..1515fe353 100644
--- a/letsencrypt-auto-source/pieces/fetch.py
+++ b/letsencrypt-auto-source/pieces/fetch.py
@@ -50,7 +50,7 @@ class HttpsGetter(object):
         # Based on pip 1.4.1's URLOpener
         # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
         if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
-            self._opener = build_opener(HTTPSHandler(context=create_CERT_NONE_context()))
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
         else:
             self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
@@ -88,7 +88,7 @@ def latest_stable_version(get):
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in iter(metadata['releases'].keys())
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -120,7 +120,7 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
-def create_CERT_NONE_context():
+def cert_none_context():
     """Create a SSLContext object to not check hostname."""
     # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
     context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)

From 00634394f2e077ec5e621acd869e370f8619cd08 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Tue, 9 Jan 2018 21:16:44 -0800
Subject: [PATCH 11/45] Only respect LE_PYTHON inside USE_PYTHON_3 if we know a
 user must have set it version 2 (#5402)

* stop exporting LE_PYTHON

* unset LE_PYTHON sometimes
---
 letsencrypt-auto-source/letsencrypt-auto          | 8 ++++++--
 letsencrypt-auto-source/letsencrypt-auto.template | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 712ef6813..0ad089acd 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -274,7 +274,6 @@ DeterminePythonVersion() {
       return 0
     fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
@@ -801,7 +800,7 @@ elif [ -f /etc/redhat-release ]; then
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
   fi
-  export LE_PYTHON="$prev_le_python"
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -906,6 +905,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
   if [ -d "$VENV_PATH" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
@@ -1412,6 +1415,7 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
     if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index b06ac9c80..5b8b1c164 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -274,7 +274,6 @@ DeterminePythonVersion() {
       return 0
     fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
   if [ "$PYVER" -lt "$MIN_PYVER" ]; then
@@ -340,7 +339,7 @@ elif [ -f /etc/redhat-release ]; then
     }
     BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
   fi
-  export LE_PYTHON="$prev_le_python"
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -445,6 +444,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
   if [ -d "$VENV_PATH" ]; then
     # If the selected Bootstrap function isn't a noop and it differs from the
@@ -571,6 +574,7 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
     if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then

From 3acf5d1ef909b2e775588a7e045bb00728e2de83 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 10 Jan 2018 12:10:21 -0800
Subject: [PATCH 12/45] Fix rebootstraping with old venvs (#5392)

* Fix rebootstrapping before venv move

* add regression test

* dedupe test

* Cleanup case when two venvs exist.

* Add clarifying comment

* Add double venv test to leauto_upgrades

* Fix logic with the help of coffee

* redirect stderr

* pass VENV_PATH through sudo

* redirect stderr
---
 letsencrypt-auto-source/letsencrypt-auto      | 24 ++++++++++--
 .../letsencrypt-auto.template                 | 24 ++++++++++--
 .../letstest/scripts/test_leauto_upgrades.sh  | 37 ++++++++++++++++++-
 3 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 0ad089acd..b86517bb6 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -897,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -910,13 +914,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
   fi
 
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -926,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -1418,7 +1434,7 @@ else
   export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 5b8b1c164..96e5c2db0 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -436,7 +436,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -449,13 +453,21 @@ if [ "$1" = "--le-auto-phase2" ]; then
   fi
 
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -465,6 +477,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -577,7 +593,7 @@ else
   export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index a83cbd826..51472f2e6 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -64,10 +64,45 @@ iQIDAQAB
 -----END PUBLIC KEY-----
 "
 
-if ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
+if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    if command -v python3; then
+        echo "Didn't expect Python 3 to be installed!"
+        exit 1
+    fi
+    cp letsencrypt-auto cb-auto
+    if ! ./cb-auto -v --debug --version 2>&1 | grep 0.5.0 ; then
+        echo "Certbot shouldn't have updated to a new version!"
+        exit 1
+    fi
+    if [ -d "/opt/eff.org" ]; then
+        echo "New directory shouldn't have been created!"
+        exit 1
+    fi
+    # Create a 2nd venv at the new path to ensure we properly handle this case
+    export VENV_PATH="/opt/eff.org/certbot/venv"
+    if ! sudo -E ./letsencrypt-auto -v --debug --version --no-self-upgrade 2>&1 | grep 0.5.0 ; then
+        echo second installation appeared to fail
+        exit 1
+    fi
+    unset VENV_PATH
+    EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
+    if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then
+        echo "Certbot didn't upgrade as expected!"
+        exit 1
+    fi
+    if ! command -v python3; then
+        echo "Python3 wasn't properly installed"
+        exit 1
+    fi
+    if [ "$(/opt/eff.org/certbot/venv/bin/python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1)" != 3 ]; then
+        echo "Python3 wasn't used in venv!"
+        exit 1
+    fi
+elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     echo upgrade appeared to fail
     exit 1
 fi
+
 echo upgrade appeared to be successful
 
 if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then

From 39472f88dead13782f2d84e65135cf11b96258bb Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 10 Jan 2018 13:26:31 -0800
Subject: [PATCH 13/45] reduce ipdb version (#5408)

---
 tools/dev_constraints.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt
index afc362ff8..47440241d 100644
--- a/tools/dev_constraints.txt
+++ b/tools/dev_constraints.txt
@@ -21,7 +21,7 @@ futures==3.1.1
 google-api-python-client==1.5
 httplib2==0.10.3
 imagesize==0.7.1
-ipdb==0.10.3
+ipdb==0.10.2
 ipython==5.5.0
 ipython-genutils==0.2.0
 Jinja2==2.9.6

From 9e952081014b9545ce6ddb8b6ecc86a51bf94131 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 10 Jan 2018 18:34:45 -0800
Subject: [PATCH 14/45] Factor out common challengeperformer logic (#5413)

---
 .../certbot_apache/tests/tls_sni_01_test.py   |  4 +-
 .../certbot_nginx/tests/tls_sni_01_test.py    |  2 +-
 certbot/plugins/common.py                     | 42 ++++++++++++----
 certbot/plugins/common_test.py                | 48 ++++++++++++-------
 4 files changed, 67 insertions(+), 29 deletions(-)

diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
index 6c37c2ecc..42fb3021b 100644
--- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
+++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
@@ -16,8 +16,8 @@ from six.moves import xrange  # pylint: disable=redefined-builtin, import-error
 class TlsSniPerformTest(util.ApacheTest):
     """Test the ApacheTlsSni01 challenge."""
 
-    auth_key = common_test.TLSSNI01Test.auth_key
-    achalls = common_test.TLSSNI01Test.achalls
+    auth_key = common_test.AUTH_KEY
+    achalls = common_test.ACHALLS
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(TlsSniPerformTest, self).setUp()
diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py
index 32a5ed7d2..61ee293fa 100644
--- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py
+++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py
@@ -20,7 +20,7 @@ from certbot_nginx.tests import util
 class TlsSniPerformTest(util.NginxTest):
     """Test the NginxTlsSni01 challenge."""
 
-    account_key = common_test.TLSSNI01Test.auth_key
+    account_key = common_test.AUTH_KEY
     achalls = [
         achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=acme_util.chall_to_challb(
diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py
index 002d2f225..c281534ca 100644
--- a/certbot/plugins/common.py
+++ b/certbot/plugins/common.py
@@ -315,23 +315,28 @@ class Addr(object):
         return result
 
 
-class TLSSNI01(object):
-    """Abstract base for TLS-SNI-01 challenge performers"""
+class ChallengePerformer(object):
+    """Abstract base for challenge performers.
+
+    :ivar configurator: Authenticator and installer plugin
+    :ivar achalls: Annotated challenges
+    :vartype achalls: `list` of `.KeyAuthorizationAnnotatedChallenge`
+    :ivar indices: Holds the indices of challenges from a larger array
+        so the user of the class doesn't have to.
+    :vartype indices: `list` of `int`
+
+    """
 
     def __init__(self, configurator):
         self.configurator = configurator
         self.achalls = []
         self.indices = []
-        self.challenge_conf = os.path.join(
-            configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf")
-        # self.completed = 0
 
     def add_chall(self, achall, idx=None):
-        """Add challenge to TLSSNI01 object to perform at once.
+        """Store challenge to be performed when perform() is called.
 
         :param .KeyAuthorizationAnnotatedChallenge achall: Annotated
-            TLSSNI01 challenge.
-
+            challenge.
         :param int idx: index to challenge in a larger array
 
         """
@@ -339,6 +344,27 @@ class TLSSNI01(object):
         if idx is not None:
             self.indices.append(idx)
 
+    def perform(self):
+        """Perform all added challenges.
+
+        :returns: challenge respones
+        :rtype: `list` of `acme.challenges.KeyAuthorizationChallengeResponse`
+
+
+        """
+        raise NotImplementedError()
+
+
+class TLSSNI01(ChallengePerformer):
+    # pylint: disable=abstract-method
+    """Abstract base for TLS-SNI-01 challenge performers"""
+
+    def __init__(self, configurator):
+        super(TLSSNI01, self).__init__(configurator)
+        self.challenge_conf = os.path.join(
+            configurator.config.config_dir, "le_tls_sni_01_cert_challenge.conf")
+        # self.completed = 0
+
     def get_cert_path(self, achall):
         """Returns standardized name for challenge certificate.
 
diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py
index 1a1ca7dcb..103a12499 100644
--- a/certbot/plugins/common_test.py
+++ b/certbot/plugins/common_test.py
@@ -18,6 +18,17 @@ from certbot import errors
 from certbot.tests import acme_util
 from certbot.tests import util as test_util
 
+AUTH_KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
+ACHALLS = [
+    achallenges.KeyAuthorizationAnnotatedChallenge(
+        challb=acme_util.chall_to_challb(
+            challenges.TLSSNI01(token=b'token1'), "pending"),
+        domain="encryption-example.demo", account_key=AUTH_KEY),
+    achallenges.KeyAuthorizationAnnotatedChallenge(
+        challb=acme_util.chall_to_challb(
+            challenges.TLSSNI01(token=b'token2'), "pending"),
+        domain="certbot.demo", account_key=AUTH_KEY),
+]
 
 class NamespaceFunctionsTest(unittest.TestCase):
     """Tests for certbot.plugins.common.*_namespace functions."""
@@ -261,21 +272,27 @@ class AddrTest(unittest.TestCase):
         self.assertEqual(set_c, set_d)
 
 
+class ChallengePerformerTest(unittest.TestCase):
+    """Tests for certbot.plugins.common.ChallengePerformer."""
+
+    def setUp(self):
+        configurator = mock.MagicMock()
+
+        from certbot.plugins.common import ChallengePerformer
+        self.performer = ChallengePerformer(configurator)
+
+    def test_add_chall(self):
+        self.performer.add_chall(ACHALLS[0], 0)
+        self.assertEqual(1, len(self.performer.achalls))
+        self.assertEqual([0], self.performer.indices)
+
+    def test_perform(self):
+        self.assertRaises(NotImplementedError, self.performer.perform)
+
+
 class TLSSNI01Test(unittest.TestCase):
     """Tests for certbot.plugins.common.TLSSNI01."""
 
-    auth_key = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem"))
-    achalls = [
-        achallenges.KeyAuthorizationAnnotatedChallenge(
-            challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(token=b'token1'), "pending"),
-            domain="encryption-example.demo", account_key=auth_key),
-        achallenges.KeyAuthorizationAnnotatedChallenge(
-            challb=acme_util.chall_to_challb(
-                challenges.TLSSNI01(token=b'token2'), "pending"),
-            domain="certbot.demo", account_key=auth_key),
-    ]
-
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
         configurator = mock.MagicMock()
@@ -288,11 +305,6 @@ class TLSSNI01Test(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.tempdir)
 
-    def test_add_chall(self):
-        self.sni.add_chall(self.achalls[0], 0)
-        self.assertEqual(1, len(self.sni.achalls))
-        self.assertEqual([0], self.sni.indices)
-
     def test_setup_challenge_cert(self):
         # This is a helper function that can be used for handling
         # open context managers more elegantly. It avoids dealing with
@@ -325,7 +337,7 @@ class TLSSNI01Test(unittest.TestCase):
             OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))
 
     def test_get_z_domain(self):
-        achall = self.achalls[0]
+        achall = ACHALLS[0]
         self.assertEqual(self.sni.get_z_domain(achall),
             achall.response(achall.account_key).z_domain.decode("utf-8"))
 

From 2ba334a18202926e4e6b4f166a18457861bba031 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 10 Jan 2018 20:14:56 -0800
Subject: [PATCH 15/45] Add basic HTTP01 support to Apache

* Add a simple version of HTTP01

* remove cert from chall name

* make directory work on 2.2

* cleanup challenges when finished

* import shutil

* fixup perform and cleanup tests

* Add tests for http_01.py
---
 certbot-apache/certbot_apache/configurator.py |  36 ++++--
 certbot-apache/certbot_apache/http_01.py      |  94 +++++++++++++++
 .../certbot_apache/tests/configurator_test.py |  70 +++++++----
 .../certbot_apache/tests/http_01_test.py      | 112 ++++++++++++++++++
 .../certbot_apache/tests/tls_sni_01_test.py   |   2 +-
 5 files changed, 278 insertions(+), 36 deletions(-)
 create mode 100644 certbot-apache/certbot_apache/http_01.py
 create mode 100644 certbot-apache/certbot_apache/tests/http_01_test.py

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 5a33346ea..60441f30c 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -24,9 +24,10 @@ from certbot_apache import apache_util
 from certbot_apache import augeas_configurator
 from certbot_apache import constants
 from certbot_apache import display_ops
-from certbot_apache import tls_sni_01
+from certbot_apache import http_01
 from certbot_apache import obj
 from certbot_apache import parser
+from certbot_apache import tls_sni_01
 
 from collections import defaultdict
 
@@ -163,6 +164,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                               "ensure-http-header": self._set_http_header,
                               "staple-ocsp": self._enable_ocsp_stapling}
 
+        # This will be set during the perform function
+        self.http_doer = None
+
     @property
     def mod_ssl_conf(self):
         """Full absolute path to SSL configuration file."""
@@ -1855,7 +1859,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01]
+        return [challenges.TLSSNI01, challenges.HTTP01]
 
     def perform(self, achalls):
         """Perform the configuration related challenge.
@@ -1867,16 +1871,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         self._chall_out.update(achalls)
         responses = [None] * len(achalls)
-        chall_doer = tls_sni_01.ApacheTlsSni01(self)
+        self.http_doer = http_01.ApacheHttp01(self)
+        sni_doer = tls_sni_01.ApacheTlsSni01(self)
 
         for i, achall in enumerate(achalls):
             # Currently also have chall_doer hold associated index of the
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
-            chall_doer.add_chall(achall, i)
+            if isinstance(achall.chall, challenges.HTTP01):
+                self.http_doer.add_chall(achall, i)
+            else:  # tls-sni-01
+                sni_doer.add_chall(achall, i)
 
-        sni_response = chall_doer.perform()
-        if sni_response:
+        http_response = self.http_doer.perform()
+        sni_response = sni_doer.perform()
+        if http_response or sni_response:
             # Must reload in order to activate the challenges.
             # Handled here because we may be able to load up other challenge
             # types
@@ -1886,14 +1895,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # of identifying when the new configuration is being used.
             time.sleep(3)
 
-            # Go through all of the challenges and assign them to the proper
-            # place in the responses return value. All responses must be in the
-            # same order as the original challenges.
-            for i, resp in enumerate(sni_response):
-                responses[chall_doer.indices[i]] = resp
+            self._update_responses(responses, http_response, self.http_doer)
+            self._update_responses(responses, sni_response, sni_doer)
 
         return responses
 
+    def _update_responses(self, responses, chall_response, chall_doer):
+        # Go through all of the challenges and assign them to the proper
+        # place in the responses return value. All responses must be in the
+        # same order as the original challenges.
+        for i, resp in enumerate(chall_response):
+            responses[chall_doer.indices[i]] = resp
+
     def cleanup(self, achalls):
         """Revert all challenges."""
         self._chall_out.difference_update(achalls)
@@ -1903,6 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             self.revert_challenge_config()
             self.restart()
             self.parser.reset_modules()
+            self.http_doer.cleanup()
 
     def install_ssl_options_conf(self, options_ssl, options_ssl_digest):
         """Copy Certbot's SSL options file into the system's config dir if required."""
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
new file mode 100644
index 000000000..410e8e9a6
--- /dev/null
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -0,0 +1,94 @@
+"""A class that performs HTTP-01 challenges for Apache"""
+import logging
+import os
+import shutil
+import tempfile
+
+from certbot.plugins import common
+
+logger = logging.getLogger(__name__)
+
+class ApacheHttp01(common.TLSSNI01):
+    """Class that performs HTPP-01 challenges within the Apache configurator."""
+
+    CONFIG_TEMPLATE24 = """\
+Alias /.well-known/acme-challenge {0}
+
+<Directory {0} >
+    Require all granted
+</Directory>
+
+"""
+
+    CONFIG_TEMPLATE22 = """\
+Alias /.well-known/acme-challenge {0}
+
+<Directory {0} >
+    Order allow,deny
+    Allow from all
+</Directory>
+
+"""
+
+    def __init__(self, *args, **kwargs):
+        super(ApacheHttp01, self).__init__(*args, **kwargs)
+        self.challenge_conf = os.path.join(
+            self.configurator.conf("challenge-location"),
+            "le_http_01_challenge.conf")
+        self.challenge_dir = None
+
+    def perform(self):
+        """Perform all HTTP-01 challenges."""
+        if not self.achalls:
+            return []
+        # Save any changes to the configuration as a precaution
+        # About to make temporary changes to the config
+        self.configurator.save("Changes before challenge setup", True)
+
+        responses = self._set_up_challenges()
+        self._mod_config()
+        # Save reversible changes
+        self.configurator.save("HTTP Challenge", True)
+
+        return responses
+
+    def cleanup(self):
+        """Cleanup the challenge directory."""
+        shutil.rmtree(self.challenge_dir, ignore_errors=True)
+        self.challenge_dir = None
+
+    def _mod_config(self):
+        self.configurator.parser.add_include(
+            self.configurator.parser.loc["default"], self.challenge_conf)
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf)
+
+        if self.configurator.version < (2, 4):
+            config_template = self.CONFIG_TEMPLATE22
+        else:
+            config_template = self.CONFIG_TEMPLATE24
+        config_text = config_template.format(self.challenge_dir)
+
+        logger.debug("writing a config file with text:\n %s", config_text)
+        with open(self.challenge_conf, "w") as new_conf:
+            new_conf.write(config_text)
+
+    def _set_up_challenges(self):
+        self.challenge_dir = tempfile.mkdtemp()
+        os.chmod(self.challenge_dir, 0o755)
+
+        responses = []
+        for achall in self.achalls:
+            responses.append(self._set_up_challenge(achall))
+
+        return responses
+
+    def _set_up_challenge(self, achall):
+        response, validation = achall.response_and_validation()
+
+        name = os.path.join(self.challenge_dir, achall.chall.encode("token"))
+        with open(name, 'wb') as f:
+            f.write(validation.encode())
+        os.chmod(name, 0o644)
+
+        return response
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
index 4f85e1e3f..1620013c8 100644
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -676,23 +676,33 @@ class MultipleVhostsTest(util.ApacheTest):
         self.config._add_name_vhost_if_necessary(self.vh_truth[0])
         self.assertEqual(self.config.add_name_vhost.call_count, 2)
 
+    @mock.patch("certbot_apache.configurator.http_01.ApacheHttp01.perform")
     @mock.patch("certbot_apache.configurator.tls_sni_01.ApacheTlsSni01.perform")
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
-    def test_perform(self, mock_restart, mock_perform):
+    def test_perform(self, mock_restart, mock_tls_perform, mock_http_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
-        account_key, achall1, achall2 = self.get_achalls()
+        account_key, achalls = self.get_key_and_achalls()
 
-        expected = [
-            achall1.response(account_key),
-            achall2.response(account_key),
-        ]
+        all_expected = []
+        http_expected = []
+        tls_expected = []
+        for achall in achalls:
+            response = achall.response(account_key)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_expected.append(response)
+            else:
+                tls_expected.append(response)
+            all_expected.append(response)
 
-        mock_perform.return_value = expected
-        responses = self.config.perform([achall1, achall2])
+        mock_http_perform.return_value = http_expected
+        mock_tls_perform.return_value = tls_expected
 
-        self.assertEqual(mock_perform.call_count, 1)
-        self.assertEqual(responses, expected)
+        responses = self.config.perform(achalls)
+
+        self.assertEqual(mock_http_perform.call_count, 1)
+        self.assertEqual(mock_tls_perform.call_count, 1)
+        self.assertEqual(responses, all_expected)
 
         self.assertEqual(mock_restart.call_count, 1)
 
@@ -700,30 +710,38 @@ class MultipleVhostsTest(util.ApacheTest):
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
+        self.config.http_doer = mock.MagicMock()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
-        self.config._chall_out.add(achall2)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall1])
-        self.assertFalse(mock_restart.called)
-
-        self.config.cleanup([achall2])
-        self.assertTrue(mock_restart.called)
+        for i, achall in enumerate(achalls):
+            self.config.cleanup([achall])
+            if i == len(achalls) - 1:
+                self.assertTrue(mock_restart.called)
+                self.assertTrue(self.config.http_doer.cleanup.called)
+            else:
+                self.assertFalse(mock_restart.called)
+                self.assertFalse(self.config.http_doer.cleanup.called)
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
     def test_cleanup_no_errors(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
-        _, achall1, achall2 = self.get_achalls()
+        _, achalls = self.get_key_and_achalls()
+        self.config.http_doer = mock.MagicMock()
 
-        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
+        for achall in achalls:
+            self.config._chall_out.add(achall)  # pylint: disable=protected-access
 
-        self.config.cleanup([achall2])
+        self.config.cleanup([achalls[-1]])
         self.assertFalse(mock_restart.called)
+        self.assertFalse(self.config.http_doer.cleanup.called)
 
-        self.config.cleanup([achall1, achall2])
+        self.config.cleanup(achalls)
         self.assertTrue(mock_restart.called)
+        self.assertTrue(self.config.http_doer.cleanup.called)
 
     @mock.patch("certbot.util.run_script")
     def test_get_version(self, mock_script):
@@ -1151,7 +1169,7 @@ class MultipleVhostsTest(util.ApacheTest):
         not_rewriterule = "NotRewriteRule ^ ..."
         self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule))
 
-    def get_achalls(self):
+    def get_key_and_achalls(self):
         """Return testing achallenges."""
         account_key = self.rsa512jwk
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
@@ -1166,8 +1184,12 @@ class MultipleVhostsTest(util.ApacheTest):
                     token=b"uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
                 "pending"),
             domain="certbot.demo", account_key=account_key)
+        achall3 = achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=(b'x' * 16)), "pending"),
+            domain="example.org", account_key=account_key)
 
-        return account_key, achall1, achall2
+        return account_key, (achall1, achall2, achall3)
 
     def test_make_addrs_sni_ready(self):
         self.config.version = (2, 2)
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
new file mode 100644
index 000000000..11121fae2
--- /dev/null
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -0,0 +1,112 @@
+"""Test for certbot_apache.http_01."""
+import os
+import unittest
+
+from acme import challenges
+
+from certbot import achallenges
+
+from certbot.tests import acme_util
+
+from certbot_apache.tests import util
+
+
+NUM_ACHALLS = 3
+
+
+class ApacheHttp01TestMeta(type):
+    """Generates parmeterized tests for testing perform."""
+    def __new__(mcs, name, bases, class_dict):
+
+        def _gen_test(num_achalls, minor_version):
+            def _test(self):
+                achalls = self.achalls[:num_achalls]
+                self.config.version = (2, minor_version)
+                self.common_perform_test(achalls)
+            return _test
+
+        for i in range(1, NUM_ACHALLS + 1):
+            for j in (2, 4):
+                test_name = "test_perform_{0}_{1}".format(i, j)
+                class_dict[test_name] = _gen_test(i, j)
+        return type.__new__(mcs, name, bases, class_dict)
+
+
+class ApacheHttp01Test(util.ApacheTest):
+    """Test for certbot_apache.http_01.ApacheHttp01."""
+
+    __metaclass__ = ApacheHttp01TestMeta
+
+    def setUp(self, *args, **kwargs):
+        super(ApacheHttp01Test, self).setUp(*args, **kwargs)
+        self.maxDiff = None
+
+        self.account_key = self.rsa512jwk
+        self.achalls = []
+        for i in range(NUM_ACHALLS):
+            self.achalls.append(
+                achallenges.KeyAuthorizationAnnotatedChallenge(
+                    challb=acme_util.chall_to_challb(
+                        challenges.HTTP01(token=((chr(ord('a') + i) * 16))),
+                        "pending"),
+                    domain="example{0}.com".format(i),
+                    account_key=self.account_key))
+
+        from certbot_apache.http_01 import ApacheHttp01
+        self.http = ApacheHttp01(self.config)
+
+    def test_empty_perform(self):
+        self.assertFalse(self.http.perform())
+
+    def common_perform_test(self, achalls):
+        """Tests perform with the given achalls."""
+        for achall in achalls:
+            self.http.add_chall(achall)
+
+        expected_response = [
+            achall.response(self.account_key) for achall in achalls]
+        self.assertEqual(self.http.perform(), expected_response)
+
+        self.assertTrue(os.path.isdir(self.http.challenge_dir))
+        self._has_min_permissions(self.http.challenge_dir, 0o755)
+        self._test_challenge_conf()
+
+        for achall in achalls:
+            self._test_challenge_file(achall)
+
+        challenge_dir = self.http.challenge_dir
+        self.http.cleanup()
+        self.assertFalse(os.path.exists(challenge_dir))
+
+    def _test_challenge_conf(self):
+        self.assertEqual(
+            len(self.config.parser.find_dir(
+                "Include", self.http.challenge_conf)), 1)
+
+        with open(self.http.challenge_conf) as f:
+            conf_contents = f.read()
+
+        alias_fmt = "Alias /.well-known/acme-challenge {0}"
+        alias = alias_fmt.format(self.http.challenge_dir)
+        self.assertTrue(alias in conf_contents)
+        if self.config.version < (2, 4):
+            self.assertTrue("Allow from all" in conf_contents)
+        else:
+            self.assertTrue("Require all granted" in conf_contents)
+
+    def _test_challenge_file(self, achall):
+        name = os.path.join(self.http.challenge_dir, achall.chall.encode("token"))
+        validation = achall.validation(self.account_key)
+
+        self._has_min_permissions(name, 0o644)
+        with open(name, 'rb') as f:
+            self.assertEqual(f.read(), validation.encode())
+
+    def _has_min_permissions(self, path, min_mode):
+        """Tests the given file has at least the permissions in mode."""
+        st_mode = os.stat(path).st_mode
+        self.assertEqual(st_mode, st_mode | min_mode)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover
diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
index 42fb3021b..8cea97f04 100644
--- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
+++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py
@@ -1,6 +1,6 @@
 """Test for certbot_apache.tls_sni_01."""
-import unittest
 import shutil
+import unittest
 
 import mock
 

From fa97877cfb058a8a6eda0e86b079876230bcfb7d Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Thu, 11 Jan 2018 14:46:48 +0200
Subject: [PATCH 16/45] Make sure that Apache is listening on port 80 and has
 mod_alias

* Ensure that mod_alias is enabled

* Make sure we listen to port http01_port
---
 certbot-apache/certbot_apache/configurator.py | 62 +++++++++++++++----
 certbot-apache/certbot_apache/http_01.py      | 11 ++++
 .../certbot_apache/tests/configurator_test.py | 38 +++++++++++-
 .../certbot_apache/tests/http_01_test.py      | 11 ++++
 certbot-apache/certbot_apache/tests/util.py   |  1 +
 5 files changed, 111 insertions(+), 12 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 60441f30c..3b91617e5 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -740,28 +740,40 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
 
-        # If nonstandard port, add service definition for matching
-        if port != "443":
+        self.prepare_https_modules(temp)
+        self.ensure_listen(port, https=True)
+
+    def ensure_listen(self, port, https=False):
+        """Make sure that Apache is listening on the port. Checks if the
+        Listen statement for the port already exists, and adds it to the
+        configuration if necessary.
+
+        :param str port: Port number to check and add Listen for if not in
+            place already
+        :param bool https: If the port will be used for HTTPS
+
+        """
+
+        # If HTTPS requested for nonstandard port, add service definition
+        if https and port != "443":
             port_service = "%s %s" % (port, "https")
         else:
             port_service = port
 
-        self.prepare_https_modules(temp)
         # Check for Listen <port>
         # Note: This could be made to also look for ip:443 combo
         listens = [self.parser.get_arg(x).split()[0] for
                    x in self.parser.find_dir("Listen")]
 
-        # In case no Listens are set (which really is a broken apache config)
-        if not listens:
-            listens = ["80"]
-
         # Listen already in place
         if self._has_port_already(listens, port):
             return
 
         listen_dirs = set(listens)
 
+        if not listens:
+            listen_dirs.add(port_service)
+
         for listen in listens:
             # For any listen statement, check if the machine also listens on
             # Port 443. If not, add such a listen statement.
@@ -776,11 +788,39 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 if "%s:%s" % (ip, port_service) not in listen_dirs and (
                    "%s:%s" % (ip, port_service) not in listen_dirs):
                     listen_dirs.add("%s:%s" % (ip, port_service))
-        self._add_listens(listen_dirs, listens, port)
+        if https:
+            self._add_listens_https(listen_dirs, listens, port)
+        else:
+            self._add_listens_http(listen_dirs, listens, port)
 
-    def _add_listens(self, listens, listens_orig, port):
-        """Helper method for prepare_server_https to figure out which new
-        listen statements need adding
+    def _add_listens_http(self, listens, listens_orig, port):
+        """Helper method for ensure_listen to figure out which new
+        listen statements need adding for listening HTTP on port
+
+        :param set listens: Set of all needed Listen statements
+        :param list listens_orig: List of existing listen statements
+        :param string port: Port number we're adding
+        """
+
+        new_listens = listens.difference(listens_orig)
+
+        if port in new_listens:
+            # We have wildcard, skip the rest
+            self.parser.add_dir(parser.get_aug_path(self.parser.loc["listen"]),
+                                "Listen", port)
+            self.save_notes += "Added Listen %s directive to %s\n" % (
+                port, self.parser.loc["listen"])
+        else:
+            for listen in new_listens:
+                self.parser.add_dir(parser.get_aug_path(
+                    self.parser.loc["listen"]), "Listen", listen.split(" "))
+                self.save_notes += ("Added Listen %s directive to "
+                                    "%s\n") % (listen,
+                                               self.parser.loc["listen"])
+
+    def _add_listens_https(self, listens, listens_orig, port):
+        """Helper method for ensure_listen to figure out which new
+        listen statements need adding for listening HTTPS on port
 
         :param set listens: Set of all needed Listen statements
         :param list listens_orig: List of existing listen statements
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 410e8e9a6..9c25b1f17 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -45,6 +45,10 @@ Alias /.well-known/acme-challenge {0}
         # About to make temporary changes to the config
         self.configurator.save("Changes before challenge setup", True)
 
+        self.configurator.ensure_listen(str(
+            self.configurator.config.http01_port))
+        self.prepare_http01_modules()
+
         responses = self._set_up_challenges()
         self._mod_config()
         # Save reversible changes
@@ -57,6 +61,13 @@ Alias /.well-known/acme-challenge {0}
         shutil.rmtree(self.challenge_dir, ignore_errors=True)
         self.challenge_dir = None
 
+    def prepare_http01_modules(self):
+        """Make sure that we have the needed modules available for http01"""
+
+        if self.configurator.conf("handle-modules"):
+            if "alias_module" not in self.configurator.parser.modules:
+                self.configurator.enable_mod("alias", temp=True)
+
     def _mod_config(self):
         self.configurator.parser.add_include(
             self.configurator.parser.loc["default"], self.challenge_conf)
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
index 1620013c8..ea3867061 100644
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -424,6 +424,43 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertTrue(self.config.parser.find_dir(
             "NameVirtualHost", "*:80"))
 
+    def test_add_listen_80(self):
+        mock_find = mock.Mock()
+        mock_add_dir = mock.Mock()
+        mock_find.return_value = []
+        self.config.parser.find_dir = mock_find
+        self.config.parser.add_dir = mock_add_dir
+        self.config.ensure_listen("80")
+        self.assertTrue(mock_add_dir.called)
+        self.assertTrue(mock_find.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        self.assertEqual(mock_add_dir.call_args[0][2], "80")
+
+    def test_add_listen_80_named(self):
+        mock_find = mock.Mock()
+        mock_find.return_value = ["test1", "test2", "test3"]
+        mock_get = mock.Mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+        mock_add_dir = mock.Mock()
+
+        self.config.parser.find_dir = mock_find
+        self.config.parser.get_arg = mock_get
+        self.config.parser.add_dir = mock_add_dir
+
+        self.config.ensure_listen("80")
+        self.assertEqual(mock_add_dir.call_count, 0)
+
+        # Reset return lists and inputs
+        mock_add_dir.reset_mock()
+        mock_get.side_effect = ["1.2.3.4:80", "[::1]:80", "1.1.1.1:443"]
+
+        # Test
+        self.config.ensure_listen("8080")
+        self.assertEqual(mock_add_dir.call_count, 3)
+        self.assertTrue(mock_add_dir.called)
+        self.assertEqual(mock_add_dir.call_args[0][1], "Listen")
+        self.assertEqual(mock_add_dir.call_args[0][2], ['1.2.3.4:8080'])
+
     def test_prepare_server_https(self):
         mock_enable = mock.Mock()
         self.config.enable_mod = mock_enable
@@ -435,7 +472,6 @@ class MultipleVhostsTest(util.ApacheTest):
         # This will test the Add listen
         self.config.parser.find_dir = mock_find
         self.config.parser.add_dir_to_ifmodssl = mock_add_dir
-
         self.config.prepare_server_https("443")
         # Changing the order these modules are enabled breaks the reverter
         self.assertEqual(mock_enable.call_args_list[0][0][0], "socache_shmcb")
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 11121fae2..f52b9d0cc 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -1,4 +1,5 @@
 """Test for certbot_apache.http_01."""
+import mock
 import os
 import unittest
 
@@ -53,11 +54,21 @@ class ApacheHttp01Test(util.ApacheTest):
                     account_key=self.account_key))
 
         from certbot_apache.http_01 import ApacheHttp01
+        self.config.parser.modules.add("mod_alias.c")
+        self.config.parser.modules.add("alias_module")
         self.http = ApacheHttp01(self.config)
 
     def test_empty_perform(self):
         self.assertFalse(self.http.perform())
 
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_add_alias_module(self, mock_enmod):
+        self.config.parser.modules.remove("alias_module")
+        self.config.parser.modules.remove("mod_alias.c")
+        self.http.prepare_http01_modules()
+        self.assertTrue(mock_enmod.called)
+        self.assertEqual(mock_enmod.call_args[0][0], "alias")
+
     def common_perform_test(self, achalls):
         """Tests perform with the given achalls."""
         for achall in achalls:
diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py
index ca667465c..8f5162629 100644
--- a/certbot-apache/certbot_apache/tests/util.py
+++ b/certbot-apache/certbot_apache/tests/util.py
@@ -103,6 +103,7 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
         apache_challenge_location=config_path,
         backup_dir=backups,
         config_dir=config_dir,
+        http01_port="80",
         temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)

From f0f5defb6ff9fcc54db4bbc977950610b22cd155 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Thu, 11 Jan 2018 09:27:30 -0800
Subject: [PATCH 17/45] Address minor concerns with Apache HTTP-01

* enable other modules

* change port type

* remove maxDiff from test class

* update port comment

* add -f to a2dismod
---
 certbot-apache/certbot_apache/configurator.py |  2 +-
 certbot-apache/certbot_apache/http_01.py      | 10 ++++-
 .../certbot_apache/override_debian.py         |  2 +-
 .../certbot_apache/tests/http_01_test.py      | 39 ++++++++++++++++---
 certbot-apache/certbot_apache/tests/util.py   |  2 +-
 5 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 3b91617e5..8d6995211 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -776,7 +776,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         for listen in listens:
             # For any listen statement, check if the machine also listens on
-            # Port 443. If not, add such a listen statement.
+            # the given port. If not, add such a listen statement.
             if len(listen.split(":")) == 1:
                 # Its listening to all interfaces
                 if port not in listen_dirs and port_service not in listen_dirs:
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 9c25b1f17..804644975 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -65,8 +65,14 @@ Alias /.well-known/acme-challenge {0}
         """Make sure that we have the needed modules available for http01"""
 
         if self.configurator.conf("handle-modules"):
-            if "alias_module" not in self.configurator.parser.modules:
-                self.configurator.enable_mod("alias", temp=True)
+            needed_modules = ["alias"]
+            if self.configurator.version < (2, 4):
+                needed_modules.append("authz_host")
+            else:
+                needed_modules.append("authz_core")
+            for mod in needed_modules:
+                if mod + "_module" not in self.configurator.parser.modules:
+                    self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
         self.configurator.parser.add_include(
diff --git a/certbot-apache/certbot_apache/override_debian.py b/certbot-apache/certbot_apache/override_debian.py
index 6e2e34ba9..02dffc3f7 100644
--- a/certbot-apache/certbot_apache/override_debian.py
+++ b/certbot-apache/certbot_apache/override_debian.py
@@ -140,5 +140,5 @@ class DebianConfigurator(configurator.ApacheConfigurator):
                 "a2dismod are configured correctly for certbot.")
 
         self.reverter.register_undo_command(
-            temp, [self.conf("dismod"), mod_name])
+            temp, [self.conf("dismod"), "-f", mod_name])
         util.run_script([self.conf("enmod"), mod_name])
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index f52b9d0cc..4e2a5faff 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -40,7 +40,6 @@ class ApacheHttp01Test(util.ApacheTest):
 
     def setUp(self, *args, **kwargs):
         super(ApacheHttp01Test, self).setUp(*args, **kwargs)
-        self.maxDiff = None
 
         self.account_key = self.rsa512jwk
         self.achalls = []
@@ -53,21 +52,51 @@ class ApacheHttp01Test(util.ApacheTest):
                     domain="example{0}.com".format(i),
                     account_key=self.account_key))
 
+        modules = ["alias", "authz_core", "authz_host"]
+        for mod in modules:
+            self.config.parser.modules.add("mod_{0}.c".format(mod))
+            self.config.parser.modules.add(mod + "_module")
+
         from certbot_apache.http_01 import ApacheHttp01
-        self.config.parser.modules.add("mod_alias.c")
-        self.config.parser.modules.add("alias_module")
         self.http = ApacheHttp01(self.config)
 
     def test_empty_perform(self):
         self.assertFalse(self.http.perform())
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
-    def test_add_alias_module(self, mock_enmod):
+    def test_enable_modules_22(self, mock_enmod):
+        self.config.version = (2, 2)
+        self.config.parser.modules.remove("authz_host_module")
+        self.config.parser.modules.remove("mod_authz_host.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_host")
+
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.enable_mod")
+    def test_enable_modules_24(self, mock_enmod):
+        self.config.parser.modules.remove("authz_core_module")
+        self.config.parser.modules.remove("mod_authz_core.c")
+
+        enmod_calls = self.common_enable_modules_test(mock_enmod)
+        self.assertEqual(enmod_calls[0][0][0], "authz_core")
+
+    def common_enable_modules_test(self, mock_enmod):
+        """Tests enabling mod_alias and other modules."""
         self.config.parser.modules.remove("alias_module")
         self.config.parser.modules.remove("mod_alias.c")
+
         self.http.prepare_http01_modules()
+
         self.assertTrue(mock_enmod.called)
-        self.assertEqual(mock_enmod.call_args[0][0], "alias")
+        calls = mock_enmod.call_args_list
+        other_calls = []
+        for call in calls:
+            if "alias" != call[0][0]:
+                other_calls.append(call)
+
+        # If these lists are equal, we never enabled mod_alias
+        self.assertNotEqual(calls, other_calls)
+        return other_calls
 
     def common_perform_test(self, achalls):
         """Tests perform with the given achalls."""
diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py
index 8f5162629..1ba1e2c34 100644
--- a/certbot-apache/certbot_apache/tests/util.py
+++ b/certbot-apache/certbot_apache/tests/util.py
@@ -103,7 +103,7 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
         apache_challenge_location=config_path,
         backup_dir=backups,
         config_dir=config_dir,
-        http01_port="80",
+        http01_port=80,
         temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)

From 28dad825afae8a079811114d72a53a9833155d6c Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Thu, 11 Jan 2018 20:44:40 +0200
Subject: [PATCH 18/45] Do not try to remove temp dir if it wasn't created

---
 certbot-apache/certbot_apache/http_01.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 804644975..87721d290 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -58,7 +58,8 @@ Alias /.well-known/acme-challenge {0}
 
     def cleanup(self):
         """Cleanup the challenge directory."""
-        shutil.rmtree(self.challenge_dir, ignore_errors=True)
+        if self.challenge_dir:
+            shutil.rmtree(self.challenge_dir, ignore_errors=True)
         self.challenge_dir = None
 
     def prepare_http01_modules(self):

From 2cb9d9e2aa332e4fd53aab32ad1d3946438a0dd3 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Thu, 11 Jan 2018 17:06:23 -0800
Subject: [PATCH 19/45] Implement HTTP-01 challenge for Nginx (#5414)

* get http01 challenge working

* support multiple challenge types in configurator.py

* update existing nginx tests

* lint

* refactor NginxHttp01 and NginxTlsSni01 to both now  inherit from NginxChallengePerformer

* remove TODO

* challenges_test tests with both tlssni01 and http01

* Make challenges.py more abstract to make lint happier

* add pylint disables to the tests to make pylint happier about the inheritance and abstraction situation

* no need to cover raise NotImplementedError() lines

* python3 compatibility

* test that http01 perform is called

* only remove ssl from addresses during http01

* Initialize addrs_to_add

* Change Nginx http01 to modify server block so the site doesn't stop serving while getting a cert

* pass existing unit tests

* rename sni --> http01 in unit tests

* lint

* fix configurator test

* select an http block instead of https

* properly test for port number

* use domains that have matching addresses

* remove debugger

* remove access_log and error_log cruft that wasn't being executed

* continue to return None from choose_redirect_vhost when create_if_no_match is False

* add nginx integration test
---
 certbot-nginx/certbot_nginx/configurator.py   |  35 ++++--
 certbot-nginx/certbot_nginx/http_01.py        | 106 ++++++++++++++++
 certbot-nginx/certbot_nginx/parser.py         |   4 +-
 .../certbot_nginx/tests/configurator_test.py  |  14 ++-
 .../certbot_nginx/tests/http_01_test.py       | 113 ++++++++++++++++++
 certbot-nginx/certbot_nginx/tests/util.py     |   1 +
 .../tests/boulder-integration.conf.sh         |   6 +-
 certbot-nginx/tests/boulder-integration.sh    |  19 ++-
 8 files changed, 272 insertions(+), 26 deletions(-)
 create mode 100644 certbot-nginx/certbot_nginx/http_01.py
 create mode 100644 certbot-nginx/certbot_nginx/tests/http_01_test.py

diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py
index 8af474c5e..a77cf2bc3 100644
--- a/certbot-nginx/certbot_nginx/configurator.py
+++ b/certbot-nginx/certbot_nginx/configurator.py
@@ -26,6 +26,7 @@ from certbot_nginx import constants
 from certbot_nginx import nginxparser
 from certbot_nginx import parser
 from certbot_nginx import tls_sni_01
+from certbot_nginx import http_01
 
 
 logger = logging.getLogger(__name__)
@@ -208,7 +209,8 @@ class NginxConfigurator(common.Installer):
 
         :param str target_name: domain name
         :param bool create_if_no_match: If we should create a new vhost from default
-            when there is no match found
+            when there is no match found. If we can't choose a default, raise a
+            MisconfigurationError.
 
         :returns: ssl vhost associated with name
         :rtype: :class:`~certbot_nginx.obj.VirtualHost`
@@ -366,7 +368,7 @@ class NginxConfigurator(common.Installer):
         return sorted(matches, key=lambda x: x['rank'])
 
 
-    def choose_redirect_vhost(self, target_name, port):
+    def choose_redirect_vhost(self, target_name, port, create_if_no_match=False):
         """Chooses a single virtual host for redirect enhancement.
 
         Chooses the vhost most closely matching target_name that is
@@ -380,12 +382,19 @@ class NginxConfigurator(common.Installer):
 
         :param str target_name: domain name
         :param str port: port number
+        :param bool create_if_no_match: If we should create a new vhost from default
+            when there is no match found. If we can't choose a default, raise a
+            MisconfigurationError.
+
         :returns: vhost associated with name
         :rtype: :class:`~certbot_nginx.obj.VirtualHost`
 
         """
         matches = self._get_redirect_ranked_matches(target_name, port)
-        return self._select_best_name_match(matches)
+        vhost = self._select_best_name_match(matches)
+        if not vhost and create_if_no_match:
+            vhost = self._vhost_from_duplicated_default(target_name)
+        return vhost
 
     def _get_redirect_ranked_matches(self, target_name, port):
         """Gets a ranked list of plaintextish port-listening vhosts matching target_name
@@ -394,7 +403,7 @@ class NginxConfigurator(common.Installer):
         Rank by how well these match target_name.
 
         :param str target_name: The name to match
-        :param str port: port number
+        :param str port: port number as a string
         :returns: list of dicts containing the vhost, the matching name, and
             the numerical rank
         :rtype: list
@@ -840,7 +849,7 @@ class NginxConfigurator(common.Installer):
     ###########################################################################
     def get_chall_pref(self, unused_domain):  # pylint: disable=no-self-use
         """Return list of challenge preferences."""
-        return [challenges.TLSSNI01]
+        return [challenges.TLSSNI01, challenges.HTTP01]
 
     # Entry point in main.py for performing challenges
     def perform(self, achalls):
@@ -853,15 +862,20 @@ class NginxConfigurator(common.Installer):
         """
         self._chall_out += len(achalls)
         responses = [None] * len(achalls)
-        chall_doer = tls_sni_01.NginxTlsSni01(self)
+        sni_doer = tls_sni_01.NginxTlsSni01(self)
+        http_doer = http_01.NginxHttp01(self)
 
         for i, achall in enumerate(achalls):
             # Currently also have chall_doer hold associated index of the
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
-            chall_doer.add_chall(achall, i)
+            if isinstance(achall.chall, challenges.HTTP01):
+                http_doer.add_chall(achall, i)
+            else:  # tls-sni-01
+                sni_doer.add_chall(achall, i)
 
-        sni_response = chall_doer.perform()
+        sni_response = sni_doer.perform()
+        http_response = http_doer.perform()
         # Must restart in order to activate the challenges.
         # Handled here because we may be able to load up other challenge types
         self.restart()
@@ -869,8 +883,9 @@ class NginxConfigurator(common.Installer):
         # Go through all of the challenges and assign them to the proper place
         # in the responses return value. All responses must be in the same order
         # as the original challenges.
-        for i, resp in enumerate(sni_response):
-            responses[chall_doer.indices[i]] = resp
+        for chall_response, chall_doer in ((sni_response, sni_doer), (http_response, http_doer)):
+            for i, resp in enumerate(chall_response):
+                responses[chall_doer.indices[i]] = resp
 
         return responses
 
diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py
new file mode 100644
index 000000000..1f1e37891
--- /dev/null
+++ b/certbot-nginx/certbot_nginx/http_01.py
@@ -0,0 +1,106 @@
+"""A class that performs HTTP-01 challenges for Nginx"""
+
+import logging
+import os
+
+from acme import challenges
+
+from certbot.plugins import common
+
+
+logger = logging.getLogger(__name__)
+
+
+class NginxHttp01(common.ChallengePerformer):
+    """HTTP-01 authenticator for Nginx
+
+    :ivar configurator: NginxConfigurator object
+    :type configurator: :class:`~nginx.configurator.NginxConfigurator`
+
+    :ivar list achalls: Annotated
+        class:`~certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+        challenges
+
+    :param list indices: Meant to hold indices of challenges in a
+        larger array. NginxHttp01 is capable of solving many challenges
+        at once which causes an indexing issue within NginxConfigurator
+        who must return all responses in order.  Imagine NginxConfigurator
+        maintaining state about where all of the http-01 Challenges,
+        TLS-SNI-01 Challenges belong in the response array.  This is an
+        optional utility.
+
+    """
+
+    def perform(self):
+        """Perform a challenge on Nginx.
+
+        :returns: list of :class:`certbot.acme.challenges.HTTP01Response`
+        :rtype: list
+
+        """
+        if not self.achalls:
+            return []
+
+        responses = [x.response(x.account_key) for x in self.achalls]
+
+        # Set up the configuration
+        self._mod_config()
+
+        # Save reversible changes
+        self.configurator.save("HTTP Challenge", True)
+
+        return responses
+
+    def _add_bucket_directive(self):
+        """Modifies Nginx config to include server_names_hash_bucket_size directive."""
+        root = self.configurator.parser.config_root
+
+        bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128']
+
+        main = self.configurator.parser.parsed[root]
+        for line in main:
+            if line[0] == ['http']:
+                body = line[1]
+                found_bucket = False
+                posn = 0
+                for inner_line in body:
+                    if inner_line[0] == bucket_directive[1]:
+                        if int(inner_line[1]) < int(bucket_directive[3]):
+                            body[posn] = bucket_directive
+                        found_bucket = True
+                    posn += 1
+                if not found_bucket:
+                    body.insert(0, bucket_directive)
+                break
+
+    def _mod_config(self):
+        """Modifies Nginx config to handle challenges.
+
+        """
+        self._add_bucket_directive()
+
+        for achall in self.achalls:
+            self._mod_server_block(achall)
+
+    def _get_validation_path(self, achall):
+        return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token"))
+
+    def _mod_server_block(self, achall):
+        """Modifies a server block to respond to a challenge.
+
+        :param achall: Annotated HTTP-01 challenge
+        :type achall:
+            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+
+        """
+        vhost = self.configurator.choose_redirect_vhost(achall.domain,
+            '%i' % self.configurator.config.http01_port, create_if_no_match=True)
+        validation = achall.validation(achall.account_key)
+        validation_path = self._get_validation_path(achall)
+
+        location_directive = [[['location', ' ', '=', ' ', validation_path],
+                               [['default_type', ' ', 'text/plain'],
+                                ['return', ' ', '200', ' ', validation]]]]
+
+        self.configurator.parser.add_server_directives(vhost,
+            location_directive, replace=False)
diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py
index 9f13bc59f..5497f7e63 100644
--- a/certbot-nginx/certbot_nginx/parser.py
+++ b/certbot-nginx/certbot_nginx/parser.py
@@ -524,7 +524,7 @@ def _is_ssl_on_directive(entry):
 def _add_directives(directives, replace, block):
     """Adds or replaces directives in a config block.
 
-    When replace=False, it's an error to try and add a directive that already
+    When replace=False, it's an error to try and add a nonrepeatable directive that already
     exists in the config block with a conflicting value.
 
     When replace=True and a directive with the same name already exists in the
@@ -545,7 +545,7 @@ def _add_directives(directives, replace, block):
 
 
 INCLUDE = 'include'
-REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE])
+REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location'])
 COMMENT = ' managed by Certbot'
 COMMENT_BLOCK = [' ', '#', COMMENT]
 
diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py
index e708b159a..7475df40c 100644
--- a/certbot-nginx/certbot_nginx/tests/configurator_test.py
+++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py
@@ -100,7 +100,7 @@ class NginxConfiguratorTest(util.NginxTest):
             errors.PluginError, self.config.enhance, 'myhost', 'unknown_enhancement')
 
     def test_get_chall_pref(self):
-        self.assertEqual([challenges.TLSSNI01],
+        self.assertEqual([challenges.TLSSNI01, challenges.HTTP01],
                          self.config.get_chall_pref('myhost'))
 
     def test_save(self):
@@ -291,9 +291,11 @@ class NginxConfiguratorTest(util.NginxTest):
                          parsed_migration_conf[0])
 
     @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform")
+    @mock.patch("certbot_nginx.configurator.http_01.NginxHttp01.perform")
     @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart")
     @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config")
-    def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform):
+    def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_http_perform,
+        mock_tls_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
         achall1 = achallenges.KeyAuthorizationAnnotatedChallenge(
@@ -304,7 +306,7 @@ class NginxConfiguratorTest(util.NginxTest):
             ), domain="localhost", account_key=self.rsa512jwk)
         achall2 = achallenges.KeyAuthorizationAnnotatedChallenge(
             challb=messages.ChallengeBody(
-                chall=challenges.TLSSNI01(token=b"m8TdO1qik4JVFtgPPurJmg"),
+                chall=challenges.HTTP01(token=b"m8TdO1qik4JVFtgPPurJmg"),
                 uri="https://ca.org/chall1_uri",
                 status=messages.Status("pending"),
             ), domain="example.com", account_key=self.rsa512jwk)
@@ -314,10 +316,12 @@ class NginxConfiguratorTest(util.NginxTest):
             achall2.response(self.rsa512jwk),
         ]
 
-        mock_perform.return_value = expected
+        mock_tls_perform.return_value = expected[:1]
+        mock_http_perform.return_value = expected[1:]
         responses = self.config.perform([achall1, achall2])
 
-        self.assertEqual(mock_perform.call_count, 1)
+        self.assertEqual(mock_tls_perform.call_count, 1)
+        self.assertEqual(mock_http_perform.call_count, 1)
         self.assertEqual(responses, expected)
 
         self.config.cleanup([achall1, achall2])
diff --git a/certbot-nginx/certbot_nginx/tests/http_01_test.py b/certbot-nginx/certbot_nginx/tests/http_01_test.py
new file mode 100644
index 000000000..0f764e92e
--- /dev/null
+++ b/certbot-nginx/certbot_nginx/tests/http_01_test.py
@@ -0,0 +1,113 @@
+"""Tests for certbot_nginx.http_01"""
+import unittest
+import shutil
+
+import mock
+import six
+
+from acme import challenges
+
+from certbot import achallenges
+
+from certbot.plugins import common_test
+from certbot.tests import acme_util
+
+from certbot_nginx.tests import util
+
+
+class HttpPerformTest(util.NginxTest):
+    """Test the NginxHttp01 challenge."""
+
+    account_key = common_test.AUTH_KEY
+    achalls = [
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=b"kNdwjwOeX0I_A8DXt9Msmg"), "pending"),
+            domain="www.example.com", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(
+                    token=b"\xba\xa9\xda?<m\xaewmx\xea\xad\xadv\xf4\x02\xc9y"
+                          b"\x80\xe2_X\t\xe7\xc7\xa4\t\xca\xf7&\x945"
+                ), "pending"),
+            domain="ipv6.com", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(
+                    token=b"\x8c\x8a\xbf_-f\\cw\xee\xd6\xf8/\xa5\xe3\xfd"
+                          b"\xeb9\xf1\xf5\xb9\xefVM\xc9w\xa4u\x9c\xe1\x87\xb4"
+                ), "pending"),
+            domain="www.example.org", account_key=account_key),
+        achallenges.KeyAuthorizationAnnotatedChallenge(
+            challb=acme_util.chall_to_challb(
+                challenges.HTTP01(token=b"kNdwjxOeX0I_A8DXt9Msmg"), "pending"),
+            domain="migration.com", account_key=account_key),
+    ]
+
+    def setUp(self):
+        super(HttpPerformTest, self).setUp()
+
+        config = util.get_nginx_configurator(
+            self.config_path, self.config_dir, self.work_dir, self.logs_dir)
+
+        from certbot_nginx import http_01
+        self.http01 = http_01.NginxHttp01(config)
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        shutil.rmtree(self.config_dir)
+        shutil.rmtree(self.work_dir)
+
+    def test_perform0(self):
+        responses = self.http01.perform()
+        self.assertEqual([], responses)
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.save")
+    def test_perform1(self, mock_save):
+        self.http01.add_chall(self.achalls[0])
+        response = self.achalls[0].response(self.account_key)
+
+        responses = self.http01.perform()
+
+        self.assertEqual([response], responses)
+        self.assertEqual(mock_save.call_count, 1)
+
+    def test_perform2(self):
+        acme_responses = []
+        for achall in self.achalls:
+            self.http01.add_chall(achall)
+            acme_responses.append(achall.response(self.account_key))
+
+        sni_responses = self.http01.perform()
+
+        self.assertEqual(len(sni_responses), 4)
+        for i in six.moves.range(4):
+            self.assertEqual(sni_responses[i], acme_responses[i])
+
+    def test_mod_config(self):
+        self.http01.add_chall(self.achalls[0])
+        self.http01.add_chall(self.achalls[2])
+
+        self.http01._mod_config()  # pylint: disable=protected-access
+
+        self.http01.configurator.save()
+
+        self.http01.configurator.parser.load()
+
+        # vhosts = self.http01.configurator.parser.get_vhosts()
+
+        # for vhost in vhosts:
+        #     pass
+            # if the name matches
+            # check that the location block is in there and is correct
+
+            # if vhost.addrs == set(v_addr1):
+            #     response = self.achalls[0].response(self.account_key)
+            # else:
+            #     response = self.achalls[2].response(self.account_key)
+            #     self.assertEqual(vhost.addrs, set(v_addr2_print))
+            # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')]))
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover
diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py
index 7b32d8e82..ad1af2b96 100644
--- a/certbot-nginx/certbot_nginx/tests/util.py
+++ b/certbot-nginx/certbot_nginx/tests/util.py
@@ -64,6 +64,7 @@ def get_nginx_configurator(
                     in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
                     server="https://acme-server.org:443/new",
                     tls_sni_01_port=5001,
+                    http01_port=80
                 ),
                 name="nginx",
                 version=version)
diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh
index 5295e3681..c38180698 100755
--- a/certbot-nginx/tests/boulder-integration.conf.sh
+++ b/certbot-nginx/tests/boulder-integration.conf.sh
@@ -49,10 +49,10 @@ http {
 
   server {
     # IPv4.
-    listen 8081;
+    listen 5002;
     # IPv6.
-    listen [::]:8081 default ipv6only=on;
-    server_name nginx.wtf;
+    listen [::]:5002 default ipv6only=on;
+    server_name nginx.wtf nginx2.wtf;
 
     root $root/webroot;
 
diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh
index 996cc2201..f236fb103 100755
--- a/certbot-nginx/tests/boulder-integration.sh
+++ b/certbot-nginx/tests/boulder-integration.sh
@@ -22,13 +22,20 @@ certbot_test_nginx () {
         "$@"
 }
 
-certbot_test_nginx --domains nginx.wtf run
-echo | openssl s_client -connect localhost:5001 \
-    | openssl x509 -out $root/nginx.pem
-diff -q $root/nginx.pem $root/conf/live/nginx.wtf/cert.pem
+test_deployment_and_rollback() {
+    # Arguments: certname
+    echo | openssl s_client -connect localhost:5001 \
+        | openssl x509 -out $root/nginx.pem
+    diff -q $root/nginx.pem "$root/conf/live/$1/cert.pem"
 
-certbot_test_nginx rollback --checkpoints 9001
-diff -q <(echo "$original") $nginx_conf
+    certbot_test_nginx rollback --checkpoints 9001
+    diff -q <(echo "$original") $nginx_conf
+}
+
+certbot_test_nginx --domains nginx.wtf run
+test_deployment_and_rollback nginx.wtf
+certbot_test_nginx --domains nginx2.wtf --preferred-challenges http
+test_deployment_and_rollback nginx2.wtf
 
 # note: not reached if anything above fails, hence "killall" at the
 # top

From 60dd67a60e4f17b5d46718d3ff941a2e292398c7 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joohoi@users.noreply.github.com>
Date: Mon, 15 Jan 2018 01:22:22 +0200
Subject: [PATCH 20/45] Use static directory under workdir for HTTP challenges
 (#5428)

* Use static directory under workdir for HTTP challenges

* Handle the reverter file registration before opening file handle
---
 certbot-apache/certbot_apache/configurator.py |  1 -
 certbot-apache/certbot_apache/http_01.py      | 19 ++++++++-----------
 .../certbot_apache/tests/configurator_test.py |  5 -----
 .../certbot_apache/tests/http_01_test.py      |  6 +++---
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 8d6995211..b87189dc0 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -1956,7 +1956,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             self.revert_challenge_config()
             self.restart()
             self.parser.reset_modules()
-            self.http_doer.cleanup()
 
     def install_ssl_options_conf(self, options_ssl, options_ssl_digest):
         """Copy Certbot's SSL options file into the system's config dir if required."""
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 87721d290..3b942823d 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -1,8 +1,6 @@
 """A class that performs HTTP-01 challenges for Apache"""
 import logging
 import os
-import shutil
-import tempfile
 
 from certbot.plugins import common
 
@@ -35,7 +33,9 @@ Alias /.well-known/acme-challenge {0}
         self.challenge_conf = os.path.join(
             self.configurator.conf("challenge-location"),
             "le_http_01_challenge.conf")
-        self.challenge_dir = None
+        self.challenge_dir = os.path.join(
+            self.configurator.config.work_dir,
+            "http_challenges")
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -56,12 +56,6 @@ Alias /.well-known/acme-challenge {0}
 
         return responses
 
-    def cleanup(self):
-        """Cleanup the challenge directory."""
-        if self.challenge_dir:
-            shutil.rmtree(self.challenge_dir, ignore_errors=True)
-        self.challenge_dir = None
-
     def prepare_http01_modules(self):
         """Make sure that we have the needed modules available for http01"""
 
@@ -92,8 +86,9 @@ Alias /.well-known/acme-challenge {0}
             new_conf.write(config_text)
 
     def _set_up_challenges(self):
-        self.challenge_dir = tempfile.mkdtemp()
-        os.chmod(self.challenge_dir, 0o755)
+        if not os.path.isdir(self.challenge_dir):
+            os.makedirs(self.challenge_dir)
+            os.chmod(self.challenge_dir, 0o755)
 
         responses = []
         for achall in self.achalls:
@@ -105,6 +100,8 @@ Alias /.well-known/acme-challenge {0}
         response, validation = achall.response_and_validation()
 
         name = os.path.join(self.challenge_dir, achall.chall.encode("token"))
+
+        self.configurator.reverter.register_file_creation(True, name)
         with open(name, 'wb') as f:
             f.write(validation.encode())
         os.chmod(name, 0o644)
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
index ea3867061..df67104da 100644
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -747,7 +747,6 @@ class MultipleVhostsTest(util.ApacheTest):
     def test_cleanup(self, mock_cfg, mock_restart):
         mock_cfg.return_value = ""
         _, achalls = self.get_key_and_achalls()
-        self.config.http_doer = mock.MagicMock()
 
         for achall in achalls:
             self.config._chall_out.add(achall)  # pylint: disable=protected-access
@@ -756,10 +755,8 @@ class MultipleVhostsTest(util.ApacheTest):
             self.config.cleanup([achall])
             if i == len(achalls) - 1:
                 self.assertTrue(mock_restart.called)
-                self.assertTrue(self.config.http_doer.cleanup.called)
             else:
                 self.assertFalse(mock_restart.called)
-                self.assertFalse(self.config.http_doer.cleanup.called)
 
     @mock.patch("certbot_apache.configurator.ApacheConfigurator.restart")
     @mock.patch("certbot_apache.parser.ApacheParser._get_runtime_cfg")
@@ -773,11 +770,9 @@ class MultipleVhostsTest(util.ApacheTest):
 
         self.config.cleanup([achalls[-1]])
         self.assertFalse(mock_restart.called)
-        self.assertFalse(self.config.http_doer.cleanup.called)
 
         self.config.cleanup(achalls)
         self.assertTrue(mock_restart.called)
-        self.assertTrue(self.config.http_doer.cleanup.called)
 
     @mock.patch("certbot.util.run_script")
     def test_get_version(self, mock_script):
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 4e2a5faff..204d9a76c 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -100,6 +100,8 @@ class ApacheHttp01Test(util.ApacheTest):
 
     def common_perform_test(self, achalls):
         """Tests perform with the given achalls."""
+        challenge_dir = self.http.challenge_dir
+        self.assertFalse(os.path.exists(challenge_dir))
         for achall in achalls:
             self.http.add_chall(achall)
 
@@ -114,9 +116,7 @@ class ApacheHttp01Test(util.ApacheTest):
         for achall in achalls:
             self._test_challenge_file(achall)
 
-        challenge_dir = self.http.challenge_dir
-        self.http.cleanup()
-        self.assertFalse(os.path.exists(challenge_dir))
+        self.assertTrue(os.path.exists(challenge_dir))
 
     def _test_challenge_conf(self):
         self.assertEqual(

From 368ca0c1096aa198b2e841dd52778646812e8141 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Mon, 15 Jan 2018 20:47:03 -0800
Subject: [PATCH 21/45] Small cleanup for Apache HTTP-01

* Remove http_doer from self

* Refactor _find_best_vhost
---
 certbot-apache/certbot_apache/configurator.py | 29 +++++++++++--------
 .../certbot_apache/tests/configurator_test.py |  3 +-
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index b87189dc0..05c838123 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -164,9 +164,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                               "ensure-http-header": self._set_http_header,
                               "staple-ocsp": self._enable_ocsp_stapling}
 
-        # This will be set during the perform function
-        self.http_doer = None
-
     @property
     def mod_ssl_conf(self):
         """Full absolute path to SSL configuration file."""
@@ -439,12 +436,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 return True
         return False
 
-    def _find_best_vhost(self, target_name):
+    def _find_best_vhost(self, target_name, vhosts=None):
         """Finds the best vhost for a target_name.
 
         This does not upgrade a vhost to HTTPS... it only finds the most
         appropriate vhost for the given target_name.
 
+        :param str target_name: domain handled by the desired vhost
+        :param vhosts: vhosts to consider
+        :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost`
+
         :returns: VHost or None
 
         """
@@ -456,7 +457,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # Points 1 - Address name with no SSL
         best_candidate = None
         best_points = 0
-        for vhost in self.vhosts:
+
+        if vhosts is None:
+            vhosts = self.vhosts
+
+        for vhost in vhosts:
             if vhost.modmacro is True:
                 continue
             names = vhost.get_names()
@@ -481,7 +486,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # No winners here... is there only one reasonable vhost?
         if best_candidate is None:
             # reasonable == Not all _default_ addrs
-            vhosts = self._non_default_vhosts()
+            vhosts = self._non_default_vhosts(vhosts)
             # remove mod_macro hosts from reasonable vhosts
             reasonable_vhosts = [vh for vh
                                  in vhosts if vh.modmacro is False]
@@ -490,9 +495,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         return best_candidate
 
-    def _non_default_vhosts(self):
+    def _non_default_vhosts(self, vhosts):
         """Return all non _default_ only vhosts."""
-        return [vh for vh in self.vhosts if not all(
+        return [vh for vh in vhosts if not all(
             addr.get_addr() == "_default_" for addr in vh.addrs
         )]
 
@@ -1911,7 +1916,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         self._chall_out.update(achalls)
         responses = [None] * len(achalls)
-        self.http_doer = http_01.ApacheHttp01(self)
+        http_doer = http_01.ApacheHttp01(self)
         sni_doer = tls_sni_01.ApacheTlsSni01(self)
 
         for i, achall in enumerate(achalls):
@@ -1919,11 +1924,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # challenge. This helps to put all of the responses back together
             # when they are all complete.
             if isinstance(achall.chall, challenges.HTTP01):
-                self.http_doer.add_chall(achall, i)
+                http_doer.add_chall(achall, i)
             else:  # tls-sni-01
                 sni_doer.add_chall(achall, i)
 
-        http_response = self.http_doer.perform()
+        http_response = http_doer.perform()
         sni_response = sni_doer.perform()
         if http_response or sni_response:
             # Must reload in order to activate the challenges.
@@ -1935,7 +1940,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             # of identifying when the new configuration is being used.
             time.sleep(3)
 
-            self._update_responses(responses, http_response, self.http_doer)
+            self._update_responses(responses, http_response, http_doer)
             self._update_responses(responses, sni_response, sni_doer)
 
         return responses
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
index df67104da..c846d2d42 100644
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -305,7 +305,8 @@ class MultipleVhostsTest(util.ApacheTest):
 
     def test_non_default_vhosts(self):
         # pylint: disable=protected-access
-        self.assertEqual(len(self.config._non_default_vhosts()), 8)
+        vhosts = self.config._non_default_vhosts(self.config.vhosts)
+        self.assertEqual(len(vhosts), 8)
 
     def test_deploy_cert_enable_new_vhost(self):
         # Create

From 7e463bccad469c1fa7cda6aba09b1e459a514e2d Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Tue, 16 Jan 2018 14:58:45 -0800
Subject: [PATCH 22/45] Handle more edge cases for HTTP-01 support in Nginx
 (#5421)

* only when using http01, only match default_server by port

* import errors

* put back in the code that creates a dummy block, but only when we can't find anything else
---
 certbot-nginx/certbot_nginx/configurator.py |  30 +++---
 certbot-nginx/certbot_nginx/http_01.py      | 107 ++++++++++++++++++--
 2 files changed, 113 insertions(+), 24 deletions(-)

diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py
index a77cf2bc3..bb2933a39 100644
--- a/certbot-nginx/certbot_nginx/configurator.py
+++ b/certbot-nginx/certbot_nginx/configurator.py
@@ -261,9 +261,9 @@ class NginxConfigurator(common.Installer):
                     ipv6only_present = True
         return (ipv6_active, ipv6only_present)
 
-    def _vhost_from_duplicated_default(self, domain):
+    def _vhost_from_duplicated_default(self, domain, port=None):
         if self.new_vhost is None:
-            default_vhost = self._get_default_vhost()
+            default_vhost = self._get_default_vhost(port)
             self.new_vhost = self.parser.duplicate_vhost(default_vhost, delete_default=True)
             self.new_vhost.names = set()
 
@@ -278,15 +278,16 @@ class NginxConfigurator(common.Installer):
             name_block[0].append(name)
         self.parser.add_server_directives(vhost, name_block, replace=True)
 
-    def _get_default_vhost(self):
+    def _get_default_vhost(self, port):
         vhost_list = self.parser.get_vhosts()
         # if one has default_server set, return that one
         default_vhosts = []
         for vhost in vhost_list:
             for addr in vhost.addrs:
                 if addr.default:
-                    default_vhosts.append(vhost)
-                    break
+                    if port is None or self._port_matches(port, addr.get_port()):
+                        default_vhosts.append(vhost)
+                        break
 
         if len(default_vhosts) == 1:
             return default_vhosts[0]
@@ -393,9 +394,17 @@ class NginxConfigurator(common.Installer):
         matches = self._get_redirect_ranked_matches(target_name, port)
         vhost = self._select_best_name_match(matches)
         if not vhost and create_if_no_match:
-            vhost = self._vhost_from_duplicated_default(target_name)
+            vhost = self._vhost_from_duplicated_default(target_name, port=port)
         return vhost
 
+    def _port_matches(self, test_port, matching_port):
+        # test_port is a number, matching is a number or "" or None
+        if matching_port == "" or matching_port is None:
+            # if no port is specified, Nginx defaults to listening on port 80.
+            return test_port == self.DEFAULT_LISTEN_PORT
+        else:
+            return test_port == matching_port
+
     def _get_redirect_ranked_matches(self, target_name, port):
         """Gets a ranked list of plaintextish port-listening vhosts matching target_name
 
@@ -410,13 +419,6 @@ class NginxConfigurator(common.Installer):
 
         """
         all_vhosts = self.parser.get_vhosts()
-        def _port_matches(test_port, matching_port):
-            # test_port is a number, matching is a number or "" or None
-            if matching_port == "" or matching_port is None:
-                # if no port is specified, Nginx defaults to listening on port 80.
-                return test_port == self.DEFAULT_LISTEN_PORT
-            else:
-                return test_port == matching_port
 
         def _vhost_matches(vhost, port):
             found_matching_port = False
@@ -426,7 +428,7 @@ class NginxConfigurator(common.Installer):
                 found_matching_port = (port == self.DEFAULT_LISTEN_PORT)
             else:
                 for addr in vhost.addrs:
-                    if _port_matches(port, addr.get_port()) and addr.ssl == False:
+                    if self._port_matches(port, addr.get_port()) and addr.ssl == False:
                         found_matching_port = True
 
             if found_matching_port:
diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py
index 1f1e37891..c25081ae0 100644
--- a/certbot-nginx/certbot_nginx/http_01.py
+++ b/certbot-nginx/certbot_nginx/http_01.py
@@ -5,8 +5,12 @@ import os
 
 from acme import challenges
 
+from certbot import errors
 from certbot.plugins import common
 
+from certbot_nginx import obj
+from certbot_nginx import nginxparser
+
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +35,13 @@ class NginxHttp01(common.ChallengePerformer):
 
     """
 
+    def __init__(self, configurator):
+        super(NginxHttp01, self).__init__(configurator)
+        self.challenge_conf = os.path.join(
+            configurator.config.config_dir, "le_http_01_cert_challenge.conf")
+        self._ipv6 = None
+        self._ipv6only = None
+
     def perform(self):
         """Perform a challenge on Nginx.
 
@@ -51,8 +62,16 @@ class NginxHttp01(common.ChallengePerformer):
 
         return responses
 
-    def _add_bucket_directive(self):
-        """Modifies Nginx config to include server_names_hash_bucket_size directive."""
+    def _mod_config(self):
+        """Modifies Nginx config to include server_names_hash_bucket_size directive
+           and server challenge blocks.
+
+        :raises .MisconfigurationError:
+            Unable to find a suitable HTTP block in which to include
+            authenticator hosts.
+        """
+        included = False
+        include_directive = ['\n', 'include', ' ', self.challenge_conf]
         root = self.configurator.parser.config_root
 
         bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128']
@@ -71,21 +90,82 @@ class NginxHttp01(common.ChallengePerformer):
                     posn += 1
                 if not found_bucket:
                     body.insert(0, bucket_directive)
+                if include_directive not in body:
+                    body.insert(0, include_directive)
+                included = True
                 break
+        if not included:
+            raise errors.MisconfigurationError(
+                'Certbot could not find a block to include '
+                'challenges in %s.' % root)
+        config = [self._make_or_mod_server_block(achall) for achall in self.achalls]
+        config = [x for x in config if x is not None]
+        config = nginxparser.UnspacedList(config)
 
-    def _mod_config(self):
-        """Modifies Nginx config to handle challenges.
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf)
 
+        with open(self.challenge_conf, "w") as new_conf:
+            nginxparser.dump(config, new_conf)
+
+    def _default_listen_addresses(self):
+        """Finds addresses for a challenge block to listen on.
+        :returns: list of :class:`certbot_nginx.obj.Addr` to apply
+        :rtype: list
         """
-        self._add_bucket_directive()
+        addresses = []
+        default_addr = "%s" % self.configurator.config.http01_port
+        ipv6_addr = "[::]:{0}".format(
+            self.configurator.config.http01_port)
+        port = self.configurator.config.http01_port
 
-        for achall in self.achalls:
-            self._mod_server_block(achall)
+        if self._ipv6 is None or self._ipv6only is None:
+            self._ipv6, self._ipv6only = self.configurator.ipv6_info(port)
+        ipv6, ipv6only = self._ipv6, self._ipv6only
+
+        if ipv6:
+            # If IPv6 is active in Nginx configuration
+            if not ipv6only:
+                # If ipv6only=on is not already present in the config
+                ipv6_addr = ipv6_addr + " ipv6only=on"
+            addresses = [obj.Addr.fromstring(default_addr),
+                         obj.Addr.fromstring(ipv6_addr)]
+            logger.info(("Using default addresses %s and %s for authentication."),
+                        default_addr,
+                        ipv6_addr)
+        else:
+            addresses = [obj.Addr.fromstring(default_addr)]
+            logger.info("Using default address %s for authentication.",
+                        default_addr)
+        return addresses
 
     def _get_validation_path(self, achall):
         return os.sep + os.path.join(challenges.HTTP01.URI_ROOT_PATH, achall.chall.encode("token"))
 
-    def _mod_server_block(self, achall):
+    def _make_server_block(self, achall):
+        """Creates a server block for a challenge.
+        :param achall: Annotated HTTP-01 challenge
+        :type achall:
+            :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
+        :param list addrs: addresses of challenged domain
+            :class:`list` of type :class:`~nginx.obj.Addr`
+        :returns: server block for the challenge host
+        :rtype: list
+        """
+        addrs = self._default_listen_addresses()
+        block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs]
+
+        validation = achall.validation(achall.account_key)
+        validation_path = self._get_validation_path(achall)
+
+        block.extend([['server_name', ' ', achall.domain],
+                      [['location', ' ', '=', ' ', validation_path],
+                        [['default_type', ' ', 'text/plain'],
+                         ['return', ' ', '200', ' ', validation]]]])
+        # TODO: do we want to return something else if they otherwise access this block?
+        return [['server'], block]
+
+    def _make_or_mod_server_block(self, achall):
         """Modifies a server block to respond to a challenge.
 
         :param achall: Annotated HTTP-01 challenge
@@ -93,8 +173,15 @@ class NginxHttp01(common.ChallengePerformer):
             :class:`certbot.achallenges.KeyAuthorizationAnnotatedChallenge`
 
         """
-        vhost = self.configurator.choose_redirect_vhost(achall.domain,
-            '%i' % self.configurator.config.http01_port, create_if_no_match=True)
+        try:
+            vhost = self.configurator.choose_redirect_vhost(achall.domain,
+                '%i' % self.configurator.config.http01_port, create_if_no_match=True)
+        except errors.MisconfigurationError:
+            # Couldn't find either a matching name+port server block
+            # or a port+default_server block, so create a dummy block
+            return self._make_server_block(achall)
+
+        # Modify existing server block
         validation = achall.validation(achall.account_key)
         validation_path = self._get_validation_path(achall)
 

From 314c5f19e51ccefc740f87751c589e2ca0fe9754 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Tue, 16 Jan 2018 20:33:25 +0200
Subject: [PATCH 23/45] Set up vhost discovery and overrides for HTTP-01

* Finalized HTTP vhost discovery and added overrides

* Include overrides to every VirtualHost
---
 certbot-apache/certbot_apache/configurator.py | 14 ++++-
 certbot-apache/certbot_apache/http_01.py      | 60 +++++++++++++------
 certbot-apache/certbot_apache/parser.py       | 17 ++++++
 .../certbot_apache/tests/http_01_test.py      |  6 +-
 .../certbot_apache/tests/parser_test.py       | 17 ++++++
 5 files changed, 92 insertions(+), 22 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 05c838123..8f1aff8d7 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -436,6 +436,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 return True
         return False
 
+    def find_best_http_vhost(self, target):
+        """Returns non-HTTPS vhost objects found from the Apache config
+
+        :param str target: Domain name of the desired VirtualHost
+
+        :returns: VirtualHost object that's the best match for target name
+        :rtype: `obj.VirtualHost` or None
+        """
+        nonssl_vhosts = [i for i in self.vhosts if not i.ssl]
+        return self._find_best_vhost(target, nonssl_vhosts)
+
+
     def _find_best_vhost(self, target_name, vhosts=None):
         """Finds the best vhost for a target_name.
 
@@ -508,7 +520,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                   virtual host addresses
         :rtype: set
 
-        """
+    """
         all_names = set()
 
         vhost_macro = []
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 3b942823d..a2cd43845 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -7,26 +7,38 @@ from certbot.plugins import common
 logger = logging.getLogger(__name__)
 
 class ApacheHttp01(common.TLSSNI01):
-    """Class that performs HTPP-01 challenges within the Apache configurator."""
+    """Class that performs HTTP-01 challenges within the Apache configurator."""
 
-    CONFIG_TEMPLATE24 = """\
-Alias /.well-known/acme-challenge {0}
+    CONFIG_TEMPLATE_COMMON = """\
+        Alias /.well-known/acme-challenge {0}"
 
-<Directory {0} >
-    Require all granted
-</Directory>
-
-"""
+        <IfModule mod_proxy.c>
+            ProxyPass "/.well-known/acme-challenge" !
+        </IfModule>
+    """
 
     CONFIG_TEMPLATE22 = """\
-Alias /.well-known/acme-challenge {0}
+        <IfModule mod_rewrite.c>
+            RewriteEngine on
+            RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999]
+        </IfModule>
 
-<Directory {0} >
-    Order allow,deny
-    Allow from all
-</Directory>
+        <Directory {0}>
+            Order allow deny
+            Allow from all
+        </Directory>
+    """
 
-"""
+    CONFIG_TEMPLATE24 = """\
+        <IfModule mod_rewrite.c>
+            RewriteEngine on
+            RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END]
+        </IfModule>
+
+        <Directory {0}>
+            Require all granted
+        </Directory>
+    """
 
     def __init__(self, *args, **kwargs):
         super(ApacheHttp01, self).__init__(*args, **kwargs)
@@ -50,6 +62,7 @@ Alias /.well-known/acme-challenge {0}
         self.prepare_http01_modules()
 
         responses = self._set_up_challenges()
+
         self._mod_config()
         # Save reversible changes
         self.configurator.save("HTTP Challenge", True)
@@ -70,21 +83,26 @@ Alias /.well-known/acme-challenge {0}
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
-        self.configurator.parser.add_include(
-            self.configurator.parser.loc["default"], self.challenge_conf)
+        for chall in self.achalls:
+            vh = self.configurator.find_best_http_vhost(chall.domain)
+            if vh:
+                self._set_up_include_directive(vh)
+
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
 
         if self.configurator.version < (2, 4):
-            config_template = self.CONFIG_TEMPLATE22
+            config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22
         else:
-            config_template = self.CONFIG_TEMPLATE24
+            config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24
+
         config_text = config_template.format(self.challenge_dir)
 
         logger.debug("writing a config file with text:\n %s", config_text)
         with open(self.challenge_conf, "w") as new_conf:
             new_conf.write(config_text)
 
+
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
             os.makedirs(self.challenge_dir)
@@ -107,3 +125,9 @@ Alias /.well-known/acme-challenge {0}
         os.chmod(name, 0o644)
 
         return response
+
+    def _set_up_include_directive(self, vhost):
+        """Includes override configuration to the beginning of VirtualHost.
+        Note that this include isn't added to Augeas search tree"""
+        self.configurator.parser.add_dir_beginning(vhost.path, "Include",
+                                                   self.challenge_conf)
diff --git a/certbot-apache/certbot_apache/parser.py b/certbot-apache/certbot_apache/parser.py
index 7715d2c35..d7da1e55e 100644
--- a/certbot-apache/certbot_apache/parser.py
+++ b/certbot-apache/certbot_apache/parser.py
@@ -332,6 +332,23 @@ class ApacheParser(object):
         else:
             self.aug.set(aug_conf_path + "/directive[last()]/arg", args)
 
+    def add_dir_beginning(self, aug_conf_path, dirname, args):
+        """Adds the directive to the beginning of defined aug_conf_path.
+
+        :param str aug_conf_path: Augeas configuration path to add directive
+        :param str dirname: Directive to add
+        :param args: Value of the directive. ie. Listen 443, 443 is arg
+        :type args: list or str
+        """
+        first_dir = aug_conf_path + "/directive[1]"
+        self.aug.insert(first_dir, "directive", True)
+        self.aug.set(first_dir, dirname)
+        if isinstance(args, list):
+            for i, value in enumerate(args, 1):
+                self.aug.set(first_dir + "/arg[%d]" % (i), value)
+        else:
+            self.aug.set(first_dir + "/arg", args)
+
     def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 204d9a76c..54b4ff208 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -119,9 +119,9 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertTrue(os.path.exists(challenge_dir))
 
     def _test_challenge_conf(self):
-        self.assertEqual(
-            len(self.config.parser.find_dir(
-                "Include", self.http.challenge_conf)), 1)
+        #self.assertEqual(
+        #    len(self.config.parser.find_dir(
+        #        "Include", self.http.challenge_conf)), 1)
 
         with open(self.http.challenge_conf) as f:
             conf_contents = f.read()
diff --git a/certbot-apache/certbot_apache/tests/parser_test.py b/certbot-apache/certbot_apache/tests/parser_test.py
index a9eb129c2..4496781c9 100644
--- a/certbot-apache/certbot_apache/tests/parser_test.py
+++ b/certbot-apache/certbot_apache/tests/parser_test.py
@@ -66,6 +66,23 @@ class BasicParserTest(util.ParserTest):
         for i, match in enumerate(matches):
             self.assertEqual(self.parser.aug.get(match), str(i + 1))
 
+    def test_add_dir_beginning(self):
+        aug_default = "/files" + self.parser.loc["default"]
+        self.parser.add_dir_beginning(aug_default,
+                                      "AddDirectiveBeginning",
+                                      "testBegin")
+
+        self.assertTrue(
+            self.parser.find_dir("AddDirectiveBeginning", "testBegin", aug_default))
+
+        self.assertEqual(
+            self.parser.aug.get(aug_default+"/directive[1]"),
+                                "AddDirectiveBeginning")
+        self.parser.add_dir_beginning(aug_default, "AddList", ["1", "2", "3", "4"])
+        matches = self.parser.find_dir("AddList", None, aug_default)
+        for i, match in enumerate(matches):
+            self.assertEqual(self.parser.aug.get(match), str(i + 1))
+
     def test_empty_arg(self):
         self.assertEquals(None,
                           self.parser.get_arg("/files/whatever/nonexistent"))

From f420b19492075b569eea349d5c6a4ec9bafa236f Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Tue, 16 Jan 2018 18:16:33 -0800
Subject: [PATCH 24/45] Apache HTTP01 Improvements

* Fix docstring quote spacing

* Remove unneeded directives

* Enable mod_rewrite

* Remove ifmod rewrite

* Use stricter rewriterule

* Uncomment tests

* Fix order args

* Remove S which doesn't seem to work across contexts

* Use double backslash to make pylint

* Fix enmod test

* Fix http-01 tests

* Test for rewrite

* check for Include in vhost

* add test_same_vhost

* Don't add includes twice

* Include default vhosts in search

* Respect port in find_best_http_vhost

* Add find_best_http_vhost port test

* Filter by port in http01
---
 certbot-apache/certbot_apache/configurator.py | 23 ++++---
 certbot-apache/certbot_apache/http_01.py      | 36 ++++------
 .../certbot_apache/tests/configurator_test.py | 18 ++++-
 .../certbot_apache/tests/http_01_test.py      | 67 ++++++++++++++-----
 .../apache2/sites-available/certbot.conf      |  1 +
 certbot-apache/certbot_apache/tests/util.py   |  2 +-
 6 files changed, 97 insertions(+), 50 deletions(-)

diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py
index 8f1aff8d7..b32eda921 100644
--- a/certbot-apache/certbot_apache/configurator.py
+++ b/certbot-apache/certbot_apache/configurator.py
@@ -436,19 +436,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 return True
         return False
 
-    def find_best_http_vhost(self, target):
+    def find_best_http_vhost(self, target, filter_defaults, port="80"):
         """Returns non-HTTPS vhost objects found from the Apache config
 
         :param str target: Domain name of the desired VirtualHost
+        :param bool filter_defaults: whether _default_ vhosts should be
+            included if it is the best match
+        :param str port: port number the vhost should be listening on
 
         :returns: VirtualHost object that's the best match for target name
         :rtype: `obj.VirtualHost` or None
         """
-        nonssl_vhosts = [i for i in self.vhosts if not i.ssl]
-        return self._find_best_vhost(target, nonssl_vhosts)
+        filtered_vhosts = []
+        for vhost in self.vhosts:
+            if any(a.is_wildcard() or a.get_port() == port for a in vhost.addrs) and not vhost.ssl:
+                filtered_vhosts.append(vhost)
+        return self._find_best_vhost(target, filtered_vhosts, filter_defaults)
 
-
-    def _find_best_vhost(self, target_name, vhosts=None):
+    def _find_best_vhost(self, target_name, vhosts=None, filter_defaults=True):
         """Finds the best vhost for a target_name.
 
         This does not upgrade a vhost to HTTPS... it only finds the most
@@ -457,6 +462,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :param str target_name: domain handled by the desired vhost
         :param vhosts: vhosts to consider
         :type vhosts: `collections.Iterable` of :class:`~certbot_apache.obj.VirtualHost`
+        :param bool filter_defaults: whether a vhost with a _default_
+            addr is acceptable
 
         :returns: VHost or None
 
@@ -497,8 +504,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         # No winners here... is there only one reasonable vhost?
         if best_candidate is None:
-            # reasonable == Not all _default_ addrs
-            vhosts = self._non_default_vhosts(vhosts)
+            if filter_defaults:
+                vhosts = self._non_default_vhosts(vhosts)
             # remove mod_macro hosts from reasonable vhosts
             reasonable_vhosts = [vh for vh
                                  in vhosts if vh.modmacro is False]
@@ -520,7 +527,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                   virtual host addresses
         :rtype: set
 
-    """
+        """
         all_names = set()
 
         vhost_macro = []
diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index a2cd43845..edcca8c9c 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -9,31 +9,19 @@ logger = logging.getLogger(__name__)
 class ApacheHttp01(common.TLSSNI01):
     """Class that performs HTTP-01 challenges within the Apache configurator."""
 
-    CONFIG_TEMPLATE_COMMON = """\
-        Alias /.well-known/acme-challenge {0}"
-
-        <IfModule mod_proxy.c>
-            ProxyPass "/.well-known/acme-challenge" !
-        </IfModule>
-    """
-
     CONFIG_TEMPLATE22 = """\
-        <IfModule mod_rewrite.c>
-            RewriteEngine on
-            RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [L,S=9999]
-        </IfModule>
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L]
 
         <Directory {0}>
-            Order allow deny
+            Order Allow,Deny
             Allow from all
         </Directory>
     """
 
     CONFIG_TEMPLATE24 = """\
-        <IfModule mod_rewrite.c>
-            RewriteEngine on
-            RewriteRule /.well-known/acme-challenge/(.*) {0}/$1 [END]
-        </IfModule>
+        RewriteEngine on
+        RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END]
 
         <Directory {0}>
             Require all granted
@@ -73,7 +61,7 @@ class ApacheHttp01(common.TLSSNI01):
         """Make sure that we have the needed modules available for http01"""
 
         if self.configurator.conf("handle-modules"):
-            needed_modules = ["alias"]
+            needed_modules = ["rewrite"]
             if self.configurator.version < (2, 4):
                 needed_modules.append("authz_host")
             else:
@@ -83,18 +71,22 @@ class ApacheHttp01(common.TLSSNI01):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
+        moded_vhosts = set()
         for chall in self.achalls:
-            vh = self.configurator.find_best_http_vhost(chall.domain)
-            if vh:
+            vh = self.configurator.find_best_http_vhost(
+                chall.domain, filter_defaults=False,
+                port=str(self.configurator.config.http01_port))
+            if vh and vh not in moded_vhosts:
                 self._set_up_include_directive(vh)
+                moded_vhosts.add(vh)
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
 
         if self.configurator.version < (2, 4):
-            config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE22
+            config_template = self.CONFIG_TEMPLATE22
         else:
-            config_template = self.CONFIG_TEMPLATE_COMMON + self.CONFIG_TEMPLATE24
+            config_template = self.CONFIG_TEMPLATE24
 
         config_text = config_template.format(self.challenge_dir)
 
diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py
index c846d2d42..530d75a92 100644
--- a/certbot-apache/certbot_apache/tests/configurator_test.py
+++ b/certbot-apache/certbot_apache/tests/configurator_test.py
@@ -126,7 +126,7 @@ class MultipleVhostsTest(util.ApacheTest):
         names = self.config.get_all_names()
         self.assertEqual(names, set(
             ["certbot.demo", "ocspvhost.com", "encryption-example.demo",
-             "nonsym.link", "vhost.in.rootconf"]
+             "nonsym.link", "vhost.in.rootconf", "www.certbot.demo"]
         ))
 
     @certbot_util.patch_get_utility()
@@ -146,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest):
 
         names = self.config.get_all_names()
         # Names get filtered, only 5 are returned
-        self.assertEqual(len(names), 7)
+        self.assertEqual(len(names), 8)
         self.assertTrue("zombo.com" in names)
         self.assertTrue("google.com" in names)
         self.assertTrue("certbot.demo" in names)
@@ -260,6 +260,20 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertRaises(
             errors.PluginError, self.config.choose_vhost, "none.com")
 
+    def test_find_best_http_vhost_default(self):
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True)
+        self.config.vhosts = [vh]
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh)
+
+    def test_find_best_http_vhost_port(self):
+        port = "8080"
+        vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr.fromstring("*:" + port)]),
+            False, True, "encryption-example.demo")
+        self.config.vhosts.append(vh)
+        self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh)
+
     def test_findbest_continues_on_short_domain(self):
         # pylint: disable=protected-access
         chosen_vhost = self.config._find_best_vhost("purple.com")
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 54b4ff208..768d904e8 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -22,8 +22,9 @@ class ApacheHttp01TestMeta(type):
         def _gen_test(num_achalls, minor_version):
             def _test(self):
                 achalls = self.achalls[:num_achalls]
+                vhosts = self.vhosts[:num_achalls]
                 self.config.version = (2, minor_version)
-                self.common_perform_test(achalls)
+                self.common_perform_test(achalls, vhosts)
             return _test
 
         for i in range(1, NUM_ACHALLS + 1):
@@ -43,16 +44,30 @@ class ApacheHttp01Test(util.ApacheTest):
 
         self.account_key = self.rsa512jwk
         self.achalls = []
+        self.vhosts = []
+        vhost_index = 0
         for i in range(NUM_ACHALLS):
+            domain = None
+            # Find a vhost with a name/alias we can use
+            for j in range(vhost_index + 1, len(self.config.vhosts)):
+                vhost = self.config.vhosts[j]
+                domain = vhost.name if vhost.name else next(iter(vhost.aliases), None)
+                if domain:
+                    self.vhosts.append(vhost)
+                    vhost_index = j + 1
+                    break
+            else:  # pragma: no cover
+                # If we didn't find a domain, we shouldn't continue the test.
+                self.fail("No usable vhost found")
+
             self.achalls.append(
                 achallenges.KeyAuthorizationAnnotatedChallenge(
                     challb=acme_util.chall_to_challb(
                         challenges.HTTP01(token=((chr(ord('a') + i) * 16))),
                         "pending"),
-                    domain="example{0}.com".format(i),
-                    account_key=self.account_key))
+                    domain=domain, account_key=self.account_key))
 
-        modules = ["alias", "authz_core", "authz_host"]
+        modules = ["rewrite", "authz_core", "authz_host"]
         for mod in modules:
             self.config.parser.modules.add("mod_{0}.c".format(mod))
             self.config.parser.modules.add(mod + "_module")
@@ -81,9 +96,9 @@ class ApacheHttp01Test(util.ApacheTest):
         self.assertEqual(enmod_calls[0][0][0], "authz_core")
 
     def common_enable_modules_test(self, mock_enmod):
-        """Tests enabling mod_alias and other modules."""
-        self.config.parser.modules.remove("alias_module")
-        self.config.parser.modules.remove("mod_alias.c")
+        """Tests enabling mod_rewrite and other modules."""
+        self.config.parser.modules.remove("rewrite_module")
+        self.config.parser.modules.remove("mod_rewrite.c")
 
         self.http.prepare_http01_modules()
 
@@ -91,14 +106,30 @@ class ApacheHttp01Test(util.ApacheTest):
         calls = mock_enmod.call_args_list
         other_calls = []
         for call in calls:
-            if "alias" != call[0][0]:
+            if "rewrite" != call[0][0]:
                 other_calls.append(call)
 
-        # If these lists are equal, we never enabled mod_alias
+        # If these lists are equal, we never enabled mod_rewrite
         self.assertNotEqual(calls, other_calls)
         return other_calls
 
-    def common_perform_test(self, achalls):
+    def test_same_vhost(self):
+        vhost = next(v for v in self.config.vhosts if v.name == "certbot.demo")
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain=vhost.name, account_key=self.account_key),
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'b' * 16))),
+                    "pending"),
+                domain=next(iter(vhost.aliases)), account_key=self.account_key)
+        ]
+        self.common_perform_test(achalls, [vhost])
+
+    def common_perform_test(self, achalls, vhosts):
         """Tests perform with the given achalls."""
         challenge_dir = self.http.challenge_dir
         self.assertFalse(os.path.exists(challenge_dir))
@@ -116,19 +147,21 @@ class ApacheHttp01Test(util.ApacheTest):
         for achall in achalls:
             self._test_challenge_file(achall)
 
+        for vhost in vhosts:
+            matches = self.config.parser.find_dir("Include",
+                                                  self.http.challenge_conf,
+                                                  vhost.path)
+            self.assertEqual(len(matches), 1)
+
         self.assertTrue(os.path.exists(challenge_dir))
 
     def _test_challenge_conf(self):
-        #self.assertEqual(
-        #    len(self.config.parser.find_dir(
-        #        "Include", self.http.challenge_conf)), 1)
-
         with open(self.http.challenge_conf) as f:
             conf_contents = f.read()
 
-        alias_fmt = "Alias /.well-known/acme-challenge {0}"
-        alias = alias_fmt.format(self.http.challenge_dir)
-        self.assertTrue(alias in conf_contents)
+        self.assertTrue("RewriteEngine on" in conf_contents)
+        self.assertTrue("RewriteRule" in conf_contents)
+        self.assertTrue(self.http.challenge_dir in conf_contents)
         if self.config.version < (2, 4):
             self.assertTrue("Allow from all" in conf_contents)
         else:
diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf
index b3147a523..965ca2222 100644
--- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf
+++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/certbot.conf
@@ -1,5 +1,6 @@
 <VirtualHost *:80>
 ServerName certbot.demo
+ServerAlias www.certbot.demo
 ServerAdmin webmaster@localhost
 
 DocumentRoot /var/www-certbot-reworld/static/
diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py
index 1ba1e2c34..1daaa00c5 100644
--- a/certbot-apache/certbot_apache/tests/util.py
+++ b/certbot-apache/certbot_apache/tests/util.py
@@ -170,7 +170,7 @@ def get_vh_truth(temp_dir, config_name):
                 os.path.join(prefix, "certbot.conf"),
                 os.path.join(aug_pre, "certbot.conf/VirtualHost"),
                 set([obj.Addr.fromstring("*:80")]), False, True,
-                "certbot.demo"),
+                "certbot.demo", aliases=["www.certbot.demo"]),
             obj.VirtualHost(
                 os.path.join(prefix, "mod_macro-example.conf"),
                 os.path.join(aug_pre,

From b8f288a3720fda4f2f1ef56ada30f887f5b41a63 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Wed, 17 Jan 2018 14:08:45 +0200
Subject: [PATCH 25/45] Add include to every VirtualHost if definite one not
 found based on name

---
 certbot-apache/certbot_apache/http_01.py      | 19 ++++++++++++++-----
 .../certbot_apache/tests/http_01_test.py      | 10 ++++++++++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index edcca8c9c..2b2b8e796 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -36,6 +36,7 @@ class ApacheHttp01(common.TLSSNI01):
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
+        self.moded_vhosts = set()
 
     def perform(self):
         """Perform all HTTP-01 challenges."""
@@ -71,14 +72,16 @@ class ApacheHttp01(common.TLSSNI01):
                     self.configurator.enable_mod(mod, temp=True)
 
     def _mod_config(self):
-        moded_vhosts = set()
         for chall in self.achalls:
             vh = self.configurator.find_best_http_vhost(
                 chall.domain, filter_defaults=False,
                 port=str(self.configurator.config.http01_port))
-            if vh and vh not in moded_vhosts:
+            if vh:
                 self._set_up_include_directive(vh)
-                moded_vhosts.add(vh)
+            else:
+                for vh in self.configurator.vhosts:
+                    if not vh.ssl:
+                        self._set_up_include_directive(vh)
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
@@ -121,5 +124,11 @@ class ApacheHttp01(common.TLSSNI01):
     def _set_up_include_directive(self, vhost):
         """Includes override configuration to the beginning of VirtualHost.
         Note that this include isn't added to Augeas search tree"""
-        self.configurator.parser.add_dir_beginning(vhost.path, "Include",
-                                                   self.challenge_conf)
+
+        if vhost not in self.moded_vhosts:
+            logger.debug(
+                "Adding a temporary challenge validation Include for name: %s " +
+                "in: %s", vhost.name, vhost.filep)
+            self.configurator.parser.add_dir_beginning(
+                vhost.path, "Include", self.challenge_conf)
+            self.moded_vhosts.add(vhost)
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 768d904e8..12f571354 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -129,6 +129,16 @@ class ApacheHttp01Test(util.ApacheTest):
         ]
         self.common_perform_test(achalls, [vhost])
 
+    def test_anonymous_vhost(self):
+        vhosts = [v for v in self.config.vhosts if not v.ssl]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="something.nonexistent", account_key=self.account_key)]
+        self.common_perform_test(achalls, vhosts)
+
     def common_perform_test(self, achalls, vhosts):
         """Tests perform with the given achalls."""
         challenge_dir = self.http.challenge_dir

From 2c379cd363143720644ff38eccc85a349c2f2b00 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Wed, 17 Jan 2018 08:01:44 -0800
Subject: [PATCH 26/45] Add a rewrite directive for the .well-known location so
 we don't hit existing rewrites (#5436)

---
 certbot-nginx/certbot_nginx/http_01.py |  5 +++++
 certbot-nginx/certbot_nginx/parser.py  | 28 +++++++++++++++++---------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py
index c25081ae0..2885e7ac0 100644
--- a/certbot-nginx/certbot_nginx/http_01.py
+++ b/certbot-nginx/certbot_nginx/http_01.py
@@ -191,3 +191,8 @@ class NginxHttp01(common.ChallengePerformer):
 
         self.configurator.parser.add_server_directives(vhost,
             location_directive, replace=False)
+
+        rewrite_directive = [['rewrite', ' ', '^(/.well-known/acme-challenge/.*)',
+                                ' ', '$1', ' ', 'break']]
+        self.configurator.parser.add_server_directives(vhost,
+            rewrite_directive, replace=False, insert_at_top=True)
diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py
index 5497f7e63..fbd6c0ade 100644
--- a/certbot-nginx/certbot_nginx/parser.py
+++ b/certbot-nginx/certbot_nginx/parser.py
@@ -276,7 +276,7 @@ class NginxParser(object):
 
         return False
 
-    def add_server_directives(self, vhost, directives, replace):
+    def add_server_directives(self, vhost, directives, replace, insert_at_top=False):
         """Add or replace directives in the server block identified by vhost.
 
         This method modifies vhost to be fully consistent with the new directives.
@@ -293,10 +293,12 @@ class NginxParser(object):
             whose information we use to match on
         :param list directives: The directives to add
         :param bool replace: Whether to only replace existing directives
+        :param bool insert_at_top: True if the directives need to be inserted at the top
+            of the server block instead of the bottom
 
         """
         self._modify_server_directives(vhost,
-            functools.partial(_add_directives, directives, replace))
+            functools.partial(_add_directives, directives, replace, insert_at_top))
 
     def remove_server_directives(self, vhost, directive_name, match_func=None):
         """Remove all directives of type directive_name.
@@ -521,7 +523,7 @@ def _is_ssl_on_directive(entry):
             len(entry) == 2 and entry[0] == 'ssl' and
             entry[1] == 'on')
 
-def _add_directives(directives, replace, block):
+def _add_directives(directives, replace, insert_at_top, block):
     """Adds or replaces directives in a config block.
 
     When replace=False, it's an error to try and add a nonrepeatable directive that already
@@ -535,17 +537,18 @@ def _add_directives(directives, replace, block):
 
     :param list directives: The new directives.
     :param bool replace: Described above.
+    :param bool insert_at_top: Described above.
     :param list block: The block to replace in
 
     """
     for directive in directives:
-        _add_directive(block, directive, replace)
+        _add_directive(block, directive, replace, insert_at_top)
     if block and '\n' not in block[-1]:  # could be "   \n  " or ["\n"] !
         block.append(nginxparser.UnspacedList('\n'))
 
 
 INCLUDE = 'include'
-REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location'])
+REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'location', 'rewrite'])
 COMMENT = ' managed by Certbot'
 COMMENT_BLOCK = [' ', '#', COMMENT]
 
@@ -597,7 +600,7 @@ def _find_location(block, directive_name, match_func=None):
     return next((index for index, line in enumerate(block) \
         if line and line[0] == directive_name and (match_func is None or match_func(line))), None)
 
-def _add_directive(block, directive, replace):
+def _add_directive(block, directive, replace, insert_at_top):
     """Adds or replaces a single directive in a config block.
 
     See _add_directives for more documentation.
@@ -619,7 +622,7 @@ def _add_directive(block, directive, replace):
             block[location] = directive
             comment_directive(block, location)
             return
-    # Append directive. Fail if the name is not a repeatable directive name,
+    # Append or prepend directive. Fail if the name is not a repeatable directive name,
     # and there is already a copy of that directive with a different value
     # in the config file.
 
@@ -652,8 +655,15 @@ def _add_directive(block, directive, replace):
                     _comment_out_directive(block, included_dir_loc, directive[1])
 
     if can_append(location, directive_name):
-        block.append(directive)
-        comment_directive(block, len(block) - 1)
+        if insert_at_top:
+            # Add a newline so the comment doesn't comment
+            # out existing directives
+            block.insert(0, nginxparser.UnspacedList('\n'))
+            block.insert(0, directive)
+            comment_directive(block, 0)
+        else:
+            block.append(directive)
+            comment_directive(block, len(block) - 1)
     elif block[location] != directive:
         raise errors.MisconfigurationError(err_fmt.format(directive, block[location]))
 

From e9b57e17833e19d36158ee7c1f9b7c684f118027 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Wed, 17 Jan 2018 08:02:10 -0800
Subject: [PATCH 27/45] Add (nonexistent) document root so we don't use the
 default value (#5437)

---
 certbot-nginx/certbot_nginx/http_01.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/certbot-nginx/certbot_nginx/http_01.py b/certbot-nginx/certbot_nginx/http_01.py
index 2885e7ac0..c0dec061a 100644
--- a/certbot-nginx/certbot_nginx/http_01.py
+++ b/certbot-nginx/certbot_nginx/http_01.py
@@ -155,10 +155,15 @@ class NginxHttp01(common.ChallengePerformer):
         addrs = self._default_listen_addresses()
         block = [['listen', ' ', addr.to_string(include_default=False)] for addr in addrs]
 
+        # Ensure we 404 on any other request by setting a root
+        document_root = os.path.join(
+            self.configurator.config.work_dir, "http_01_nonexistent")
+
         validation = achall.validation(achall.account_key)
         validation_path = self._get_validation_path(achall)
 
         block.extend([['server_name', ' ', achall.domain],
+                      ['root', ' ', document_root],
                       [['location', ' ', '=', ' ', validation_path],
                         [['default_type', ' ', 'text/plain'],
                          ['return', ' ', '200', ' ', validation]]]])

From bd231a3855c03d2943a2e3fe117605b4962726a8 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 17 Jan 2018 09:27:36 -0800
Subject: [PATCH 28/45] Error without vhosts and fix tests token type

---
 certbot-apache/certbot_apache/http_01.py      | 20 ++++++++++++++++---
 .../certbot_apache/tests/http_01_test.py      |  9 ++++++++-
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 2b2b8e796..4aa3215e6 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -2,6 +2,8 @@
 import logging
 import os
 
+from certbot import errors
+
 from certbot.plugins import common
 
 logger = logging.getLogger(__name__)
@@ -79,9 +81,8 @@ class ApacheHttp01(common.TLSSNI01):
             if vh:
                 self._set_up_include_directive(vh)
             else:
-                for vh in self.configurator.vhosts:
-                    if not vh.ssl:
-                        self._set_up_include_directive(vh)
+                for vh in self._relevant_vhosts():
+                    self._set_up_include_directive(vh)
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
@@ -97,6 +98,19 @@ class ApacheHttp01(common.TLSSNI01):
         with open(self.challenge_conf, "w") as new_conf:
             new_conf.write(config_text)
 
+    def _relevant_vhosts(self):
+        http01_port = str(self.configurator.config.http01_port)
+        relevant_vhosts = []
+        for vhost in self.configurator.vhosts:
+            if any(a.is_wildcard() or a.get_port() == http01_port for a in vhost.addrs):
+                if not vhost.ssl:
+                    relevant_vhosts.append(vhost)
+        if not relevant_vhosts:
+            raise errors.PluginError(
+                "Unable to find a virtual host listening on port {0}."
+                " Please add one.".format(http01_port))
+
+        return relevant_vhosts
 
     def _set_up_challenges(self):
         if not os.path.isdir(self.challenge_dir):
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 12f571354..652b65a09 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -6,6 +6,7 @@ import unittest
 from acme import challenges
 
 from certbot import achallenges
+from certbot import errors
 
 from certbot.tests import acme_util
 
@@ -63,7 +64,7 @@ class ApacheHttp01Test(util.ApacheTest):
             self.achalls.append(
                 achallenges.KeyAuthorizationAnnotatedChallenge(
                     challb=acme_util.chall_to_challb(
-                        challenges.HTTP01(token=((chr(ord('a') + i) * 16))),
+                        challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))),
                         "pending"),
                     domain=domain, account_key=self.account_key))
 
@@ -139,6 +140,12 @@ class ApacheHttp01Test(util.ApacheTest):
                 domain="something.nonexistent", account_key=self.account_key)]
         self.common_perform_test(achalls, vhosts)
 
+    def test_no_vhost(self):
+        for achall in self.achalls:
+            self.http.add_chall(achall)
+        self.config.config.http01_port = 12345
+        self.assertRaises(errors.PluginError, self.http.perform)
+
     def common_perform_test(self, achalls, vhosts):
         """Tests perform with the given achalls."""
         challenge_dir = self.http.challenge_dir

From 63136be2e55473a12a0de36ec6eaa72e9cd0ab70 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Wed, 17 Jan 2018 20:07:38 +0200
Subject: [PATCH 29/45] Make sure the HTTP tests do not use wrong vhosts for
 asserts

---
 certbot-apache/certbot_apache/tests/http_01_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 12f571354..af32697d1 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -161,7 +161,8 @@ class ApacheHttp01Test(util.ApacheTest):
             matches = self.config.parser.find_dir("Include",
                                                   self.http.challenge_conf,
                                                   vhost.path)
-            self.assertEqual(len(matches), 1)
+            if not vhost.ssl:
+                self.assertEqual(len(matches), 1)
 
         self.assertTrue(os.path.exists(challenge_dir))
 

From 522532dc300d4f6921320d1a56167b5b6372b562 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 17 Jan 2018 10:33:51 -0800
Subject: [PATCH 30/45] Improve no vhost error message

---
 certbot-apache/certbot_apache/http_01.py            | 6 ++++--
 certbot-apache/certbot_apache/tests/http_01_test.py | 6 +++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index 4aa3215e6..e463f3880 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -107,8 +107,10 @@ class ApacheHttp01(common.TLSSNI01):
                     relevant_vhosts.append(vhost)
         if not relevant_vhosts:
             raise errors.PluginError(
-                "Unable to find a virtual host listening on port {0}."
-                " Please add one.".format(http01_port))
+                "Unable to find a virtual host listening on port {0} which is"
+                " currently needed for Certbot to prove to the CA that you"
+                " control your domain. Please add a virtual host for port"
+                " {0}.".format(http01_port))
 
         return relevant_vhosts
 
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 00e05e9fc..ee8566396 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -165,10 +165,10 @@ class ApacheHttp01Test(util.ApacheTest):
             self._test_challenge_file(achall)
 
         for vhost in vhosts:
-            matches = self.config.parser.find_dir("Include",
-                                                  self.http.challenge_conf,
-                                                  vhost.path)
             if not vhost.ssl:
+                matches = self.config.parser.find_dir("Include",
+                                                      self.http.challenge_conf,
+                                                      vhost.path)
                 self.assertEqual(len(matches), 1)
 
         self.assertTrue(os.path.exists(challenge_dir))

From 1bb2cfadf7d18ee01f70ebb3b34cedf356946837 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 17 Jan 2018 15:34:34 -0800
Subject: [PATCH 31/45] hardcode vhosts and names for test (#5444)

---
 .../certbot_apache/tests/http_01_test.py      | 23 ++++++-------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index ee8566396..64a76649a 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -45,28 +45,19 @@ class ApacheHttp01Test(util.ApacheTest):
 
         self.account_key = self.rsa512jwk
         self.achalls = []
-        self.vhosts = []
-        vhost_index = 0
-        for i in range(NUM_ACHALLS):
-            domain = None
-            # Find a vhost with a name/alias we can use
-            for j in range(vhost_index + 1, len(self.config.vhosts)):
-                vhost = self.config.vhosts[j]
-                domain = vhost.name if vhost.name else next(iter(vhost.aliases), None)
-                if domain:
-                    self.vhosts.append(vhost)
-                    vhost_index = j + 1
-                    break
-            else:  # pragma: no cover
-                # If we didn't find a domain, we shouldn't continue the test.
-                self.fail("No usable vhost found")
+        vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/multiple_vhosts")
+        # Takes the vhosts for encryption-example.demo, certbot.demo, and
+        # vhost.in.rootconf
+        self.vhosts = [vh_truth[0], vh_truth[3], vh_truth[10]]
 
+        for i in range(NUM_ACHALLS):
             self.achalls.append(
                 achallenges.KeyAuthorizationAnnotatedChallenge(
                     challb=acme_util.chall_to_challb(
                         challenges.HTTP01(token=((chr(ord('a') + i).encode() * 16))),
                         "pending"),
-                    domain=domain, account_key=self.account_key))
+                    domain=self.vhosts[i].name, account_key=self.account_key))
 
         modules = ["rewrite", "authz_core", "authz_host"]
         for mod in modules:

From bf695d048dd1d3b14b1d521b5940d3c9a635fd64 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 17 Jan 2018 15:55:29 -0800
Subject: [PATCH 32/45] Release 0.21.0

---
 acme/setup.py                                 |   2 +-
 certbot-apache/setup.py                       |   2 +-
 certbot-auto                                  | 307 +++++++++++++-----
 certbot-compatibility-test/setup.py           |   2 +-
 certbot-dns-cloudflare/setup.py               |   2 +-
 certbot-dns-cloudxns/setup.py                 |   2 +-
 certbot-dns-digitalocean/setup.py             |   2 +-
 certbot-dns-dnsimple/setup.py                 |   2 +-
 certbot-dns-dnsmadeeasy/setup.py              |   2 +-
 certbot-dns-google/setup.py                   |   2 +-
 certbot-dns-luadns/setup.py                   |   2 +-
 certbot-dns-nsone/setup.py                    |   2 +-
 certbot-dns-rfc2136/setup.py                  |   2 +-
 certbot-dns-route53/setup.py                  |   2 +-
 certbot-nginx/setup.py                        |   2 +-
 certbot/__init__.py                           |   2 +-
 docs/cli-help.txt                             |  10 +-
 letsencrypt-auto                              | 307 +++++++++++++-----
 letsencrypt-auto-source/certbot-auto.asc      |  14 +-
 letsencrypt-auto-source/letsencrypt-auto      |  26 +-
 letsencrypt-auto-source/letsencrypt-auto.sig  | Bin 256 -> 256 bytes
 .../pieces/certbot-requirements.txt           |  24 +-
 22 files changed, 494 insertions(+), 224 deletions(-)

diff --git a/acme/setup.py b/acme/setup.py
index 7ebdfe46e..a3e5fd744 100644
--- a/acme/setup.py
+++ b/acme/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py
index 3270f2c79..6a874c813 100644
--- a/certbot-apache/setup.py
+++ b/certbot-apache/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-auto b/certbot-auto
index 444bee1b9..558c330b2 100755
--- a/certbot-auto
+++ b/certbot-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.20.0"
+LE_AUTO_VERSION="0.21.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
@@ -244,23 +246,42 @@ DeprecationBootstrap() {
   fi
 }
 
-
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
+  if [ "$USE_PYTHON_3" = 1 ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
+  if [ "$?" != "0" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -384,23 +405,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -408,15 +425,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -425,14 +442,20 @@ BootstrapRpmCommon() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -444,10 +467,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -455,9 +507,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -468,8 +519,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -477,16 +527,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -816,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -983,9 +1092,16 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+josepy==1.0.1 \
+    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
+    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
+# Using an older version of mock here prevents regressions of #5276.
+mock==1.3.0 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
     --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
 packaging==16.8 \
@@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-# Using an older version of mock here prevents regressions of #5276.
-mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1319,9 +1431,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1353,17 +1466,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1385,8 +1503,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1408,7 +1529,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1416,13 +1537,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def cert_none_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
@@ -1475,8 +1604,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."
diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py
index 1faf30643..1015fec47 100644
--- a/certbot-compatibility-test/setup.py
+++ b/certbot-compatibility-test/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 install_requires = [
     'certbot',
diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py
index 428271045..f7f1b0b38 100644
--- a/certbot-dns-cloudflare/setup.py
+++ b/certbot-dns-cloudflare/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py
index 4a103193f..cd6f9458e 100644
--- a/certbot-dns-cloudxns/setup.py
+++ b/certbot-dns-cloudxns/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py
index 23098d4b6..35659acac 100644
--- a/certbot-dns-digitalocean/setup.py
+++ b/certbot-dns-digitalocean/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py
index 4ed5a06ca..447a1c1f4 100644
--- a/certbot-dns-dnsimple/setup.py
+++ b/certbot-dns-dnsimple/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py
index 8a0b88aab..0def6d7cb 100644
--- a/certbot-dns-dnsmadeeasy/setup.py
+++ b/certbot-dns-dnsmadeeasy/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py
index b00bd1ac3..d51b9271b 100644
--- a/certbot-dns-google/setup.py
+++ b/certbot-dns-google/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py
index b8f50254e..35c018706 100644
--- a/certbot-dns-luadns/setup.py
+++ b/certbot-dns-luadns/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py
index 2a388e487..027b3bbae 100644
--- a/certbot-dns-nsone/setup.py
+++ b/certbot-dns-nsone/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py
index 78007afb5..5fb54922a 100644
--- a/certbot-dns-rfc2136/setup.py
+++ b/certbot-dns-rfc2136/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py
index 7d1eb0bc9..882834852 100644
--- a/certbot-dns-route53/setup.py
+++ b/certbot-dns-route53/setup.py
@@ -3,7 +3,7 @@ import sys
 from distutils.core import setup
 from setuptools import find_packages
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 install_requires = [
     'acme=={0}'.format(version),
diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py
index 2ad7aaf08..b5585bf42 100644
--- a/certbot-nginx/setup.py
+++ b/certbot-nginx/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0.dev0'
+version = '0.21.0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot/__init__.py b/certbot/__init__.py
index cbea701ee..b0a212646 100644
--- a/certbot/__init__.py
+++ b/certbot/__init__.py
@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.21.0.dev0'
+__version__ = '0.21.0'
diff --git a/docs/cli-help.txt b/docs/cli-help.txt
index abaa95b9b..f7318f0b3 100644
--- a/docs/cli-help.txt
+++ b/docs/cli-help.txt
@@ -107,7 +107,7 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/0.20.0 (certbot;
+                        "". (default: CertbotACMEClient/0.21.0 (certbot;
                         Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY
                         (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags
                         encoded in the user agent are: --duplicate, --force-
@@ -331,6 +331,14 @@ revoke:
   --reason {unspecified,keycompromise,affiliationchanged,superseded,cessationofoperation}
                         Specify reason for revoking certificate. (default:
                         unspecified)
+  --delete-after-revoke
+                        Delete certificates after revoking them. (default:
+                        None)
+  --no-delete-after-revoke
+                        Do not delete certificates after revoking them. This
+                        option should be used with caution because the 'renew'
+                        subcommand will attempt to renew undeleted revoked
+                        certificates. (default: None)
 
 register:
   Options for account registration & modification
diff --git a/letsencrypt-auto b/letsencrypt-auto
index 444bee1b9..558c330b2 100755
--- a/letsencrypt-auto
+++ b/letsencrypt-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.20.0"
+LE_AUTO_VERSION="0.21.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -68,10 +68,12 @@ for arg in "$@" ; do
       NO_BOOTSTRAP=1;;
     --help)
       HELP=1;;
-    --noninteractive|--non-interactive|renew)
-      ASSUME_YES=1;;
+    --noninteractive|--non-interactive)
+      NONINTERACTIVE=1;;
     --quiet)
       QUIET=1;;
+    renew)
+      ASSUME_YES=1;;
     --verbose)
       VERBOSE=1;;
     -[!-]*)
@@ -93,7 +95,7 @@ done
 
 if [ $BASENAME = "letsencrypt-auto" ]; then
   # letsencrypt-auto does not respect --help or --yes for backwards compatibility
-  ASSUME_YES=1
+  NONINTERACTIVE=1
   HELP=0
 fi
 
@@ -244,23 +246,42 @@ DeprecationBootstrap() {
   fi
 }
 
-
+MIN_PYTHON_VERSION="2.6"
+MIN_PYVER=$(echo "$MIN_PYTHON_VERSION" | sed 's/\.//')
+# Sets LE_PYTHON to Python version string and PYVER to the first two
+# digits of the python version
 DeterminePythonVersion() {
-  for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    error "Cannot find any Pythons; please install one!"
-    exit 1
+  # Arguments: "NOCRASH" if we shouldn't crash if we don't find a good python
+  #
+  # If no Python is found, PYVER is set to 0.
+  if [ "$USE_PYTHON_3" = 1 ]; then
+    for LE_PYTHON in "$LE_PYTHON" python3; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  else
+    for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do
+      # Break (while keeping the LE_PYTHON value) if found.
+      $EXISTS "$LE_PYTHON" > /dev/null && break
+    done
+  fi
+  if [ "$?" != "0" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "Cannot find any Pythons; please install one!"
+      exit 1
+    else
+      PYVER=0
+      return 0
+    fi
   fi
-  export LE_PYTHON
 
   PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
-  if [ "$PYVER" -lt 26 ]; then
-    error "You have an ancient version of Python entombed in your operating system..."
-    error "This isn't going to work; you'll need at least version 2.6."
-    exit 1
+  if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+    if [ "$1" != "NOCRASH" ]; then
+      error "You have an ancient version of Python entombed in your operating system..."
+      error "This isn't going to work; you'll need at least version $MIN_PYTHON_VERSION."
+      exit 1
+    fi
   fi
 }
 
@@ -384,23 +405,19 @@ BootstrapDebCommon() {
   fi
 }
 
-# If new packages are installed by BootstrapRpmCommon below, this version
-# number must be increased.
-BOOTSTRAP_RPM_COMMON_VERSION=1
-
-BootstrapRpmCommon() {
-  # Tested with:
-  #   - Fedora 20, 21, 22, 23 (x64)
-  #   - Centos 7 (x64: on DigitalOcean droplet)
-  #   - CentOS 7 Minimal install in a Hyper-V VM
-  #   - CentOS 6 (EPEL must be installed manually)
+# If new packages are installed by BootstrapRpmCommonBase below, version
+# numbers in rpm_common.sh and rpm_python3.sh must be increased.
 
+# Sets TOOL to the name of the package manager
+# Sets appropriate values for YES_FLAG and QUIET_FLAG based on $ASSUME_YES and $QUIET_FLAG.
+# Enables EPEL if applicable and possible.
+InitializeRPMCommonBase() {
   if type dnf 2>/dev/null
   then
-    tool=dnf
+    TOOL=dnf
   elif type yum 2>/dev/null
   then
-    tool=yum
+    TOOL=yum
 
   else
     error "Neither yum nor dnf found. Aborting bootstrap!"
@@ -408,15 +425,15 @@ BootstrapRpmCommon() {
   fi
 
   if [ "$ASSUME_YES" = 1 ]; then
-    yes_flag="-y"
+    YES_FLAG="-y"
   fi
   if [ "$QUIET" = 1 ]; then
     QUIET_FLAG='--quiet'
   fi
 
-  if ! $tool list *virtualenv >/dev/null 2>&1; then
+  if ! $TOOL list *virtualenv >/dev/null 2>&1; then
     echo "To use Certbot, packages from the EPEL repository need to be installed."
-    if ! $tool list epel-release >/dev/null 2>&1; then
+    if ! $TOOL list epel-release >/dev/null 2>&1; then
       error "Enable the EPEL repository and try running Certbot again."
       exit 1
     fi
@@ -425,14 +442,20 @@ BootstrapRpmCommon() {
       sleep 1s
       /bin/echo -ne "\e[0K\rEnabling the EPEL repository in 2 seconds..."
       sleep 1s
-      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 seconds..."
+      /bin/echo -e "\e[0K\rEnabling the EPEL repository in 1 second..."
       sleep 1s
     fi
-    if ! $tool install $yes_flag $QUIET_FLAG epel-release; then
+    if ! $TOOL install $YES_FLAG $QUIET_FLAG epel-release; then
       error "Could not enable EPEL. Aborting bootstrap!"
       exit 1
     fi
   fi
+}
+
+BootstrapRpmCommonBase() {
+  # Arguments: whitespace-delimited python packages to install
+
+  InitializeRPMCommonBase # This call is superfluous in practice
 
   pkgs="
     gcc
@@ -444,10 +467,39 @@ BootstrapRpmCommon() {
     ca-certificates
   "
 
-  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
-  if $tool list python >/dev/null 2>&1; then
+  # Add the python packages
+  pkgs="$pkgs
+    $1
+  "
+
+  if $TOOL list installed "httpd" >/dev/null 2>&1; then
     pkgs="$pkgs
-      python
+      mod_ssl
+    "
+  fi
+
+  if ! $TOOL install $YES_FLAG $QUIET_FLAG $pkgs; then
+    error "Could not install OS dependencies. Aborting bootstrap!"
+    exit 1
+  fi
+}
+
+# If new packages are installed by BootstrapRpmCommon below, this version
+# number must be increased.
+BOOTSTRAP_RPM_COMMON_VERSION=1
+
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 20, 21, 22, 23 (x64)
+  #   - Centos 7 (x64: on DigitalOcean droplet)
+  #   - CentOS 7 Minimal install in a Hyper-V VM
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # Most RPM distros use the "python" or "python-" naming convention.  Let's try that first.
+  if $TOOL list python >/dev/null 2>&1; then
+    python_pkgs="$python
       python-devel
       python-virtualenv
       python-tools
@@ -455,9 +507,8 @@ BootstrapRpmCommon() {
     "
   # Fedora 26 starts to use the prefix python2 for python2 based packages.
   # this elseif is theoretically for any Fedora over version 26:
-  elif $tool list python2 >/dev/null 2>&1; then
-    pkgs="$pkgs
-      python2
+  elif $TOOL list python2 >/dev/null 2>&1; then
+    python_pkgs="$python2
       python2-libs
       python2-setuptools
       python2-devel
@@ -468,8 +519,7 @@ BootstrapRpmCommon() {
   # Some distros and older versions of current distros use a "python27"
   # instead of the "python" or "python-" naming convention.
   else
-    pkgs="$pkgs
-      python27
+    python_pkgs="$python27
       python27-devel
       python27-virtualenv
       python27-tools
@@ -477,16 +527,31 @@ BootstrapRpmCommon() {
     "
   fi
 
-  if $tool list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
+  BootstrapRpmCommonBase "$python_pkgs"
+}
 
-  if ! $tool install $yes_flag $QUIET_FLAG $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
+# If new packages are installed by BootstrapRpmPython3 below, this version
+# number must be increased.
+BOOTSTRAP_RPM_PYTHON3_VERSION=1
+
+BootstrapRpmPython3() {
+  # Tested with:
+  #   - CentOS 6
+
+  InitializeRPMCommonBase
+
+  # EPEL uses python34
+  if $TOOL list python34 >/dev/null 2>&1; then
+    python_pkgs="python34
+      python34-devel
+      python34-tools
+    "
+  else
+    error "No supported Python package available to install. Aborting bootstrap!"
     exit 1
   fi
+
+  BootstrapRpmCommonBase "$python_pkgs"
 }
 
 # If new packages are installed by BootstrapSuseCommon below, this version
@@ -715,11 +780,27 @@ elif [ -f /etc/mageia-release ]; then
   }
   BOOTSTRAP_VERSION="BootstrapMageiaCommon $BOOTSTRAP_MAGEIA_COMMON_VERSION"
 elif [ -f /etc/redhat-release ]; then
-  Bootstrap() {
-    BootstrapMessage "RedHat-based OSes"
-    BootstrapRpmCommon
-  }
-  BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  # Run DeterminePythonVersion to decide on the basis of available Python versions
+  # whether to use 2.x or 3.x on RedHat-like systems.
+  # Then, revert LE_PYTHON to its previous state.
+  prev_le_python="$LE_PYTHON"
+  unset LE_PYTHON
+  DeterminePythonVersion "NOCRASH"
+  if [ "$PYVER" -eq 26 ]; then
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes that will use Python3"
+      BootstrapRpmPython3
+    }
+    USE_PYTHON_3=1
+    BOOTSTRAP_VERSION="BootstrapRpmPython3 $BOOTSTRAP_RPM_PYTHON3_VERSION"
+  else
+    Bootstrap() {
+      BootstrapMessage "RedHat-based OSes"
+      BootstrapRpmCommon
+    }
+    BOOTSTRAP_VERSION="BootstrapRpmCommon $BOOTSTRAP_RPM_COMMON_VERSION"
+  fi
+  LE_PYTHON="$prev_le_python"
 elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then
   Bootstrap() {
     BootstrapMessage "openSUSE-based OSes"
@@ -816,7 +897,11 @@ TempDir() {
   mktemp -d 2>/dev/null || mktemp -d -t 'le'  # Linux || macOS
 }
 
-
+# Returns 0 if a letsencrypt installation exists at $OLD_VENV_PATH, otherwise,
+# returns a non-zero number.
+OldVenvExists() {
+    [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ]
+}
 
 if [ "$1" = "--le-auto-phase2" ]; then
   # Phase 2: Create venv, install LE, and run.
@@ -824,14 +909,26 @@ if [ "$1" = "--le-auto-phase2" ]; then
   shift 1  # the --le-auto-phase2 arg
   SetPrevBootstrapVersion
 
+  if [ -z "$PHASE_1_VERSION" -a "$USE_PYTHON_3" = 1 ]; then
+    unset LE_PYTHON
+  fi
+
   INSTALLED_VERSION="none"
-  if [ -d "$VENV_PATH" ]; then
+  if [ -d "$VENV_PATH" ] || OldVenvExists; then
     # If the selected Bootstrap function isn't a noop and it differs from the
     # previously used version
     if [ -n "$BOOTSTRAP_VERSION" -a "$BOOTSTRAP_VERSION" != "$PREV_BOOTSTRAP_VERSION" ]; then
       # if non-interactive mode or stdin and stdout are connected to a terminal
       if [ \( "$NONINTERACTIVE" = 1 \) -o \( \( -t 0 \) -a \( -t 1 \) \) ]; then
-        rm -rf "$VENV_PATH"
+        if [ -d "$VENV_PATH" ]; then
+          rm -rf "$VENV_PATH"
+        fi
+        # In the case the old venv was just a symlink to the new one,
+        # OldVenvExists is now false because we deleted the venv at VENV_PATH.
+        if OldVenvExists; then
+          rm -rf "$OLD_VENV_PATH"
+          ln -s "$VENV_PATH" "$OLD_VENV_PATH"
+        fi
         RerunWithArgs "$@"
       else
         error "Skipping upgrade because new OS dependencies may need to be installed."
@@ -841,6 +938,10 @@ if [ "$1" = "--le-auto-phase2" ]; then
         error "install any required packages."
         # Set INSTALLED_VERSION to be the same so we don't update the venv
         INSTALLED_VERSION="$LE_AUTO_VERSION"
+        # Continue to use OLD_VENV_PATH if the new venv doesn't exist
+        if [ ! -d "$VENV_PATH" ]; then
+          VENV_BIN="$OLD_VENV_PATH/bin"
+        fi
       fi
     elif [ -f "$VENV_BIN/letsencrypt" ]; then
       # --version output ran through grep due to python-cryptography DeprecationWarnings
@@ -858,10 +959,18 @@ if [ "$1" = "--le-auto-phase2" ]; then
     say "Creating virtual environment..."
     DeterminePythonVersion
     rm -rf "$VENV_PATH"
-    if [ "$VERBOSE" = 1 ]; then
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+    if [ "$PYVER" -le 27 ]; then
+      if [ "$VERBOSE" = 1 ]; then
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH"
+      else
+        virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      fi
     else
-      virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null
+      if [ "$VERBOSE" = 1 ]; then
+        "$LE_PYTHON" -m venv "$VENV_PATH"
+      else
+        "$LE_PYTHON" -m venv "$VENV_PATH" > /dev/null
+      fi
     fi
 
     if [ -n "$BOOTSTRAP_VERSION" ]; then
@@ -983,9 +1092,16 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
+josepy==1.0.1 \
+    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
+    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
+# Using an older version of mock here prevents regressions of #5276.
+mock==1.3.0 \
+    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
+    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 ordereddict==1.1 \
     --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
 packaging==16.8 \
@@ -1062,10 +1178,6 @@ zope.interface==4.1.3 \
     --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \
     --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \
     --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392
-# Using an older version of mock here prevents regressions of #5276.
-mock==1.3.0 \
-    --hash=sha256:3f573a18be94de886d1191f27c168427ef693e8dcfcecf95b170577b2eb69cbb \
-    --hash=sha256:1e247dbecc6ce057299eb7ee019ad68314bb93152e81d9a6110d35f4d5eca0f6
 
 # Contains the requirements for the letsencrypt package.
 #
@@ -1078,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
@@ -1319,9 +1431,10 @@ else
   # upgrading. Phase 1 checks the version of the latest release of
   # certbot-auto (which is always the same as that of the certbot
   # package). Phase 2 checks the version of the locally installed certbot.
+  export PHASE_1_VERSION="$LE_AUTO_VERSION"
 
   if [ ! -f "$VENV_BIN/letsencrypt" ]; then
-    if [ -z "$OLD_VENV_PATH" -o ! -f "$OLD_VENV_PATH/bin/letsencrypt" ]; then
+    if ! OldVenvExists; then
       if [ "$HELP" = 1 ]; then
         echo "$USAGE"
         exit 0
@@ -1353,17 +1466,22 @@ On failure, return non-zero.
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, unicode_literals
 
 from distutils.version import LooseVersion
 from json import loads
 from os import devnull, environ
 from os.path import dirname, join
 import re
+import ssl
 from subprocess import check_call, CalledProcessError
 from sys import argv, exit
-from urllib2 import build_opener, HTTPHandler, HTTPSHandler
-from urllib2 import HTTPError, URLError
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+    from urllib2 import HTTPError, URLError
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+    from urllib.error import HTTPError, URLError
 
 PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq
@@ -1385,8 +1503,11 @@ class HttpsGetter(object):
     def __init__(self):
         """Build an HTTPS opener."""
         # Based on pip 1.4.1's URLOpener
-        # This verifies certs on only Python >=2.7.9.
-        self._opener = build_opener(HTTPSHandler())
+        # This verifies certs on only Python >=2.7.9, and when NO_CERT_VERIFY isn't set.
+        if environ.get('NO_CERT_VERIFY') == '1' and hasattr(ssl, 'SSLContext'):
+            self._opener = build_opener(HTTPSHandler(context=cert_none_context()))
+        else:
+            self._opener = build_opener(HTTPSHandler())
         # Strip out HTTPHandler to prevent MITM spoof:
         for handler in self._opener.handlers:
             if isinstance(handler, HTTPHandler):
@@ -1408,7 +1529,7 @@ class HttpsGetter(object):
 
 def write(contents, dir, filename):
     """Write something to a file in a certain directory."""
-    with open(join(dir, filename), 'w') as file:
+    with open(join(dir, filename), 'wb') as file:
         file.write(contents)
 
 
@@ -1416,13 +1537,13 @@ def latest_stable_version(get):
     """Return the latest stable release of letsencrypt."""
     metadata = loads(get(
         environ.get('LE_AUTO_JSON_URL',
-                    'https://pypi.python.org/pypi/certbot/json')))
+                    'https://pypi.python.org/pypi/certbot/json')).decode('UTF-8'))
     # metadata['info']['version'] actually returns the latest of any kind of
     # release release, contrary to https://wiki.python.org/moin/PyPIJSON.
     # The regex is a sufficient regex for picking out prereleases for most
     # packages, LE included.
     return str(max(LooseVersion(r) for r
-                   in metadata['releases'].iterkeys()
+                   in metadata['releases'].keys()
                    if re.match('^[0-9.]+$', r)))
 
 
@@ -1439,7 +1560,7 @@ def verified_new_le_auto(get, tag, temp_dir):
         'letsencrypt-auto-source/') % tag
     write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto')
     write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig')
-    write(PUBLIC_KEY, temp_dir, 'public_key.pem')
+    write(PUBLIC_KEY.encode('UTF-8'), temp_dir, 'public_key.pem')
     try:
         with open(devnull, 'w') as dev_null:
             check_call(['openssl', 'dgst', '-sha256', '-verify',
@@ -1454,6 +1575,14 @@ def verified_new_le_auto(get, tag, temp_dir):
                             "certbot-auto.", exc)
 
 
+def cert_none_context():
+    """Create a SSLContext object to not check hostname."""
+    # PROTOCOL_TLS isn't available before 2.7.13 but this code is for 2.7.9+, so use this.
+    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+    context.verify_mode = ssl.CERT_NONE
+    return context
+
+
 def main():
     get = HttpsGetter().get
     flag = argv[1]
@@ -1475,8 +1604,10 @@ if __name__ == '__main__':
 
 UNLIKELY_EOF
     # ---------------------------------------------------------------------------
-    DeterminePythonVersion
-    if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
+    DeterminePythonVersion "NOCRASH"
+    if [ "$PYVER" -lt "$MIN_PYVER" ]; then
+      error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates."
+    elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then
       error "WARNING: unable to check for updates."
     elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then
       say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..."
diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc
index eeab78cd6..9df8013a0 100644
--- a/letsencrypt-auto-source/certbot-auto.asc
+++ b/letsencrypt-auto-source/certbot-auto.asc
@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
-iQEcBAABCAAGBQJaKHMlAAoJEE0XyZXNl3Xy6OEH/iPg6D6+zco4NHMwxYIcTWVt
-XE4u3CjuLcEVsvEnJYNSA48NHyi9rIqMHd+IneLU+lCG2D7eBsisNNyVPIgHktTf
-p9i0WoZB+axe1glv9FJSZvjvr2d/ic4/wYHBF1c+szb9p8Z7o5Lhqa9/gtLJ/SZX
-OGU0wok4hPIB6emq5zvmi/+r1AiOECXE26lZ0STp6wDkvz+ahTJSk6UaPCDY+Az4
-X2VmnRSks/gk7Q8cloFnyiPXyFMQHdGIBRrIXsSix90QqmNUF7iYb8sbHksU23EI
-/LmIwSJlDm6KNOO2nllBB/uIg2ki7g0z7R4uf7XF4im+P95PAL/tQQ45lVj8DXE=
-=Is56
+iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n
+e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW
+AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL
+RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn
+IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf
+yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo=
+=d/hS
 -----END PGP SIGNATURE-----
diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index b86517bb6..558c330b2 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.0.dev0"
+LE_AUTO_VERSION="0.21.0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -1190,18 +1190,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig
index e276aae5353e8c5bdf78fc91950e60649baa241f..2e52f03fdf9a1fbe2049d89f071a583bcf919976 100644
GIT binary patch
literal 256
zcmV+b0ssD{xCAf=x;KV2@d03ctAnF7mn+aqUc{p=)VXkyhWU!~cN;%G^8WY$S?ta1
z2hmQE4T`~+Zwq%*K~cAA%;XQy8S-ox4a$2d0@tMjK|wQYPjC$Cv}rUZ3(4Cmz7$?G
zU;e#Fi7%+uTC73=Hbzf*EfB0nOu(b3_b6<Q!!|zvZ7y0Gq&)xES&(8)`G1!t?3#Mf
zkkXdN{8n>CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&*
zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz
GZCemMz<GWE

literal 256
zcmV+b0ssC;bi>+EWKAt%U!XWg-L;RUoOlOZsAFb4Ic}LGE#!b+HMXRo%-QglHm~xc
z7rA=rPZ17(d9O44<nV7T<ik#Wft{1qx#_wSo4~TsZY6~acXuZzeQ`VG%&<BN2MkT8
zG_B0w?Od#^K))#+fUqsL2je=&x_U$%m01`Po5A7!kv;I6?4PYFhTtcj;5>~^dV!y)
z3$?Ok9e8=A1v3-fBldYu$u&26@w%#BftaFQ;t)7s73=9Vm2g;i>-a0vSdi>_B;&sD
z-BgJaKSacB+~@MWWUpduDsx8T<c)}{s)8bQeWtNzlp}tFiD*JkvBw?kjVpRd{yhk!
GSIJ(NDS?*&

diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt
index 74336de7b..406086fb5 100644
--- a/letsencrypt-auto-source/pieces/certbot-requirements.txt
+++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt
@@ -1,12 +1,12 @@
-certbot==0.20.0 \
-    --hash=sha256:c6b6bd288700898d1eb31a65b605e3a5fc10f1e3213ce468207d76a2decb9d35 \
-    --hash=sha256:cabf505b64fb400c4239dcdbaeb882079477eb6a8442268596a8791b9e34de88
-acme==0.20.0 \
-    --hash=sha256:8b0cee192c0d76d6f4045bdb14b3cfd29d9720e0dad2046794a2a555f1eaccb7 \
-    --hash=sha256:45121aed6c8cc2f31896ac1083068dfdeb613f3edeff9576dc0d10632ea5a3d5
-certbot-apache==0.20.0 \
-    --hash=sha256:f7e4dbc154d2e9d1461118b6dd3dbd16f6892da468f060eeaa162aff673347e2 \
-    --hash=sha256:0ba499706451ffbccb172bcf93d6ef4c6cc8599157077a4fa6dfbe5a83c7921f
-certbot-nginx==0.20.0 \
-    --hash=sha256:b6e372e8740b20dd9bd63837646157ac97b3c9a65affd3954571b8e872ae9ecf \
-    --hash=sha256:6379fdf20d9a7651fe30bb8d4b828cbea178cc263d7af5a380fc4508d793b9ae
+certbot==0.21.0 \
+    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
+    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
+acme==0.21.0 \
+    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
+    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
+certbot-apache==0.21.0 \
+    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
+    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
+certbot-nginx==0.21.0 \
+    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
+    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642

From 325a97c1ed47b1cd6757e46733eeed4e251296be Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 17 Jan 2018 15:55:41 -0800
Subject: [PATCH 33/45] Bump version to 0.22.0

---
 acme/setup.py                            | 2 +-
 certbot-apache/setup.py                  | 2 +-
 certbot-compatibility-test/setup.py      | 2 +-
 certbot-dns-cloudflare/setup.py          | 2 +-
 certbot-dns-cloudxns/setup.py            | 2 +-
 certbot-dns-digitalocean/setup.py        | 2 +-
 certbot-dns-dnsimple/setup.py            | 2 +-
 certbot-dns-dnsmadeeasy/setup.py         | 2 +-
 certbot-dns-google/setup.py              | 2 +-
 certbot-dns-luadns/setup.py              | 2 +-
 certbot-dns-nsone/setup.py               | 2 +-
 certbot-dns-rfc2136/setup.py             | 2 +-
 certbot-dns-route53/setup.py             | 2 +-
 certbot-nginx/setup.py                   | 2 +-
 certbot/__init__.py                      | 2 +-
 letsencrypt-auto-source/letsencrypt-auto | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/acme/setup.py b/acme/setup.py
index a3e5fd744..ce426cf74 100644
--- a/acme/setup.py
+++ b/acme/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py
index 6a874c813..38f41e9f1 100644
--- a/certbot-apache/setup.py
+++ b/certbot-apache/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py
index 1015fec47..8f9f897cf 100644
--- a/certbot-compatibility-test/setup.py
+++ b/certbot-compatibility-test/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 install_requires = [
     'certbot',
diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py
index f7f1b0b38..612e7259f 100644
--- a/certbot-dns-cloudflare/setup.py
+++ b/certbot-dns-cloudflare/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py
index cd6f9458e..3157400c6 100644
--- a/certbot-dns-cloudxns/setup.py
+++ b/certbot-dns-cloudxns/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py
index 35659acac..1a68400fa 100644
--- a/certbot-dns-digitalocean/setup.py
+++ b/certbot-dns-digitalocean/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py
index 447a1c1f4..35de47308 100644
--- a/certbot-dns-dnsimple/setup.py
+++ b/certbot-dns-dnsimple/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py
index 0def6d7cb..a946d00a4 100644
--- a/certbot-dns-dnsmadeeasy/setup.py
+++ b/certbot-dns-dnsmadeeasy/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py
index d51b9271b..8585fc848 100644
--- a/certbot-dns-google/setup.py
+++ b/certbot-dns-google/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py
index 35c018706..4fec37e29 100644
--- a/certbot-dns-luadns/setup.py
+++ b/certbot-dns-luadns/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py
index 027b3bbae..dca9ebf27 100644
--- a/certbot-dns-nsone/setup.py
+++ b/certbot-dns-nsone/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py
index 5fb54922a..bfa72b50b 100644
--- a/certbot-dns-rfc2136/setup.py
+++ b/certbot-dns-rfc2136/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py
index 882834852..8df687972 100644
--- a/certbot-dns-route53/setup.py
+++ b/certbot-dns-route53/setup.py
@@ -3,7 +3,7 @@ import sys
 from distutils.core import setup
 from setuptools import find_packages
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 install_requires = [
     'acme=={0}'.format(version),
diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py
index b5585bf42..152f77de8 100644
--- a/certbot-nginx/setup.py
+++ b/certbot-nginx/setup.py
@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.21.0'
+version = '0.22.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
diff --git a/certbot/__init__.py b/certbot/__init__.py
index b0a212646..2869d29b0 100644
--- a/certbot/__init__.py
+++ b/certbot/__init__.py
@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.21.0'
+__version__ = '0.22.0.dev0'
diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 558c330b2..9c8ccf344 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.0"
+LE_AUTO_VERSION="0.22.0.dev0"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates

From 103039ca40c3a300816c5a957a0bf13a033234de Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@eff.org>
Date: Wed, 17 Jan 2018 17:46:56 -0800
Subject: [PATCH 34/45] Add 0.21.0 changelog

---
 CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4acfc0401..f6f7be9fe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,51 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.21.0 - 2018-01-17
+
+### Added
+
+* Support for the HTTP-01 challenge type was added to our Apache and Nginx
+  plugins. For those not aware, Let's Encrypt disabled the TLS-SNI-01 challenge
+  type which was what was previously being used by our Apache and Nginx plugins
+  last week due to a security issue. For more information about Let's Encrypt's
+  change, click
+  [here](https://community.letsencrypt.org/t/2018-01-11-update-regarding-acme-tls-sni-and-shared-hosting-infrastructure/50188).
+  Our Apache and Nginx plugins will automatically switch to use HTTP-01 so no
+  changes need to be made to your Certbot configuration, however, you should
+  make sure your server is accessible on port 80 and isn't behind an external
+  proxy doing things like redirecting all traffic from HTTP to HTTPS. HTTP to
+  HTTPS redirects inside Apache and Nginx are fine.
+* IPv6 support was added to the Nginx plugin.
+* Support for automatically creating server blocks based on the default server
+  block was added to the Nginx plugin.
+* The flags --delete-after-revoke and --no-delete-after-revoke were added
+  allowing users to control whether the revoke subcommand also deletes the
+  certificates it is revoking.
+
+### Changed
+
+* We deprecated support for Python 2.6 and Python 3.3 in Certbot and its ACME
+  library. Support for these versions of Python will be removed in the next
+  major release of Certbot. If you are using certbot-auto on a RHEL 6 based
+  system, it will guide you through the process of installing Python 3.
+* We split our implementation of JOSE (Javascript Object Signing and
+  Encryption) out of our ACME library and into a separate package named josepy.
+  This package is available on [PyPI](https://pypi.python.org/pypi/josepy) and
+  on [GitHub](https://github.com/certbot/josepy).
+* We updated the ciphersuites used in Apache to the new [values recommended by
+  Mozilla](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29).
+  The major change here is adding ChaCha20 to the list of supported
+  ciphersuites.
+
+### Fixed
+
+* An issue with our Apache plugin on Gentoo due to differences in their
+  apache2ctl command have been resolved.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/47?closed=1
+
 ## 0.20.0 - 2017-12-06
 
 ### Added

From b0aa8b7c0bcddb0837d9a57d021152bc05473b8e Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joohoi@users.noreply.github.com>
Date: Wed, 24 Jan 2018 02:46:36 +0200
Subject: [PATCH 35/45] Work around Basic Authentication for challenge dir in
 Apache (#5461)

Unfortunately, the way that Apache merges the configuration directives is different for mod_rewrite and <Location> / <Directory> directives.

To work around basic auth in VirtualHosts, the challenge override Include had to be split in two. The first part handles overrides for RewriteRule and the other part will handle overrides for <Directory> and <Location> directives.
---
 certbot-apache/certbot_apache/http_01.py      | 60 +++++++++++++------
 .../certbot_apache/tests/http_01_test.py      | 24 +++++---
 2 files changed, 58 insertions(+), 26 deletions(-)

diff --git a/certbot-apache/certbot_apache/http_01.py b/certbot-apache/certbot_apache/http_01.py
index e463f3880..cce93a646 100644
--- a/certbot-apache/certbot_apache/http_01.py
+++ b/certbot-apache/certbot_apache/http_01.py
@@ -11,30 +11,43 @@ logger = logging.getLogger(__name__)
 class ApacheHttp01(common.TLSSNI01):
     """Class that performs HTTP-01 challenges within the Apache configurator."""
 
-    CONFIG_TEMPLATE22 = """\
+    CONFIG_TEMPLATE22_PRE = """\
         RewriteEngine on
         RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [L]
 
+    """
+    CONFIG_TEMPLATE22_POST = """\
         <Directory {0}>
             Order Allow,Deny
             Allow from all
         </Directory>
+        <Location /.well-known/acme-challenge>
+            Order Allow,Deny
+            Allow from all
+        </Location>
     """
 
-    CONFIG_TEMPLATE24 = """\
+    CONFIG_TEMPLATE24_PRE = """\
         RewriteEngine on
         RewriteRule ^/\\.well-known/acme-challenge/([A-Za-z0-9-_=]+)$ {0}/$1 [END]
-
+    """
+    CONFIG_TEMPLATE24_POST = """\
         <Directory {0}>
             Require all granted
         </Directory>
+        <Location /.well-known/acme-challenge>
+            Require all granted
+        </Location>
     """
 
     def __init__(self, *args, **kwargs):
         super(ApacheHttp01, self).__init__(*args, **kwargs)
-        self.challenge_conf = os.path.join(
+        self.challenge_conf_pre = os.path.join(
             self.configurator.conf("challenge-location"),
-            "le_http_01_challenge.conf")
+            "le_http_01_challenge_pre.conf")
+        self.challenge_conf_post = os.path.join(
+            self.configurator.conf("challenge-location"),
+            "le_http_01_challenge_post.conf")
         self.challenge_dir = os.path.join(
             self.configurator.config.work_dir,
             "http_challenges")
@@ -79,24 +92,32 @@ class ApacheHttp01(common.TLSSNI01):
                 chall.domain, filter_defaults=False,
                 port=str(self.configurator.config.http01_port))
             if vh:
-                self._set_up_include_directive(vh)
+                self._set_up_include_directives(vh)
             else:
                 for vh in self._relevant_vhosts():
-                    self._set_up_include_directive(vh)
+                    self._set_up_include_directives(vh)
 
         self.configurator.reverter.register_file_creation(
-            True, self.challenge_conf)
+            True, self.challenge_conf_pre)
+        self.configurator.reverter.register_file_creation(
+            True, self.challenge_conf_post)
 
         if self.configurator.version < (2, 4):
-            config_template = self.CONFIG_TEMPLATE22
+            config_template_pre = self.CONFIG_TEMPLATE22_PRE
+            config_template_post = self.CONFIG_TEMPLATE22_POST
         else:
-            config_template = self.CONFIG_TEMPLATE24
+            config_template_pre = self.CONFIG_TEMPLATE24_PRE
+            config_template_post = self.CONFIG_TEMPLATE24_POST
 
-        config_text = config_template.format(self.challenge_dir)
+        config_text_pre = config_template_pre.format(self.challenge_dir)
+        config_text_post = config_template_post.format(self.challenge_dir)
 
-        logger.debug("writing a config file with text:\n %s", config_text)
-        with open(self.challenge_conf, "w") as new_conf:
-            new_conf.write(config_text)
+        logger.debug("writing a pre config file with text:\n %s", config_text_pre)
+        with open(self.challenge_conf_pre, "w") as new_conf:
+            new_conf.write(config_text_pre)
+        logger.debug("writing a post config file with text:\n %s", config_text_post)
+        with open(self.challenge_conf_post, "w") as new_conf:
+            new_conf.write(config_text_post)
 
     def _relevant_vhosts(self):
         http01_port = str(self.configurator.config.http01_port)
@@ -137,14 +158,17 @@ class ApacheHttp01(common.TLSSNI01):
 
         return response
 
-    def _set_up_include_directive(self, vhost):
-        """Includes override configuration to the beginning of VirtualHost.
-        Note that this include isn't added to Augeas search tree"""
+    def _set_up_include_directives(self, vhost):
+        """Includes override configuration to the beginning and to the end of
+        VirtualHost. Note that this include isn't added to Augeas search tree"""
 
         if vhost not in self.moded_vhosts:
             logger.debug(
                 "Adding a temporary challenge validation Include for name: %s " +
                 "in: %s", vhost.name, vhost.filep)
             self.configurator.parser.add_dir_beginning(
-                vhost.path, "Include", self.challenge_conf)
+                vhost.path, "Include", self.challenge_conf_pre)
+            self.configurator.parser.add_dir(
+                vhost.path, "Include", self.challenge_conf_post)
+
             self.moded_vhosts.add(vhost)
diff --git a/certbot-apache/certbot_apache/tests/http_01_test.py b/certbot-apache/certbot_apache/tests/http_01_test.py
index 64a76649a..9ed4ee509 100644
--- a/certbot-apache/certbot_apache/tests/http_01_test.py
+++ b/certbot-apache/certbot_apache/tests/http_01_test.py
@@ -158,23 +158,31 @@ class ApacheHttp01Test(util.ApacheTest):
         for vhost in vhosts:
             if not vhost.ssl:
                 matches = self.config.parser.find_dir("Include",
-                                                      self.http.challenge_conf,
+                                                      self.http.challenge_conf_pre,
+                                                      vhost.path)
+                self.assertEqual(len(matches), 1)
+                matches = self.config.parser.find_dir("Include",
+                                                      self.http.challenge_conf_post,
                                                       vhost.path)
                 self.assertEqual(len(matches), 1)
 
         self.assertTrue(os.path.exists(challenge_dir))
 
     def _test_challenge_conf(self):
-        with open(self.http.challenge_conf) as f:
-            conf_contents = f.read()
+        with open(self.http.challenge_conf_pre) as f:
+            pre_conf_contents = f.read()
 
-        self.assertTrue("RewriteEngine on" in conf_contents)
-        self.assertTrue("RewriteRule" in conf_contents)
-        self.assertTrue(self.http.challenge_dir in conf_contents)
+        with open(self.http.challenge_conf_post) as f:
+            post_conf_contents = f.read()
+
+        self.assertTrue("RewriteEngine on" in pre_conf_contents)
+        self.assertTrue("RewriteRule" in pre_conf_contents)
+
+        self.assertTrue(self.http.challenge_dir in post_conf_contents)
         if self.config.version < (2, 4):
-            self.assertTrue("Allow from all" in conf_contents)
+            self.assertTrue("Allow from all" in post_conf_contents)
         else:
-            self.assertTrue("Require all granted" in conf_contents)
+            self.assertTrue("Require all granted" in post_conf_contents)
 
     def _test_challenge_file(self, achall):
         name = os.path.join(self.http.challenge_dir, achall.chall.encode("token"))

From c0068791ce2a4edbe4b7293c6eab21d8b62347ca Mon Sep 17 00:00:00 2001
From: Noah Swartz <swartzcr@gmail.com>
Date: Wed, 24 Jan 2018 13:56:40 -0800
Subject: [PATCH 36/45] add let's encrypt status to footer and fix link

---
 docs/_templates/footer.html | 52 +++++++++++++++++++++++++++++++++++++
 docs/conf.py                |  3 ++-
 2 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 docs/_templates/footer.html

diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html
new file mode 100644
index 000000000..8fd0f127d
--- /dev/null
+++ b/docs/_templates/footer.html
@@ -0,0 +1,52 @@
+<footer>
+  {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %}
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      {% if next %}
+        <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}" accesskey="n" rel="next">{{ _('Next') }} <span class="fa fa-arrow-circle-right"></span></a>
+      {% endif %}
+      {% if prev %}
+        <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> {{ _('Previous') }}</a>
+      {% endif %}
+    </div>
+  {% endif %}
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+    <span class="copyright">
+    &copy; Copyright 2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at <a href="https://eff.org/cb-license">https://eff.org/cb-license</a>.
+    </span>
+    <br>
+    <br>
+    <span class="status">
+        <a href="https://letsencrypt.status.io/">Let's Encrypt Status</a>
+    </span>
+
+    {%- if build_id and build_url %}
+      {% trans build_url=build_url, build_id=build_id %}
+        <span class="build">
+          Build
+          <a href="{{ build_url }}">{{ build_id }}</a>.
+        </span>
+      {% endtrans %}
+    {%- elif commit %}
+      {% trans commit=commit %}
+        <span class="commit">
+          Revision <code>{{ commit }}</code>.
+        </span>
+      {% endtrans %}
+    {%- elif last_updated %}
+      {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
+    {%- endif %}
+
+    </p>
+  </div>
+
+  {%- if show_sphinx %}
+  {% trans %}Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>{% endtrans %}.
+  {%- endif %}
+
+  {%- block extrafooter %} {% endblock %}
+
+</footer>
diff --git a/docs/conf.py b/docs/conf.py
index 73df47dbd..09bb44285 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -64,7 +64,8 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Certbot'
-copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license '
+# this is now overridden by the footer.html template
+#copyright = u'2014-2018 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From 0a4f926b160cf118c81974e91df7f196bc7a379d Mon Sep 17 00:00:00 2001
From: Jacob Hoffman-Andrews <github@hoffman-andrews.com>
Date: Wed, 24 Jan 2018 15:01:42 -0800
Subject: [PATCH 37/45] Remove Default Detector log line. (#5372)

This produces a super-long log line that wraps to 30-60 lines, depending on
screen width. Even though it's just at debug level, it clutters up the integration
test output without providing proportional debugging value.

* Remove Default Detector log line.

This produces about 30 lines of log output. Even though it's just at debug
level, it clutters up the integration test output without providing proportional
debugging value.

* Add more useful logs.
---
 certbot/cli.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/certbot/cli.py b/certbot/cli.py
index f0fa7eb7e..dec7474f9 100644
--- a/certbot/cli.py
+++ b/certbot/cli.py
@@ -207,13 +207,15 @@ def set_by_cli(var):
         # propagate plugin requests: eg --standalone modifies config.authenticator
         detector.authenticator, detector.installer = (
             plugin_selection.cli_plugin_requests(detector))
-        logger.debug("Default Detector is %r", detector)
 
     if not isinstance(getattr(detector, var), _Default):
+        logger.debug("Var %s=%s (set by user).", var, getattr(detector, var))
         return True
 
     for modifier in VAR_MODIFIERS.get(var, []):
         if set_by_cli(modifier):
+            logger.debug("Var %s=%s (set by user).",
+                var, VAR_MODIFIERS.get(var, []))
             return True
 
     return False

From 8a9f21cdd36fb9a424430f9c807bacb27c1cfbf4 Mon Sep 17 00:00:00 2001
From: ohemorange <ebportnoy@gmail.com>
Date: Wed, 24 Jan 2018 22:19:32 -0800
Subject: [PATCH 38/45] Fix Nginx redirect issue (#5479)

* wrap redirect in if host matches

* return 404 if we've created a new block

* change domain matching to exact match

* insert new redirect directive at the top

* add a redirect block to the top if it doesn't already exist, even if there's an existing redirect

* fix obj tests

* remove active parameter

* update tests

* add back spaces

* move imports

* remove unused code
---
 certbot-nginx/certbot_nginx/configurator.py   |  50 ++++----
 certbot-nginx/certbot_nginx/obj.py            |  21 ----
 .../certbot_nginx/tests/configurator_test.py  | 110 +++---------------
 certbot-nginx/certbot_nginx/tests/obj_test.py |  14 +--
 4 files changed, 44 insertions(+), 151 deletions(-)

diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py
index bb2933a39..9f091c0fd 100644
--- a/certbot-nginx/certbot_nginx/configurator.py
+++ b/certbot-nginx/certbot_nginx/configurator.py
@@ -31,16 +31,6 @@ from certbot_nginx import http_01
 
 logger = logging.getLogger(__name__)
 
-REDIRECT_BLOCK = [
-    ['\n    ', 'return', ' ', '301', ' ', 'https://$host$request_uri'],
-    ['\n']
-]
-
-REDIRECT_COMMENT_BLOCK = [
-    ['\n    ', '#', ' Redirect non-https traffic to https'],
-    ['\n    ', '#', ' return 301 https://$host$request_uri;'],
-    ['\n']
-]
 
 @zope.interface.implementer(interfaces.IAuthenticator, interfaces.IInstaller)
 @zope.interface.provider(interfaces.IPluginFactory)
@@ -571,24 +561,17 @@ class NginxConfigurator(common.Installer):
             logger.warning("Failed %s for %s", enhancement, domain)
             raise
 
-    def _has_certbot_redirect(self, vhost):
-        test_redirect_block = _test_block_from_block(REDIRECT_BLOCK)
+    def _has_certbot_redirect(self, vhost, domain):
+        test_redirect_block = _test_block_from_block(_redirect_block_for_domain(domain))
         return vhost.contains_list(test_redirect_block)
 
-    def _has_certbot_redirect_comment(self, vhost):
-        test_redirect_comment_block = _test_block_from_block(REDIRECT_COMMENT_BLOCK)
-        return vhost.contains_list(test_redirect_comment_block)
-
-    def _add_redirect_block(self, vhost, active=True):
+    def _add_redirect_block(self, vhost, domain):
         """Add redirect directive to vhost
         """
-        if active:
-            redirect_block = REDIRECT_BLOCK
-        else:
-            redirect_block = REDIRECT_COMMENT_BLOCK
+        redirect_block = _redirect_block_for_domain(domain)
 
         self.parser.add_server_directives(
-            vhost, redirect_block, replace=False)
+            vhost, redirect_block, replace=False, insert_at_top=True)
 
     def _enable_redirect(self, domain, unused_options):
         """Redirect all equivalent HTTP traffic to ssl_vhost.
@@ -615,6 +598,7 @@ class NginxConfigurator(common.Installer):
                 self.DEFAULT_LISTEN_PORT)
             return
 
+        new_vhost = None
         if vhost.ssl:
             new_vhost = self.parser.duplicate_vhost(vhost,
                 only_directives=['listen', 'server_name'])
@@ -631,20 +615,18 @@ class NginxConfigurator(common.Installer):
             # remove all non-ssl addresses from the existing block
             self.parser.remove_server_directives(vhost, 'listen', match_func=_no_ssl_match_func)
 
+            # Add this at the bottom to get the right order of directives
+            return_404_directive = [['\n    ', 'return', ' ', '404']]
+            self.parser.add_server_directives(new_vhost, return_404_directive, replace=False)
+
             vhost = new_vhost
 
-        if self._has_certbot_redirect(vhost):
+        if self._has_certbot_redirect(vhost, domain):
             logger.info("Traffic on port %s already redirecting to ssl in %s",
                 self.DEFAULT_LISTEN_PORT, vhost.filep)
-        elif vhost.has_redirect():
-            if not self._has_certbot_redirect_comment(vhost):
-                self._add_redirect_block(vhost, active=False)
-            logger.info("The appropriate server block is already redirecting "
-                        "traffic. To enable redirect anyway, uncomment the "
-                        "redirect lines in %s.", vhost.filep)
         else:
             # Redirect plaintextish host to https
-            self._add_redirect_block(vhost, active=True)
+            self._add_redirect_block(vhost, domain)
             logger.info("Redirecting all traffic on port %s to ssl in %s",
                 self.DEFAULT_LISTEN_PORT, vhost.filep)
 
@@ -907,6 +889,14 @@ def _test_block_from_block(block):
     parser.comment_directive(test_block, 0)
     return test_block[:-1]
 
+def _redirect_block_for_domain(domain):
+    redirect_block = [[
+        ['\n    ', 'if', ' ', '($host', ' ', '=', ' ', '%s)' % domain, ' '],
+        [['\n        ', 'return', ' ', '301', ' ', 'https://$host$request_uri'],
+        '\n    ']],
+        ['\n']]
+    return redirect_block
+
 def nginx_restart(nginx_ctl, nginx_conf):
     """Restarts the Nginx Server.
 
diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py
index f5ac5c2e3..e8dc8936d 100644
--- a/certbot-nginx/certbot_nginx/obj.py
+++ b/certbot-nginx/certbot_nginx/obj.py
@@ -193,15 +193,6 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
 
         return False
 
-    def has_redirect(self):
-        """Determine if this vhost has a redirecting statement
-        """
-        for directive_name in REDIRECT_DIRECTIVES:
-            found = _find_directive(self.raw, directive_name)
-            if found is not None:
-                return True
-        return False
-
     def contains_list(self, test):
         """Determine if raw server block contains test list at top level
         """
@@ -225,15 +216,3 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
         for a in self.addrs:
             if not a.ipv6:
                 return True
-
-def _find_directive(directives, directive_name):
-    """Find a directive of type directive_name in directives
-    """
-    if not directives or isinstance(directives, six.string_types) or len(directives) == 0:
-        return None
-
-    if directives[0] == directive_name:
-        return directives
-
-    matches = (_find_directive(line, directive_name) for line in directives)
-    return next((m for m in matches if m is not None), None)
diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py
index 7475df40c..acb7ee282 100644
--- a/certbot-nginx/certbot_nginx/tests/configurator_test.py
+++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py
@@ -18,6 +18,8 @@ from certbot.tests import util as certbot_test_util
 from certbot_nginx import constants
 from certbot_nginx import obj
 from certbot_nginx import parser
+from certbot_nginx.configurator import _redirect_block_for_domain
+from certbot_nginx.nginxparser import UnspacedList
 from certbot_nginx.tests import util
 
 
@@ -447,7 +449,7 @@ class NginxConfiguratorTest(util.NginxTest):
     def test_redirect_enhance(self):
         # Test that we successfully add a redirect when there is
         # a listen directive
-        expected = ['return', '301', 'https://$host$request_uri']
+        expected = UnspacedList(_redirect_block_for_domain("www.example.com"))[0]
 
         example_conf = self.config.parser.abs_path('sites-enabled/example.com')
         self.config.enhance("www.example.com", "redirect")
@@ -460,6 +462,8 @@ class NginxConfiguratorTest(util.NginxTest):
         migration_conf = self.config.parser.abs_path('sites-enabled/migration.com')
         self.config.enhance("migration.com", "redirect")
 
+        expected = UnspacedList(_redirect_block_for_domain("migration.com"))[0]
+
         generated_conf = self.config.parser.parsed[migration_conf]
         self.assertTrue(util.contains_at_depth(generated_conf, expected, 2))
 
@@ -484,101 +488,27 @@ class NginxConfiguratorTest(util.NginxTest):
                ['ssl_dhparam', self.config.ssl_dhparams], ['#', ' managed by Certbot'],
                [], []]],
              [['server'], [
+               [['if', '($host', '=', 'www.example.com)'], [
+                 ['return', '301', 'https://$host$request_uri']]],
+               ['#', ' managed by Certbot'], [],
                ['listen', '69.50.225.155:9000'],
                ['listen', '127.0.0.1'],
                ['server_name', '.example.com'],
                ['server_name', 'example.*'],
-               ['return', '301', 'https://$host$request_uri'], ['#', ' managed by Certbot'],
-               [], []]]],
+               ['return', '404'], ['#', ' managed by Certbot'], [], [], []]]],
             generated_conf)
 
     @mock.patch('certbot_nginx.obj.VirtualHost.contains_list')
-    @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect')
-    def test_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list):
+    def test_certbot_redirect_exists(self, mock_contains_list):
         # Test that we add no redirect statement if there is already a
         # redirect in the block that is managed by certbot
         # Has a certbot redirect
-        mock_has_redirect.return_value = True
         mock_contains_list.return_value = True
         with mock.patch("certbot_nginx.configurator.logger") as mock_logger:
             self.config.enhance("www.example.com", "redirect")
             self.assertEqual(mock_logger.info.call_args[0][0],
                 "Traffic on port %s already redirecting to ssl in %s")
 
-    @mock.patch('certbot_nginx.obj.VirtualHost.contains_list')
-    @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect')
-    def test_non_certbot_redirect_exists(self, mock_has_redirect, mock_contains_list):
-        # Test that we add a redirect as a comment if there is already a
-        # redirect-class statement in the block that isn't managed by certbot
-        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
-
-        # Has a non-Certbot redirect, and has no existing comment
-        mock_contains_list.return_value = False
-        mock_has_redirect.return_value = True
-        with mock.patch("certbot_nginx.configurator.logger") as mock_logger:
-            self.config.enhance("www.example.com", "redirect")
-            self.assertEqual(mock_logger.info.call_args[0][0],
-                "The appropriate server block is already redirecting "
-                "traffic. To enable redirect anyway, uncomment the "
-                "redirect lines in %s.")
-            generated_conf = self.config.parser.parsed[example_conf]
-            expected = [
-                ['#', ' Redirect non-https traffic to https'],
-                ['#', ' return 301 https://$host$request_uri;'],
-            ]
-            for line in expected:
-                self.assertTrue(util.contains_at_depth(generated_conf, line, 2))
-
-    @mock.patch('certbot_nginx.obj.VirtualHost.contains_list')
-    @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect')
-    def test_non_certbot_redirect_exists_has_ssl_copy(self, mock_has_redirect, mock_contains_list):
-        # Test that we add a redirect as a comment if there is already a
-        # redirect-class statement in the block that isn't managed by certbot
-        example_conf = self.config.parser.abs_path('sites-enabled/example.com')
-
-        self.config.deploy_cert(
-            "example.org",
-            "example/cert.pem",
-            "example/key.pem",
-            "example/chain.pem",
-            "example/fullchain.pem")
-
-        # Has a non-Certbot redirect, and has no existing comment
-        mock_contains_list.return_value = False
-        mock_has_redirect.return_value = True
-        with mock.patch("certbot_nginx.configurator.logger") as mock_logger:
-            self.config.enhance("www.example.com", "redirect")
-            self.assertEqual(mock_logger.info.call_args[0][0],
-                "The appropriate server block is already redirecting "
-                "traffic. To enable redirect anyway, uncomment the "
-                "redirect lines in %s.")
-            generated_conf = self.config.parser.parsed[example_conf]
-            expected = [
-                ['#', ' Redirect non-https traffic to https'],
-                ['#', ' return 301 https://$host$request_uri;'],
-            ]
-            for line in expected:
-                self.assertTrue(util.contains_at_depth(generated_conf, line, 2))
-
-    @mock.patch('certbot_nginx.obj.VirtualHost.contains_list')
-    @mock.patch('certbot_nginx.obj.VirtualHost.has_redirect')
-    @mock.patch('certbot_nginx.configurator.NginxConfigurator._has_certbot_redirect_comment')
-    @mock.patch('certbot_nginx.configurator.NginxConfigurator._add_redirect_block')
-    def test_redirect_comment_exists(self, mock_add_redirect_block,
-        mock_has_cb_redirect_comment, mock_has_redirect, mock_contains_list):
-        # Test that we add nothing if there is a non-Certbot redirect and a
-        # preexisting comment
-        # Has a non-Certbot redirect and a comment
-        mock_has_redirect.return_value = True
-        mock_contains_list.return_value = False # self._has_certbot_redirect(vhost):
-        mock_has_cb_redirect_comment.return_value = True
-
-        # assert _add_redirect_block not called
-        with mock.patch("certbot_nginx.configurator.logger") as mock_logger:
-            self.config.enhance("www.example.com", "redirect")
-            self.assertFalse(mock_add_redirect_block.called)
-            self.assertTrue(mock_logger.info.called)
-
     def test_redirect_dont_enhance(self):
         # Test that we don't accidentally add redirect to ssl-only block
         with mock.patch("certbot_nginx.configurator.logger") as mock_logger:
@@ -586,22 +516,18 @@ class NginxConfiguratorTest(util.NginxTest):
         self.assertEqual(mock_logger.info.call_args[0][0],
                 'No matching insecure server blocks listening on port %s found.')
 
-    def test_no_double_redirect(self):
-        # Test that we don't also add the commented redirect if we've just added
-        # a redirect to that vhost this run
+    def test_double_redirect(self):
+        # Test that we add one redirect for each domain
         example_conf = self.config.parser.abs_path('sites-enabled/example.com')
         self.config.enhance("example.com", "redirect")
         self.config.enhance("example.org", "redirect")
 
-        unexpected = [
-            ['#', ' Redirect non-https traffic to https'],
-            ['#', ' if ($scheme != "https") {'],
-            ['#', '     return 301 https://$host$request_uri;'],
-            ['#', ' } # managed by Certbot']
-        ]
+        expected1 = UnspacedList(_redirect_block_for_domain("example.com"))[0]
+        expected2 = UnspacedList(_redirect_block_for_domain("example.org"))[0]
+
         generated_conf = self.config.parser.parsed[example_conf]
-        for line in unexpected:
-            self.assertFalse(util.contains_at_depth(generated_conf, line, 2))
+        self.assertTrue(util.contains_at_depth(generated_conf, expected1, 2))
+        self.assertTrue(util.contains_at_depth(generated_conf, expected2, 2))
 
     def test_staple_ocsp_bad_version(self):
         self.config.version = (1, 3, 1)
@@ -763,7 +689,7 @@ class NginxConfiguratorTest(util.NginxTest):
 
         self.config.parser.load()
 
-        expected = ['return', '301', 'https://$host$request_uri']
+        expected = UnspacedList(_redirect_block_for_domain("www.nomatch.com"))[0]
 
         generated_conf = self.config.parser.parsed[default_conf]
         self.assertTrue(util.contains_at_depth(generated_conf, expected, 2))
diff --git a/certbot-nginx/certbot_nginx/tests/obj_test.py b/certbot-nginx/certbot_nginx/tests/obj_test.py
index 92cb0e086..b30338b5b 100644
--- a/certbot-nginx/certbot_nginx/tests/obj_test.py
+++ b/certbot-nginx/certbot_nginx/tests/obj_test.py
@@ -162,17 +162,15 @@ class VirtualHostTest(unittest.TestCase):
                                  'enabled: False'])
         self.assertEqual(stringified, str(self.vhost1))
 
-    def test_has_redirect(self):
-        self.assertTrue(self.vhost1.has_redirect())
-        self.assertTrue(self.vhost2.has_redirect())
-        self.assertTrue(self.vhost3.has_redirect())
-        self.assertFalse(self.vhost4.has_redirect())
-
     def test_contains_list(self):
         from certbot_nginx.obj import VirtualHost
         from certbot_nginx.obj import Addr
-        from certbot_nginx.configurator import REDIRECT_BLOCK, _test_block_from_block
-        test_needle = _test_block_from_block(REDIRECT_BLOCK)
+        from certbot_nginx.configurator import _test_block_from_block
+        test_block = [
+            ['\n    ', 'return', ' ', '301', ' ', 'https://$host$request_uri'],
+            ['\n']
+        ]
+        test_needle = _test_block_from_block(test_block)
         test_haystack = [['listen', '80'], ['root', '/var/www/html'],
             ['index', 'index.html index.htm index.nginx-debian.html'],
             ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'],

From a1aba5842e979379d98f3128d140d1270fb951c5 Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 24 Jan 2018 22:23:20 -0800
Subject: [PATCH 39/45] Fix --no-bootstrap on CentOS/RHEL 6 (#5476)

* fix --no-bootstrap on RHEL6

* Add regression test
---
 letsencrypt-auto-source/letsencrypt-auto      | 20 ++++++++++++-------
 .../letsencrypt-auto.template                 | 20 ++++++++++++-------
 .../tests/centos6_tests.sh                    |  8 ++++++++
 3 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index 9c8ccf344..b5cac23e6 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -761,13 +761,8 @@ BootstrapMageiaCommon() {
 # Set Bootstrap to the function that installs OS dependencies on this system
 # and BOOTSTRAP_VERSION to the unique identifier for the current version of
 # that function. If Bootstrap is set to a function that doesn't install any
-# packages (either because --no-bootstrap was included on the command line or
-# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set.
-if [ "$NO_BOOTSTRAP" = 1 ]; then
-  Bootstrap() {
-    :
-  }
-elif [ -f /etc/debian_version ]; then
+# packages BOOTSTRAP_VERSION is not set.
+if [ -f /etc/debian_version ]; then
   Bootstrap() {
     BootstrapMessage "Debian-based OSes"
     BootstrapDebCommon
@@ -863,6 +858,17 @@ else
   }
 fi
 
+# We handle this case after determining the normal bootstrap version to allow
+# variables like USE_PYTHON_3 to be properly set. As described above, if the
+# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not
+# be set so we unset it here.
+if [ "$NO_BOOTSTRAP" = 1 ]; then
+  Bootstrap() {
+    :
+  }
+  unset BOOTSTRAP_VERSION
+fi
+
 # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used
 # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set
 # if it is unknown how OS dependencies were installed on this system.
diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template
index 96e5c2db0..2ce337002 100755
--- a/letsencrypt-auto-source/letsencrypt-auto.template
+++ b/letsencrypt-auto-source/letsencrypt-auto.template
@@ -300,13 +300,8 @@ DeterminePythonVersion() {
 # Set Bootstrap to the function that installs OS dependencies on this system
 # and BOOTSTRAP_VERSION to the unique identifier for the current version of
 # that function. If Bootstrap is set to a function that doesn't install any
-# packages (either because --no-bootstrap was included on the command line or
-# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set.
-if [ "$NO_BOOTSTRAP" = 1 ]; then
-  Bootstrap() {
-    :
-  }
-elif [ -f /etc/debian_version ]; then
+# packages BOOTSTRAP_VERSION is not set.
+if [ -f /etc/debian_version ]; then
   Bootstrap() {
     BootstrapMessage "Debian-based OSes"
     BootstrapDebCommon
@@ -402,6 +397,17 @@ else
   }
 fi
 
+# We handle this case after determining the normal bootstrap version to allow
+# variables like USE_PYTHON_3 to be properly set. As described above, if the
+# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not
+# be set so we unset it here.
+if [ "$NO_BOOTSTRAP" = 1 ]; then
+  Bootstrap() {
+    :
+  }
+  unset BOOTSTRAP_VERSION
+fi
+
 # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used
 # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set
 # if it is unknown how OS dependencies were installed on this system.
diff --git a/letsencrypt-auto-source/tests/centos6_tests.sh b/letsencrypt-auto-source/tests/centos6_tests.sh
index a0e96edf8..2c6dcf734 100644
--- a/letsencrypt-auto-source/tests/centos6_tests.sh
+++ b/letsencrypt-auto-source/tests/centos6_tests.sh
@@ -69,5 +69,13 @@ fi
 echo "PASSED: Successfully upgraded to Python3 when only Python2.6 is present."
 echo ""
 
+export VENV_PATH=$(mktemp -d)
+"$LE_AUTO" -n --no-bootstrap --no-self-upgrade --version >/dev/null 2>&1
+if [ "$($VENV_PATH/bin/python -V 2>&1 | cut -d" " -f2 | cut -d. -f1)" != 3 ]; then
+  echo "Python 3 wasn't used with --no-bootstrap!"
+  exit 1
+fi
+unset VENV_PATH
+
 # test using python3
 pytest -v -s certbot/letsencrypt-auto-source/tests

From a2239baa45de446ab246047171ad86e014bf580c Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Wed, 24 Jan 2018 22:38:36 -0800
Subject: [PATCH 40/45] fix test_tests.sh (#5478)

---
 tests/letstest/scripts/test_tests.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh
index 4bed2dd3a..e6ab836b8 100755
--- a/tests/letstest/scripts/test_tests.sh
+++ b/tests/letstest/scripts/test_tests.sh
@@ -1,11 +1,13 @@
 #!/bin/sh -xe
 
+LE_AUTO="letsencrypt/letsencrypt-auto-source/letsencrypt-auto"
+LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive"
 MODULES="acme certbot certbot_apache certbot_nginx"
 VENV_NAME=venv
 
 # *-auto respects VENV_PATH
-letsencrypt/certbot-auto --debug --os-packages-only --non-interactive
-LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-bootstrap --non-interactive --version
+$LE_AUTO --os-packages-only
+LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version
 . $VENV_NAME/bin/activate
 
 # change to an empty directory to ensure CWD doesn't affect tests

From 43bbaadd1154c0cd44eeb761bf33563953e793ae Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Thu, 25 Jan 2018 15:29:38 -0800
Subject: [PATCH 41/45] Update certbot-auto and help (#5487)

* Release 0.21.1

(cherry picked from commit ff60d70e68f7b4ddc60a61848190fbb6e55b5d2b)

* Bump version to 0.22.0
---
 certbot-auto                                  |  46 ++++++++++--------
 docs/cli-help.txt                             |  18 +++----
 letsencrypt-auto                              |  46 ++++++++++--------
 letsencrypt-auto-source/certbot-auto.asc      |  16 +++---
 letsencrypt-auto-source/letsencrypt-auto      |  24 ++++-----
 letsencrypt-auto-source/letsencrypt-auto.sig  | Bin 256 -> 256 bytes
 .../pieces/certbot-requirements.txt           |  24 ++++-----
 7 files changed, 92 insertions(+), 82 deletions(-)

diff --git a/certbot-auto b/certbot-auto
index 558c330b2..d3a5c23e5 100755
--- a/certbot-auto
+++ b/certbot-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.0"
+LE_AUTO_VERSION="0.21.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -761,13 +761,8 @@ BootstrapMageiaCommon() {
 # Set Bootstrap to the function that installs OS dependencies on this system
 # and BOOTSTRAP_VERSION to the unique identifier for the current version of
 # that function. If Bootstrap is set to a function that doesn't install any
-# packages (either because --no-bootstrap was included on the command line or
-# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set.
-if [ "$NO_BOOTSTRAP" = 1 ]; then
-  Bootstrap() {
-    :
-  }
-elif [ -f /etc/debian_version ]; then
+# packages BOOTSTRAP_VERSION is not set.
+if [ -f /etc/debian_version ]; then
   Bootstrap() {
     BootstrapMessage "Debian-based OSes"
     BootstrapDebCommon
@@ -863,6 +858,17 @@ else
   }
 fi
 
+# We handle this case after determining the normal bootstrap version to allow
+# variables like USE_PYTHON_3 to be properly set. As described above, if the
+# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not
+# be set so we unset it here.
+if [ "$NO_BOOTSTRAP" = 1 ]; then
+  Bootstrap() {
+    :
+  }
+  unset BOOTSTRAP_VERSION
+fi
+
 # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used
 # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set
 # if it is unknown how OS dependencies were installed on this system.
@@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.21.0 \
-    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
-    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
-acme==0.21.0 \
-    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
-    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
-certbot-apache==0.21.0 \
-    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
-    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
-certbot-nginx==0.21.0 \
-    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
-    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
+certbot==0.21.1 \
+    --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \
+    --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea
+acme==0.21.1 \
+    --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \
+    --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b
+certbot-apache==0.21.1 \
+    --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \
+    --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5
+certbot-nginx==0.21.1 \
+    --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \
+    --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
diff --git a/docs/cli-help.txt b/docs/cli-help.txt
index f7318f0b3..abebdb9c9 100644
--- a/docs/cli-help.txt
+++ b/docs/cli-help.txt
@@ -107,9 +107,9 @@ optional arguments:
                         case, and to know when to deprecate support for past
                         Python versions and flags. If you wish to hide this
                         information from the Let's Encrypt server, set this to
-                        "". (default: CertbotACMEClient/0.21.0 (certbot;
-                        Ubuntu 16.04.3 LTS) Authenticator/XXX Installer/YYY
-                        (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags
+                        "". (default: CertbotACMEClient/0.21.1 (certbot;
+                        darwin 10.13.3) Authenticator/XXX Installer/YYY
+                        (SUBCOMMAND; flags: FLAGS) Py/2.7.14). The flags
                         encoded in the user agent are: --duplicate, --force-
                         renew, --allow-subset-of-names, -n, and whether any
                         hooks are set.
@@ -448,11 +448,9 @@ apache:
   Apache Web Server plugin - Beta
 
   --apache-enmod APACHE_ENMOD
-                        Path to the Apache 'a2enmod' binary. (default:
-                        a2enmod)
+                        Path to the Apache 'a2enmod' binary. (default: None)
   --apache-dismod APACHE_DISMOD
-                        Path to the Apache 'a2dismod' binary. (default:
-                        a2dismod)
+                        Path to the Apache 'a2dismod' binary. (default: None)
   --apache-le-vhost-ext APACHE_LE_VHOST_EXT
                         SSL vhost configuration extension. (default: -le-
                         ssl.conf)
@@ -466,13 +464,13 @@ apache:
                         /var/log/apache2)
   --apache-challenge-location APACHE_CHALLENGE_LOCATION
                         Directory path for challenge configuration. (default:
-                        /etc/apache2)
+                        /etc/apache2/other)
   --apache-handle-modules APACHE_HANDLE_MODULES
                         Let installer handle enabling required modules for
-                        you.(Only Ubuntu/Debian currently) (default: True)
+                        you.(Only Ubuntu/Debian currently) (default: False)
   --apache-handle-sites APACHE_HANDLE_SITES
                         Let installer handle enabling sites for you.(Only
-                        Ubuntu/Debian currently) (default: True)
+                        Ubuntu/Debian currently) (default: False)
 
 certbot-route53:auth:
   Obtain certificates using a DNS TXT record (if you are using AWS Route53
diff --git a/letsencrypt-auto b/letsencrypt-auto
index 558c330b2..d3a5c23e5 100755
--- a/letsencrypt-auto
+++ b/letsencrypt-auto
@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.21.0"
+LE_AUTO_VERSION="0.21.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -761,13 +761,8 @@ BootstrapMageiaCommon() {
 # Set Bootstrap to the function that installs OS dependencies on this system
 # and BOOTSTRAP_VERSION to the unique identifier for the current version of
 # that function. If Bootstrap is set to a function that doesn't install any
-# packages (either because --no-bootstrap was included on the command line or
-# we don't know how to bootstrap on this system), BOOTSTRAP_VERSION is not set.
-if [ "$NO_BOOTSTRAP" = 1 ]; then
-  Bootstrap() {
-    :
-  }
-elif [ -f /etc/debian_version ]; then
+# packages BOOTSTRAP_VERSION is not set.
+if [ -f /etc/debian_version ]; then
   Bootstrap() {
     BootstrapMessage "Debian-based OSes"
     BootstrapDebCommon
@@ -863,6 +858,17 @@ else
   }
 fi
 
+# We handle this case after determining the normal bootstrap version to allow
+# variables like USE_PYTHON_3 to be properly set. As described above, if the
+# Bootstrap function doesn't install any packages, BOOTSTRAP_VERSION should not
+# be set so we unset it here.
+if [ "$NO_BOOTSTRAP" = 1 ]; then
+  Bootstrap() {
+    :
+  }
+  unset BOOTSTRAP_VERSION
+fi
+
 # Sets PREV_BOOTSTRAP_VERSION to the identifier for the bootstrap script used
 # to install OS dependencies on this system. PREV_BOOTSTRAP_VERSION isn't set
 # if it is unknown how OS dependencies were installed on this system.
@@ -1190,18 +1196,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.21.0 \
-    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
-    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
-acme==0.21.0 \
-    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
-    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
-certbot-apache==0.21.0 \
-    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
-    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
-certbot-nginx==0.21.0 \
-    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
-    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
+certbot==0.21.1 \
+    --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \
+    --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea
+acme==0.21.1 \
+    --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \
+    --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b
+certbot-apache==0.21.1 \
+    --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \
+    --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5
+certbot-nginx==0.21.1 \
+    --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \
+    --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc
index 9df8013a0..f28fd9893 100644
--- a/letsencrypt-auto-source/certbot-auto.asc
+++ b/letsencrypt-auto-source/certbot-auto.asc
@@ -1,11 +1,11 @@
 -----BEGIN PGP SIGNATURE-----
-Version: GnuPG v2
 
-iQEcBAABCAAGBQJaX+JUAAoJEE0XyZXNl3XyUCkH/jowI7yayXREoBUWpLuByd/n
-e1wGLQjnZYkxv/AJGJ63G3QvwpzmIqo3r/6K4ARlUcdOnepZRDpF6jC4F5q9vBwW
-AvUVU2B7e6mC6l/jXNepS8xowEwkQptQBDfnqh8TTeTb3rQTFod8X41skZ2633HL
-RX4ditKaGMbcswMn6+5/juz0YK5ujVdVTcMeMcZKP2tvPJ9Y08YdpY6IdrM0Mfhn
-IqssjM06CzsiYHeNOXfRY4vAPw4Oq/md3bf6ZpPCee1HPiDm0NvHtTemWBkPIehf
-yy0U8JIDIZha4WKo3yifbZFL5Zf5czVkrtqQ3DBRcLrCFtBh2aTVsIMJkpW/wFo=
-=d/hS
+iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlpqMlYACgkQTRfJlc2X
+dfKHfQgAnZQJ34jFoVqEodT0EjvkFKZif4V/zXTsVwTHn107BcLCpH/9gjANrSo3
+JpvseH2q0odhOAZA4rZKH4Geh+5fsUl3Ew9YB28RXeyqEfCATUqPq6q+jAi55SLc
+a064Ux5N7eOIh9gxvpDKBeSFD0eNB8IDtPQhUspr+WnoycawrJHNGawL8WIfrWY3
+0ZPF981iPCWCdN3woDP9wHA2QtBClAk2pQ1aMgdkK9r/QLO+DY92xmT/Uu4ik2jR
+zv+QplsQLftjD+bRar5R9jiCWV5phPqrOF3ypMiU0K5bsnrZfGBzBcoEyfKuB+UR
+F/j/631OC6yLRasr+xcL1gc+SCryfA==
+=tkZT
 -----END PGP SIGNATURE-----
diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index b5cac23e6..8ff7944b5 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -1196,18 +1196,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.21.0 \
-    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
-    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
-acme==0.21.0 \
-    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
-    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
-certbot-apache==0.21.0 \
-    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
-    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
-certbot-nginx==0.21.0 \
-    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
-    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
+certbot==0.21.1 \
+    --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \
+    --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea
+acme==0.21.1 \
+    --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \
+    --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b
+certbot-apache==0.21.1 \
+    --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \
+    --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5
+certbot-nginx==0.21.1 \
+    --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \
+    --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig
index 2e52f03fdf9a1fbe2049d89f071a583bcf919976..8dd6837754301180039bdcd2192ec2ffa9911f1e 100644
GIT binary patch
literal 256
zcmV+b0ssEOC2SB&#jbL|t)G85E<)sA{+|s~F5m$!7y+-fA{(45p3@Fk=k3Xi+RklE
z=idnaoUY1-<hn0}%G~DcIaujceO${UXN;oYRmVuo3Fu3L44}pUBu#2&7pO_ubv+91
zUshjVkZJa=M3@2xhe{vAi~dA6Ijl&#xqvNzISGOGo3dlh6TS30PkJlD)maXL6Ss)z
zV6Ny7C*HHHSGODHc2_y=-P7w=F*xUlp-#j}G^+5X4EXC+$?MQJA}xdSnZIw$i@}j`
zp4_UIKy{4yZPjr*E?>Gy6T(2&LYe?<tC9>9T(@a*<ZoPeqe2e!*;8qY!wcQ&&3WBo
GnlYnGn1R^<

literal 256
zcmV+b0ssD{xCAf=x;KV2@d03ctAnF7mn+aqUc{p=)VXkyhWU!~cN;%G^8WY$S?ta1
z2hmQE4T`~+Zwq%*K~cAA%;XQy8S-ox4a$2d0@tMjK|wQYPjC$Cv}rUZ3(4Cmz7$?G
zU;e#Fi7%+uTC73=Hbzf*EfB0nOu(b3_b6<Q!!|zvZ7y0Gq&)xES&(8)`G1!t?3#Mf
zkkXdN{8n>CNN5}h_$-t&E@K&w437B8*P8QflhULg&MO?mxD$RuiY1;Ik~L_O?oC&*
zWM|7qv=mZh!YK!wmdDwpz@wj96nhJniJ_laAj^74?$6FZ`p5@HrirST)N_#L(FvXz
GZCemMz<GWE

diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt
index 406086fb5..0a5994afc 100644
--- a/letsencrypt-auto-source/pieces/certbot-requirements.txt
+++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt
@@ -1,12 +1,12 @@
-certbot==0.21.0 \
-    --hash=sha256:b6fc9cf80e8e2925827c61ca92c32faa935bbadaf14448e2d7f40e1f8f2cccdb \
-    --hash=sha256:07ca3246d3462fe73418113cc5c1036545f4b2312831024da923054de3a85857
-acme==0.21.0 \
-    --hash=sha256:4ef91a62c30b9d6bd1dd0b5ac3a8c7e70203e08e5269d3d26311dd6648aaacda \
-    --hash=sha256:d64eae267c0bb21c98fa889b4e0be4c473ca8e80488d3de057e803d6d167544d
-certbot-apache==0.21.0 \
-    --hash=sha256:026c23fec4def727f88acd15f66b5641f7ba1f767f0728fd56798cf3500be0c5 \
-    --hash=sha256:185dae50c680fa3c09646907a6256c6b4ddf8525723d3b13b9b33d1a3118663b
-certbot-nginx==0.21.0 \
-    --hash=sha256:e5ac3a203871f13e7e72d4922e401364342f2999d130c959f90949305c33d2bc \
-    --hash=sha256:88be95916935980edc4c6ec3f39031ac47f5b73d6e43dfa3694b927226432642
+certbot==0.21.1 \
+    --hash=sha256:08f026078807fbcfd7bfab44c4d827ee287738fefcc86fbe1493ce752d2fdccb \
+    --hash=sha256:e6c8e9b0b5e38834330831d5a91e1c08accdb9b4923855d14d524e7327e6c4ea
+acme==0.21.1 \
+    --hash=sha256:4b2b5ef80c755dfa30eb5c67ab4b4e66e7f205ad922b43170502c5f8d8ef1242 \
+    --hash=sha256:296e8abf4f5a69af1a892416faceea90e15f39e2920bf87beeaad1d6ce70a60b
+certbot-apache==0.21.1 \
+    --hash=sha256:faa4af1033564a0e676d16940775593fb849527b494a15f6a816ad0ed4fa273c \
+    --hash=sha256:0bce4419d4fdabbdda2223cff8db6794c5717632fb9511b00498ec00982a3fa5
+certbot-nginx==0.21.1 \
+    --hash=sha256:3fad3b4722544558ce03132f853e18da5e516013086aaa40f1036aa6667c70a9 \
+    --hash=sha256:55a32afe0950ff49d3118f93035463a46c85c2f399d261123f5fe973afdd4f64

From 45613fd31c3649ac5fb08a52153a8893f394af0b Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Fri, 26 Jan 2018 16:02:19 -0800
Subject: [PATCH 42/45] update changelog for 0.21.1 (#5504)

---
 CHANGELOG.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6f7be9fe..1369b0907 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,24 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.21.1 - 2018-01-25
+
+### Fixed
+
+* When creating an HTTP to HTTPS redirect in Nginx, we now ensure the Host
+  header of the request is set to an expected value before redirecting users to
+  the domain found in the header. The previous way Certbot configured Nginx
+  redirects was a potential security issue which you can read more about at
+  https://community.letsencrypt.org/t/security-issue-with-redirects-added-by-certbots-nginx-plugin/51493.
+* Fixed a problem where Certbot's Apache plugin could fail HTTP-01 challenges
+  if basic authentication is configured for the domain you request a
+  certificate for.
+* certbot-auto --no-bootstrap now properly tries to use Python 3.4 on RHEL 6
+  based systems rather than Python 2.6.
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/49?closed=1
+
 ## 0.21.0 - 2018-01-17
 
 ### Added

From e085ff06a12a0be6033f38f092d42253c7b21515 Mon Sep 17 00:00:00 2001
From: sydneyli <sydli@stanford.edu>
Date: Mon, 5 Feb 2018 16:27:21 -0800
Subject: [PATCH 43/45] Update old issue link to point to letsencrypt community
 forums. (#5538)

---
 certbot/client.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/certbot/client.py b/certbot/client.py
index b735421f5..bc25da549 100644
--- a/certbot/client.py
+++ b/certbot/client.py
@@ -556,11 +556,11 @@ class Client(object):
             self.installer.rollback_checkpoints()
             self.installer.restart()
         except:
-            # TODO: suggest letshelp-letsencrypt here
             reporter.add_message(
                 "An error occurred and we failed to restore your config and "
-                "restart your server. Please submit a bug report to "
-                "https://github.com/letsencrypt/letsencrypt",
+                "restart your server. Please post to "
+                "https://community.letsencrypt.org/c/server-config "
+                "with details about your configuration and this error you received.",
                 reporter.HIGH_PRIORITY)
             raise
         reporter.add_message(success_msg, reporter.HIGH_PRIORITY)

From 9baf75d6c887a9b69bd92c7bffe585f091c9b08e Mon Sep 17 00:00:00 2001
From: Jacob Hoffman-Andrews <github@hoffman-andrews.com>
Date: Tue, 6 Feb 2018 16:45:33 -0800
Subject: [PATCH 44/45] client.py changes for ACMEv2 (#5287)

* Implement ACMEv2 signing of POST bodies.

* Add account, and make acme_version explicit.

* Remove separate NewAccount.

* Rename to add v2.

* Add terms_of_service_agreed.

* Split out wrap_in_jws_v2 test.

* Re-add too-many-public-methods.

* Split Client into ClientBase / Client / ClientV2

* Use camelCase for newAccount.

* Make acme_version optional parameter on .post().

This allows us to instantiate a ClientNetwork before knowing the version.

* Add kid unconditionally.
---
 acme/acme/client.py      | 324 ++++++++++++++++++++++++---------------
 acme/acme/client_test.py |  74 ++++++---
 acme/acme/messages.py    |   1 +
 3 files changed, 258 insertions(+), 141 deletions(-)

diff --git a/acme/acme/client.py b/acme/acme/client.py
index dc5efbe86..5d1add5a3 100644
--- a/acme/acme/client.py
+++ b/acme/acme/client.py
@@ -39,39 +39,24 @@ DEFAULT_NETWORK_TIMEOUT = 45
 DER_CONTENT_TYPE = 'application/pkix-cert'
 
 
-class Client(object):  # pylint: disable=too-many-instance-attributes
-    """ACME client.
-
-    .. todo::
-       Clean up raised error types hierarchy, document, and handle (wrap)
-       instances of `.DeserializationError` raised in `from_json()`.
+class ClientBase(object):  # pylint: disable=too-many-instance-attributes
+    """ACME client base object.
 
     :ivar messages.Directory directory:
-    :ivar key: `.JWK` (private)
-    :ivar alg: `.JWASignature`
-    :ivar bool verify_ssl: Verify SSL certificates?
-    :ivar .ClientNetwork net: Client network. Useful for testing. If not
-        supplied, it will be initialized using `key`, `alg` and
-        `verify_ssl`.
-
+    :ivar .ClientNetwork net: Client network.
+    :ivar int acme_version: ACME protocol version. 1 or 2.
     """
 
-    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
-                 net=None):
+    def __init__(self, directory, net, acme_version):
         """Initialize.
 
-        :param directory: Directory Resource (`.messages.Directory`) or
-            URI from which the resource will be downloaded.
-
+        :param .messages.Directory directory: Directory Resource
+        :param .ClientNetwork net: Client network.
+        :param int acme_version: ACME protocol version. 1 or 2.
         """
-        self.key = key
-        self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net
-
-        if isinstance(directory, six.string_types):
-            self.directory = messages.Directory.from_json(
-                self.net.get(directory).json())
-        else:
-            self.directory = directory
+        self.directory = directory
+        self.net = net
+        self.acme_version = acme_version
 
     @classmethod
     def _regr_from_response(cls, response, uri=None, terms_of_service=None):
@@ -83,28 +68,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             uri=response.headers.get('Location', uri),
             terms_of_service=terms_of_service)
 
-    def register(self, new_reg=None):
-        """Register.
-
-        :param .NewRegistration new_reg:
-
-        :returns: Registration Resource.
-        :rtype: `.RegistrationResource`
-
-        """
-        new_reg = messages.NewRegistration() if new_reg is None else new_reg
-        assert isinstance(new_reg, messages.NewRegistration)
-
-        response = self.net.post(self.directory[new_reg], new_reg)
-        # TODO: handle errors
-        assert response.status_code == http_client.CREATED
-
-        # "Instance of 'Field' has no key/contact member" bug:
-        # pylint: disable=no-member
-        return self._regr_from_response(response)
-
     def _send_recv_regr(self, regr, body):
-        response = self.net.post(regr.uri, body)
+        response = self.net.post(regr.uri, body, acme_version=self.acme_version)
 
         # TODO: Boulder returns httplib.ACCEPTED
         #assert response.status_code == httplib.OK
@@ -153,21 +118,6 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         """
         return self._send_recv_regr(regr, messages.UpdateRegistration())
 
-    def agree_to_tos(self, regr):
-        """Agree to the terms-of-service.
-
-        Agree to the terms-of-service in a Registration Resource.
-
-        :param regr: Registration Resource.
-        :type regr: `.RegistrationResource`
-
-        :returns: Updated Registration Resource.
-        :rtype: `.RegistrationResource`
-
-        """
-        return self.update_registration(
-            regr.update(body=regr.body.update(agreement=regr.terms_of_service)))
-
     def _authzr_from_response(self, response, identifier, uri=None):
         authzr = messages.AuthorizationResource(
             body=messages.Authorization.from_json(response.json()),
@@ -176,42 +126,6 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             raise errors.UnexpectedUpdate(authzr)
         return authzr
 
-    def request_challenges(self, identifier, new_authzr_uri=None):
-        """Request challenges.
-
-        :param .messages.Identifier identifier: Identifier to be challenged.
-        :param str new_authzr_uri: Deprecated. Do not use.
-
-        :returns: Authorization Resource.
-        :rtype: `.AuthorizationResource`
-
-        """
-        if new_authzr_uri is not None:
-            logger.debug("request_challenges with new_authzr_uri deprecated.")
-        new_authz = messages.NewAuthorization(identifier=identifier)
-        response = self.net.post(self.directory.new_authz, new_authz)
-        # TODO: handle errors
-        assert response.status_code == http_client.CREATED
-        return self._authzr_from_response(response, identifier)
-
-    def request_domain_challenges(self, domain, new_authzr_uri=None):
-        """Request challenges for domain names.
-
-        This is simply a convenience function that wraps around
-        `request_challenges`, but works with domain names instead of
-        generic identifiers. See ``request_challenges`` for more
-        documentation.
-
-        :param str domain: Domain name to be challenged.
-        :param str new_authzr_uri: Deprecated. Do not use.
-
-        :returns: Authorization Resource.
-        :rtype: `.AuthorizationResource`
-
-        """
-        return self.request_challenges(messages.Identifier(
-            typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri)
-
     def answer_challenge(self, challb, response):
         """Answer challenge.
 
@@ -227,7 +141,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :raises .UnexpectedUpdate:
 
         """
-        response = self.net.post(challb.uri, response)
+        response = self.net.post(challb.uri, response,
+            acme_version=self.acme_version)
         try:
             authzr_uri = response.links['up']['url']
         except KeyError:
@@ -288,6 +203,133 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             response, authzr.body.identifier, authzr.uri)
         return updated_authzr, response
 
+    def revoke(self, cert, rsn):
+        """Revoke certificate.
+
+        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
+            `.ComparableX509`
+
+        :param int rsn: Reason code for certificate revocation.
+
+        :raises .ClientError: If revocation is unsuccessful.
+
+        """
+        response = self.net.post(self.directory[messages.Revocation],
+                                 messages.Revocation(
+                                     certificate=cert,
+                                     reason=rsn),
+                                 content_type=None,
+                                 acme_version=self.acme_version)
+        if response.status_code != http_client.OK:
+            raise errors.ClientError(
+                'Successful revocation must return HTTP OK status')
+
+class Client(ClientBase):
+    """ACME client for a v1 API.
+
+    .. todo::
+       Clean up raised error types hierarchy, document, and handle (wrap)
+       instances of `.DeserializationError` raised in `from_json()`.
+
+    :ivar messages.Directory directory:
+    :ivar key: `.JWK` (private)
+    :ivar alg: `.JWASignature`
+    :ivar bool verify_ssl: Verify SSL certificates?
+    :ivar .ClientNetwork net: Client network. Useful for testing. If not
+        supplied, it will be initialized using `key`, `alg` and
+        `verify_ssl`.
+
+    """
+
+    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
+                 net=None):
+        """Initialize.
+
+        :param directory: Directory Resource (`.messages.Directory`) or
+            URI from which the resource will be downloaded.
+
+        """
+        # pylint: disable=too-many-arguments
+        self.key = key
+        self.net = ClientNetwork(key, alg=alg, verify_ssl=verify_ssl) if net is None else net
+
+        if isinstance(directory, six.string_types):
+            directory = messages.Directory.from_json(
+                self.net.get(directory).json())
+        super(Client, self).__init__(directory=directory,
+            net=net, acme_version=1)
+
+    def register(self, new_reg=None):
+        """Register.
+
+        :param .NewRegistration new_reg:
+
+        :returns: Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        new_reg = messages.NewRegistration() if new_reg is None else new_reg
+        response = self.net.post(self.directory[new_reg], new_reg,
+            acme_version=1)
+        # TODO: handle errors
+        assert response.status_code == http_client.CREATED
+
+        # "Instance of 'Field' has no key/contact member" bug:
+        # pylint: disable=no-member
+        return self._regr_from_response(response)
+
+    def agree_to_tos(self, regr):
+        """Agree to the terms-of-service.
+
+        Agree to the terms-of-service in a Registration Resource.
+
+        :param regr: Registration Resource.
+        :type regr: `.RegistrationResource`
+
+        :returns: Updated Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        return self.update_registration(
+            regr.update(body=regr.body.update(agreement=regr.terms_of_service)))
+
+    def request_challenges(self, identifier, new_authzr_uri=None):
+        """Request challenges.
+
+        :param .messages.Identifier identifier: Identifier to be challenged.
+        :param str new_authzr_uri: Deprecated. Do not use.
+
+        :returns: Authorization Resource.
+        :rtype: `.AuthorizationResource`
+
+        """
+        if new_authzr_uri is not None:
+            logger.debug("request_challenges with new_authzr_uri deprecated.")
+        new_authz = messages.NewAuthorization(identifier=identifier)
+        response = self.net.post(self.directory.new_authz, new_authz,
+          acme_version=1)
+        # TODO: handle errors
+        assert response.status_code == http_client.CREATED
+        return self._authzr_from_response(response, identifier)
+
+    def request_domain_challenges(self, domain, new_authzr_uri=None):
+        """Request challenges for domain names.
+
+        This is simply a convenience function that wraps around
+        `request_challenges`, but works with domain names instead of
+        generic identifiers. See ``request_challenges`` for more
+        documentation.
+
+        :param str domain: Domain name to be challenged.
+        :param str new_authzr_uri: Deprecated. Do not use.
+
+        :returns: Authorization Resource.
+        :rtype: `.AuthorizationResource`
+
+        """
+        return self.request_challenges(messages.Identifier(
+            typ=messages.IDENTIFIER_FQDN, value=domain), new_authzr_uri)
+
     def request_issuance(self, csr, authzrs):
         """Request issuance.
 
@@ -311,7 +353,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
             self.directory.new_cert,
             req,
             content_type=content_type,
-            headers={'Accept': content_type})
+            headers={'Accept': content_type},
+            acme_version=1)
 
         cert_chain_uri = response.links.get('up', {}).get('url')
 
@@ -481,37 +524,65 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
                 "Recursion limit reached. Didn't get {0}".format(uri))
         return chain
 
-    def revoke(self, cert, rsn):
-        """Revoke certificate.
 
-        :param .ComparableX509 cert: `OpenSSL.crypto.X509` wrapped in
-            `.ComparableX509`
 
-        :param int rsn: Reason code for certificate revocation.
+class ClientV2(ClientBase):
+    """ACME client for a v2 API.
 
-        :raises .ClientError: If revocation is unsuccessful.
+    :ivar messages.Directory directory:
+    :ivar .ClientNetwork net: Client network.
+    """
+
+    def __init__(self, directory, net):
+        """Initialize.
+
+        :param .messages.Directory directory: Directory Resource
+        :param .ClientNetwork net: Client network.
+        """
+        super(ClientV2, self).__init__(directory=directory,
+            net=net, acme_version=2)
+
+    def new_account(self, new_account):
+        """Register.
+
+        :param .NewRegistration new_account:
+
+        :returns: Registration Resource.
+        :rtype: `.RegistrationResource`
 
         """
-        response = self.net.post(self.directory[messages.Revocation],
-                                 messages.Revocation(
-                                     certificate=cert,
-                                     reason=rsn),
-                                 content_type=None)
-        if response.status_code != http_client.OK:
-            raise errors.ClientError(
-                'Successful revocation must return HTTP OK status')
+        response = self.net.post(self.directory['newAccount'], new_account,
+            acme_version=2)
+        # "Instance of 'Field' has no key/contact member" bug:
+        # pylint: disable=no-member
+        return self._regr_from_response(response)
 
 
 class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
-    """Client network."""
+    """Wrapper around requests that signs POSTs for authentication.
+
+    Also adds user agent, and handles Content-Type.
+    """
     JSON_CONTENT_TYPE = 'application/json'
     JOSE_CONTENT_TYPE = 'application/jose+json'
     JSON_ERROR_CONTENT_TYPE = 'application/problem+json'
     REPLAY_NONCE_HEADER = 'Replay-Nonce'
 
-    def __init__(self, key, alg=jose.RS256, verify_ssl=True,
+    """Initialize.
+
+    :param  key: Account private key
+    :param messages.Registration account: Account object. Required if you are
+            planning to use .post() with acme_version=2.
+    :param josepy.JWASignature alg: Algoritm to use in signing JWS.
+    :param bool verify_ssl: Whether to verify certificates on SSL connections.
+    :param str user_agent: String to send as User-Agent header.
+    :param float timeout: Timeout for requests.
+    """
+    def __init__(self, key, account=None, alg=jose.RS256, verify_ssl=True,
                  user_agent='acme-python', timeout=DEFAULT_NETWORK_TIMEOUT):
+        # pylint: disable=too-many-arguments
         self.key = key
+        self.account = account
         self.alg = alg
         self.verify_ssl = verify_ssl
         self._nonces = set()
@@ -527,21 +598,29 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
         except Exception:  # pylint: disable=broad-except
             pass
 
-    def _wrap_in_jws(self, obj, nonce):
+    def _wrap_in_jws(self, obj, nonce, url, acme_version):
         """Wrap `JSONDeSerializable` object in JWS.
 
         .. todo:: Implement ``acmePath``.
 
         :param .JSONDeSerializable obj:
+        :param str url: The URL to which this object will be POSTed
         :param bytes nonce:
         :rtype: `.JWS`
 
         """
         jobj = obj.json_dumps(indent=2).encode()
         logger.debug('JWS payload:\n%s', jobj)
-        return jws.JWS.sign(
-            payload=jobj, key=self.key, alg=self.alg,
-            nonce=nonce).json_dumps(indent=2)
+        kwargs = {
+            "alg": self.alg,
+            "nonce": nonce
+        }
+        if acme_version == 2:
+            kwargs["url"] = url
+            kwargs["kid"] = self.account["uri"]
+        kwargs["key"] = self.key
+        # pylint: disable=star-args
+        return jws.JWS.sign(jobj, **kwargs).json_dumps(indent=2)
 
     @classmethod
     def _check_response(cls, response, content_type=None):
@@ -714,8 +793,9 @@ class ClientNetwork(object):  # pylint: disable=too-many-instance-attributes
             else:
                 raise
 
-    def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE, **kwargs):
-        data = self._wrap_in_jws(obj, self._get_nonce(url))
+    def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE,
+            acme_version=1, **kwargs):
+        data = self._wrap_in_jws(obj, self._get_nonce(url), url, acme_version)
         kwargs.setdefault('headers', {'Content-Type': content_type})
         response = self._send_request('POST', url, data=data, **kwargs)
         self._add_nonce(response)
diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py
index 84620fc99..662c32942 100644
--- a/acme/acme/client_test.py
+++ b/acme/acme/client_test.py
@@ -104,6 +104,23 @@ class ClientTest(unittest.TestCase):
         self.assertEqual(self.regr, self.client.register(self.new_reg))
         # TODO: test POST call arguments
 
+    def test_new_account_v2(self):
+        directory = messages.Directory({
+            "newAccount": 'https://www.letsencrypt-demo.org/acme/new-account',
+        })
+        from acme.client import ClientV2
+        client = ClientV2(directory, self.net)
+        self.response.status_code = http_client.CREATED
+        self.response.json.return_value = self.regr.body.to_json()
+        self.response.headers['Location'] = self.regr.uri
+
+        self.regr = messages.RegistrationResource(
+            body=messages.Registration(
+                contact=self.contact, key=KEY.public_key()),
+            uri='https://www.letsencrypt-demo.org/acme/reg/1')
+
+        self.assertEqual(self.regr, client.new_account(self.regr))
+
     def test_update_registration(self):
         # "Instance of 'Field' has no to_json/update member" bug:
         # pylint: disable=no-member
@@ -142,20 +159,23 @@ class ClientTest(unittest.TestCase):
         self.client.request_challenges(self.identifier)
         self.net.post.assert_called_once_with(
             self.directory.new_authz,
-            messages.NewAuthorization(identifier=self.identifier))
+            messages.NewAuthorization(identifier=self.identifier),
+            acme_version=1)
 
     def test_request_challenges_deprecated_arg(self):
         self._prepare_response_for_request_challenges()
         self.client.request_challenges(self.identifier, new_authzr_uri="hi")
         self.net.post.assert_called_once_with(
             self.directory.new_authz,
-            messages.NewAuthorization(identifier=self.identifier))
+            messages.NewAuthorization(identifier=self.identifier),
+            acme_version=1)
 
     def test_request_challenges_custom_uri(self):
         self._prepare_response_for_request_challenges()
         self.client.request_challenges(self.identifier)
         self.net.post.assert_called_once_with(
-            'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY)
+            'https://www.letsencrypt-demo.org/acme/new-authz', mock.ANY,
+            acme_version=1)
 
     def test_request_challenges_unexpected_update(self):
         self._prepare_response_for_request_challenges()
@@ -417,7 +437,8 @@ class ClientTest(unittest.TestCase):
     def test_revoke(self):
         self.client.revoke(self.certr.body, self.rsn)
         self.net.post.assert_called_once_with(
-            self.directory[messages.Revocation], mock.ANY, content_type=None)
+            self.directory[messages.Revocation], mock.ANY, content_type=None,
+            acme_version=1)
 
     def test_revocation_payload(self):
         obj = messages.Revocation(certificate=self.certr.body, reason=self.rsn)
@@ -432,9 +453,22 @@ class ClientTest(unittest.TestCase):
             self.certr,
             self.rsn)
 
+class MockJSONDeSerializable(jose.JSONDeSerializable):
+    # pylint: disable=missing-docstring
+    def __init__(self, value):
+        self.value = value
+
+    def to_partial_json(self):
+        return {'foo': self.value}
+
+    @classmethod
+    def from_json(cls, value):
+        pass  # pragma: no cover
+
 
 class ClientNetworkTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork."""
+    # pylint: disable=too-many-public-methods
 
     def setUp(self):
         self.verify_ssl = mock.MagicMock()
@@ -453,25 +487,27 @@ class ClientNetworkTest(unittest.TestCase):
         self.assertTrue(self.net.verify_ssl is self.verify_ssl)
 
     def test_wrap_in_jws(self):
-        class MockJSONDeSerializable(jose.JSONDeSerializable):
-            # pylint: disable=missing-docstring
-            def __init__(self, value):
-                self.value = value
-
-            def to_partial_json(self):
-                return {'foo': self.value}
-
-            @classmethod
-            def from_json(cls, value):
-                pass  # pragma: no cover
-
         # pylint: disable=protected-access
         jws_dump = self.net._wrap_in_jws(
-            MockJSONDeSerializable('foo'), nonce=b'Tg')
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url",
+            acme_version=1)
         jws = acme_jws.JWS.json_loads(jws_dump)
         self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
         self.assertEqual(jws.signature.combined.nonce, b'Tg')
 
+    def test_wrap_in_jws_v2(self):
+        self.net.account = {'uri': 'acct-uri'}
+        # pylint: disable=protected-access
+        jws_dump = self.net._wrap_in_jws(
+            MockJSONDeSerializable('foo'), nonce=b'Tg', url="url",
+            acme_version=2)
+        jws = acme_jws.JWS.json_loads(jws_dump)
+        self.assertEqual(json.loads(jws.payload.decode()), {'foo': 'foo'})
+        self.assertEqual(jws.signature.combined.nonce, b'Tg')
+        self.assertEqual(jws.signature.combined.kid, u'acct-uri')
+        self.assertEqual(jws.signature.combined.url, u'url')
+
+
     def test_check_response_not_ok_jobj_no_error(self):
         self.response.ok = False
         self.response.json.return_value = {}
@@ -701,13 +737,13 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.assertEqual(self.checked_response, self.net.post(
             'uri', self.obj, content_type=self.content_type))
         self.net._wrap_in_jws.assert_called_once_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1)
 
         self.available_nonces = []
         self.assertRaises(errors.MissingNonce, self.net.post,
                           'uri', self.obj, content_type=self.content_type)
         self.net._wrap_in_jws.assert_called_with(
-            self.obj, jose.b64decode(self.all_nonces.pop()))
+            self.obj, jose.b64decode(self.all_nonces.pop()), "uri", 1)
 
     def test_post_wrong_initial_nonce(self):  # HEAD
         self.available_nonces = [b'f', jose.b64encode(b'good')]
diff --git a/acme/acme/messages.py b/acme/acme/messages.py
index 6daf55094..98993c4e1 100644
--- a/acme/acme/messages.py
+++ b/acme/acme/messages.py
@@ -251,6 +251,7 @@ class Registration(ResourceBody):
     contact = jose.Field('contact', omitempty=True, default=())
     agreement = jose.Field('agreement', omitempty=True)
     status = jose.Field('status', omitempty=True)
+    terms_of_service_agreed = jose.Field('terms-of-service-agreed', omitempty=True)
 
     phone_prefix = 'tel:'
     email_prefix = 'mailto:'

From 0416382633c7b37f05193c0bc42d10d03ac13b5e Mon Sep 17 00:00:00 2001
From: Brad Warren <bmw@users.noreply.github.com>
Date: Tue, 6 Feb 2018 17:01:58 -0800
Subject: [PATCH 45/45] Update leauto_upgrades with tests from #5402. (#5407)

---
 .../letstest/scripts/test_leauto_upgrades.sh  | 29 +++++++++++++------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index 51472f2e6..355fead2e 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -65,6 +65,7 @@ iQIDAQAB
 "
 
 if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then
+    RUN_PYTHON3_TESTS=1
     if command -v python3; then
         echo "Didn't expect Python 3 to be installed!"
         exit 1
@@ -85,11 +86,25 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ];
         exit 1
     fi
     unset VENV_PATH
-    EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
-    if ! ./cb-auto -v --debug --version -n 2>&1 | grep "$EXPECTED_VERSION" ; then
-        echo "Certbot didn't upgrade as expected!"
-        exit 1
-    fi
+fi
+
+if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then
+    echo "Had problems checking for updates!"
+    exit 1
+fi
+
+EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION certbot-auto | cut -d\" -f2)
+if ! /opt/eff.org/certbot/venv/bin/letsencrypt --version 2>&1 | grep "$EXPECTED_VERSION" ; then
+    echo upgrade appeared to fail
+    exit 1
+fi
+
+if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
+    echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ
+    exit 1
+fi
+
+if [ "$RUN_PYTHON3_TESTS" = 1 ]; then
     if ! command -v python3; then
         echo "Python3 wasn't properly installed"
         exit 1
@@ -98,11 +113,7 @@ if [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ];
         echo "Python3 wasn't used in venv!"
         exit 1
     fi
-elif ! ./letsencrypt-auto -v --debug --version || ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
-    echo upgrade appeared to fail
-    exit 1
 fi
-
 echo upgrade appeared to be successful
 
 if [ "$(tools/readlink.py ${XDG_DATA_HOME:-~/.local/share}/letsencrypt)" != "/opt/eff.org/certbot/venv" ]; then