From 3d0e16ece3762d5bfe345fde0af286e26c54bacb Mon Sep 17 00:00:00 2001
From: Adrien Ferrand <adferrand@users.noreply.github.com>
Date: Thu, 8 Nov 2018 02:16:16 +0100
Subject: [PATCH] [Windows|Unix] Rewrite bash scripts for tests into python
 (#6435)

Certbot relies heavily on bash scripts to deploy a development environment and to execute tests. This is fine for Linux systems, including Travis, but problematic for Windows machines.

This PR converts all theses scripts into Python, to make them platform independant.

As a consequence, tox-win.ini is not needed anymore, and tox can be run indifferently on Windows or on Linux using a common tox.ini. AppVeyor is updated accordingly to execute tests for acme, certbot and all dns plugins. Other tests are not executed as they are for Docker, unsupported Apache/Nginx/Postfix plugins (for now) or not relevant for Windows (explicit Linux distribution tests or pylint).

Another PR will be done on certbot website to update how a dev environment can be set up.

* Replace several shell scripts by python equivalent.

* Correction on tox coverage

* Extend usage of new python scripts

* Various corrections

* Replace venv construction bash scripts by python equivalents

* Update tox.ini

* Unicode lines to compare files

* Put modifications on letsencrypt-auto-source instead of generated scripts

* Add executable permissions for Linux.

* Merge tox win tests into main tox

* Skip lock_test on Windows

* Correct appveyor config

* Update appveyor.yml

* Explicit coverage py27 or py37

* Avoid to cover non supported certbot plugins on Windows

* Update tox.ini

* Remove specific warnings during CI

* No cover on a debug code for tests only.

* Update documentation and help script on venv/venv3.py

* Customize help message for Windows

* Quote correctly executable path with potential spaces in it.

* Copy pipstrap from upstream
---
 .travis.yml                                   |   4 +-
 Dockerfile-dev                                |   2 +-
 appveyor.yml                                  |  25 +++-
 certbot-compatibility-test/Dockerfile         |   5 +-
 certbot-postfix/README.rst                    |   2 +-
 certbot/tests/util.py                         |  13 +-
 docs/contributing.rst                         |   8 +-
 letsencrypt-auto-source/letsencrypt-auto      |  11 +-
 .../pieces/bootstrappers/arch_common.sh       |   2 +-
 letsencrypt-auto-source/pieces/pipstrap.py    |  10 +-
 tests/letstest/scripts/test_apache2.sh        |   2 +-
 tests/letstest/scripts/test_tox.sh            |   2 +-
 tests/lock_test.py                            |   9 +-
 tests/modification-check.py                   | 124 ++++++++++++++++++
 tests/modification-check.sh                   |  59 ---------
 tools/_venv_common.py                         |  67 ++++++++++
 tools/_venv_common.sh                         |  26 ----
 tools/install_and_test.py                     |  58 ++++++++
 tools/install_and_test.sh                     |  29 ----
 tools/merge_requirements.py                   |  16 +--
 tools/pip_install.py                          |  90 +++++++++++++
 tools/pip_install.sh                          |  44 -------
 tools/pip_install_editable.py                 |  19 +++
 tools/pip_install_editable.sh                 |  10 --
 tools/readlink.py                             |   7 +-
 tools/venv.py                                 |  58 ++++++++
 tools/venv.sh                                 |  34 -----
 tools/venv3.py                                |  53 ++++++++
 tools/venv3.sh                                |  33 -----
 tox-win.ini                                   |  13 --
 tox.cover.py                                  |  85 ++++++++++++
 tox.cover.sh                                  |  72 ----------
 tox.ini                                       |  28 ++--
 33 files changed, 643 insertions(+), 377 deletions(-)
 create mode 100755 tests/modification-check.py
 delete mode 100755 tests/modification-check.sh
 create mode 100755 tools/_venv_common.py
 delete mode 100755 tools/_venv_common.sh
 create mode 100755 tools/install_and_test.py
 delete mode 100755 tools/install_and_test.sh
 create mode 100755 tools/pip_install.py
 delete mode 100755 tools/pip_install.sh
 create mode 100755 tools/pip_install_editable.py
 delete mode 100755 tools/pip_install_editable.sh
 create mode 100755 tools/venv.py
 delete mode 100755 tools/venv.sh
 create mode 100755 tools/venv3.py
 delete mode 100755 tools/venv3.sh
 delete mode 100644 tox-win.ini
 create mode 100755 tox.cover.py
 delete mode 100755 tox.cover.sh

diff --git a/.travis.yml b/.travis.yml
index acdf365bd..2b8eafc13 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ matrix:
       sudo: required
       services: docker
     - python: "2.7"
-      env: TOXENV=cover FYI="this also tests py27"
+      env: TOXENV=py27-cover FYI="py27 tests + code coverage"
     - sudo: required
       env: TOXENV=nginx_compat
       services: docker
@@ -95,7 +95,7 @@ script:
     - travis_retry tox
     - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
-after_success: '[ "$TOXENV" == "cover" ] && codecov'
+after_success: '[ "$TOXENV" == "py27-cover" ] && codecov'
 
 notifications:
   email: false
diff --git a/Dockerfile-dev b/Dockerfile-dev
index 9e35ebec8..1ab56e081 100644
--- a/Dockerfile-dev
+++ b/Dockerfile-dev
@@ -16,6 +16,6 @@ RUN apt-get update && \
            /tmp/* \
            /var/tmp/*
 
-RUN VENV_NAME="../venv" tools/venv.sh
+RUN VENV_NAME="../venv" python tools/venv.py
 
 ENV PATH /opt/certbot/venv/bin:$PATH
diff --git a/appveyor.yml b/appveyor.yml
index 796070081..725ecfbff 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,8 +1,17 @@
-image:
-  # => Windows Server 2012 R2
-  - Visual Studio 2015
-  # => Windows Server 2016
-  - Visual Studio 2017
+environment:
+  matrix:
+    - FYI: Python 3.4 on Windows Server 2012 R2
+      TOXENV: py34
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+    - FYI: Python 3.4 on Windows Server 2016
+      TOXENV: py34
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - FYI: Python 3.5 on Windows Server 2016
+      TOXENV: py35
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+    - FYI: Python 3.7 on Windows Server 2016 + code coverage
+      TOXENV: py37-cover
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
 branches:
   only:
@@ -14,6 +23,7 @@ install:
   # Use Python 3.7 by default
   - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
   # Check env
+  - "echo %APPVEYOR_BUILD_WORKER_IMAGE%"
   - "python --version"
   # Upgrade pip to avoid warnings
   - "python -m pip install --upgrade pip"
@@ -23,7 +33,8 @@ install:
 build: off
 
 test_script:
-  - tox -c tox-win.ini -e py34,py35,py36,py37-cover
+  # Test env is set by TOXENV env variable
+  - tox
 
 on_success:
-  - codecov
+  - if exist .coverage codecov
diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile
index 803b4a1b9..cbbdf7193 100644
--- a/certbot-compatibility-test/Dockerfile
+++ b/certbot-compatibility-test/Dockerfile
@@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/
 
 # all above files are necessary for setup.py, however, package source
 # code directory has to be copied separately to a subdirectory...
@@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \
     /opt/certbot/venv/bin/pip install -U setuptools && \
     /opt/certbot/venv/bin/pip install -U pip
 ENV PATH /opt/certbot/venv/bin:$PATH
-RUN /opt/certbot/src/tools/pip_install_editable.sh \
+RUN /opt/certbot/venv/bin/python \
+    /opt/certbot/src/tools/pip_install_editable.py \
     /opt/certbot/src/acme \
     /opt/certbot/src \
     /opt/certbot/src/certbot-apache \
diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst
index e6367e365..1ae9cb980 100644
--- a/certbot-postfix/README.rst
+++ b/certbot-postfix/README.rst
@@ -7,7 +7,7 @@ feature requests for this plugin.
 
 To install this plugin, in the root of this repo, run::
 
-    ./tools/venv.sh
+    python tools/venv.py
     source venv/bin/activate
 
 You can use this installer with any `authenticator plugin
diff --git a/certbot/tests/util.py b/certbot/tests/util.py
index 822597dd4..8c5db2c2f 100644
--- a/certbot/tests/util.py
+++ b/certbot/tests/util.py
@@ -328,15 +328,16 @@ class TempDirTestCase(unittest.TestCase):
 
     def tearDown(self):
         """Execute after test"""
-        # Then we have various files which are not correctly closed at the time of tearDown.
-        # On Windows, it is visible for the same reasons as above.
+        # On Windows we have various files which are not correctly closed at the time of tearDown.
         # For know, we log them until a proper file close handling is written.
+        # Useful for development only, so no warning when we are on a CI process.
         def onerror_handler(_, path, excinfo):
             """On error handler"""
-            message = ('Following error occurred when deleting the tempdir {0}'
-                       ' for path {1} during tearDown process: {2}'
-                       .format(self.tempdir, path, str(excinfo)))
-            warnings.warn(message)
+            if not os.environ.get('APPVEYOR'): # pragma: no cover
+                message = ('Following error occurred when deleting the tempdir {0}'
+                           ' for path {1} during tearDown process: {2}'
+                           .format(self.tempdir, path, str(excinfo)))
+                warnings.warn(message)
         shutil.rmtree(self.tempdir, onerror=onerror_handler)
 
 class ConfigTestCase(TempDirTestCase):
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 58db251d4..ead4d7e2b 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -38,13 +38,13 @@ Certbot.
 
    cd certbot
    ./certbot-auto --debug --os-packages-only
-   tools/venv.sh
+   python tools/venv.py
 
-If you have Python3 available and want to use it, run the ``venv3.sh`` script.
+If you have Python3 available and want to use it, run the ``venv3.py`` script.
 
 .. code-block:: shell
 
-   tools/venv3.sh
+   python tools/venv3.py
 
 .. note:: You may need to repeat this when
   Certbot's dependencies change or when a new plugin is introduced.
@@ -353,7 +353,7 @@ Steps:
 
 1. Write your code!
 2. Make sure your environment is set up properly and that you're in your
-   virtualenv. You can do this by running ``./tools/venv.sh``.
+   virtualenv. You can do this by running ``pip tools/venv.py``.
    (this is a **very important** step)
 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors.
 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite
diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto
index b1a05f6e6..12be26e19 100755
--- a/letsencrypt-auto-source/letsencrypt-auto
+++ b/letsencrypt-auto-source/letsencrypt-auto
@@ -594,7 +594,7 @@ BootstrapArchCommon() {
   #
   # "python-virtualenv" is Python3, but "python2-virtualenv" provides
   # only "virtualenv2" binary, not "virtualenv" necessary in
-  # ./tools/_venv_common.sh
+  # ./tools/_venv_common.py
 
   deps="
     python2
@@ -1260,7 +1260,7 @@ except ImportError:
                 cmd = popenargs[0]
             raise CalledProcessError(retcode, cmd)
         return output
-from sys import exit, version_info
+from sys import exit, version_info, executable
 from tempfile import mkdtemp
 try:
     from urllib2 import build_opener, HTTPHandler, HTTPSHandler
@@ -1272,7 +1272,7 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 5, 1
+__version__ = 2, 0, 0
 PIP_VERSION = '9.0.1'
 DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
@@ -1365,7 +1365,7 @@ def get_index_base():
 
 
 def main():
-    pip_version = StrictVersion(check_output(['pip', '--version'])
+    pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version'])
                                 .decode('utf-8').split()[1])
     min_pip_version = StrictVersion(PIP_VERSION)
     if pip_version >= min_pip_version:
@@ -1378,7 +1378,7 @@ def main():
                                      temp,
                                      digest)
                      for path, digest in PACKAGES]
-        check_output('pip install --no-index --no-deps -U ' +
+        check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) +
                      # Disable cache since we're not using it and it otherwise
                      # sometimes throws permission warnings:
                      ('--no-cache-dir ' if has_pip_cache else '') +
@@ -1397,7 +1397,6 @@ def main():
 
 if __name__ == '__main__':
     exit(main())
-
 UNLIKELY_EOF
     # -------------------------------------------------------------------------
     # Set PATH so pipstrap upgrades the right (v)env:
diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh
index 5759336c5..c55527590 100755
--- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh
+++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh
@@ -8,7 +8,7 @@ BootstrapArchCommon() {
   #
   # "python-virtualenv" is Python3, but "python2-virtualenv" provides
   # only "virtualenv2" binary, not "virtualenv" necessary in
-  # ./tools/_venv_common.sh
+  # ./tools/_venv_common.py
 
   deps="
     python2
diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py
index d55d5bceb..f21d36657 100755
--- a/letsencrypt-auto-source/pieces/pipstrap.py
+++ b/letsencrypt-auto-source/pieces/pipstrap.py
@@ -45,7 +45,7 @@ except ImportError:
                 cmd = popenargs[0]
             raise CalledProcessError(retcode, cmd)
         return output
-from sys import exit, version_info
+from sys import exit, version_info, executable
 from tempfile import mkdtemp
 try:
     from urllib2 import build_opener, HTTPHandler, HTTPSHandler
@@ -57,7 +57,7 @@ except ImportError:
     from urllib.parse import urlparse  # 3.4
 
 
-__version__ = 1, 5, 1
+__version__ = 2, 0, 0
 PIP_VERSION = '9.0.1'
 DEFAULT_INDEX_BASE = 'https://pypi.python.org'
 
@@ -150,7 +150,7 @@ def get_index_base():
 
 
 def main():
-    pip_version = StrictVersion(check_output(['pip', '--version'])
+    pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version'])
                                 .decode('utf-8').split()[1])
     min_pip_version = StrictVersion(PIP_VERSION)
     if pip_version >= min_pip_version:
@@ -163,7 +163,7 @@ def main():
                                      temp,
                                      digest)
                      for path, digest in PACKAGES]
-        check_output('pip install --no-index --no-deps -U ' +
+        check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) +
                      # Disable cache since we're not using it and it otherwise
                      # sometimes throws permission warnings:
                      ('--no-cache-dir ' if has_pip_cache else '') +
@@ -181,4 +181,4 @@ def main():
 
 
 if __name__ == '__main__':
-    exit(main())
+    exit(main())
\ No newline at end of file
diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh
index 6b5d63c80..4036e6efa 100755
--- a/tests/letstest/scripts/test_apache2.sh
+++ b/tests/letstest/scripts/test_apache2.sh
@@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then
     exit 1
 fi
 
-tools/_venv_common.sh -e acme[dev] -e .[dev,docs] -e certbot-apache
+python tools/_venv_common.py -e acme[dev] -e .[dev,docs] -e certbot-apache
 sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh
index 84e4bcd22..bb9126673 100755
--- a/tests/letstest/scripts/test_tox.sh
+++ b/tests/letstest/scripts/test_tox.sh
@@ -14,5 +14,5 @@ VENV_BIN=${VENV_PATH}/bin
 "$LEA_PATH/letsencrypt-auto" --os-packages-only
 
 cd letsencrypt
-./tools/venv.sh
+python tools/venv.py
 venv/bin/tox -e py27
diff --git a/tests/lock_test.py b/tests/lock_test.py
index b01cc5d58..0266cf029 100644
--- a/tests/lock_test.py
+++ b/tests/lock_test.py
@@ -1,4 +1,6 @@
 """Tests to ensure the lock order is preserved."""
+from __future__ import print_function
+
 import atexit
 import functools
 import logging
@@ -235,4 +237,9 @@ def log_output(level, out, err):
 
 
 if __name__ == "__main__":
-    main()
+    if os.name != 'nt':
+        main()
+    else:
+        print(
+            'Warning: lock_test cannot be executed on Windows, '
+            'as it relies on a Nginx distribution for Linux.')
diff --git a/tests/modification-check.py b/tests/modification-check.py
new file mode 100755
index 000000000..e00994b04
--- /dev/null
+++ b/tests/modification-check.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+import tempfile
+import shutil
+try:
+    from urllib.request import urlretrieve
+except ImportError:
+    from urllib import urlretrieve
+
+def find_repo_path():
+    return os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+# We do not use filecmp.cmp to take advantage of universal newlines 
+# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows.
+# As a consequence, this function will not work correctly if executed by Python 2.x on Windows.
+# But it will work correctly on Linux for any version, because every file tested will be LF.
+def compare_files(path_1, path_2):
+    l1 = l2 = True
+    with open(path_1, 'r') as f1, open(path_2, 'r') as f2:
+        line = 1
+        while l1 and l2:
+            line += 1
+            l1 = f1.readline()
+            l2 = f2.readline()
+            if l1 != l2:
+                print('---')
+                print((
+                    'While comparing {0} (1) and {1} (2), a difference was found at line {2}:'
+                    .format(os.path.basename(path_1), os.path.basename(path_2), line)))
+                print('(1): {0}'.format(repr(l1)))
+                print('(2): {0}'.format(repr(l2)))
+                print('---')
+                return False
+
+    return True
+
+def validate_scripts_content(repo_path, temp_cwd):
+    errors = False
+
+    if not compare_files(
+            os.path.join(repo_path, 'certbot-auto'),
+            os.path.join(repo_path, 'letsencrypt-auto')):
+        print('Root certbot-auto and letsencrypt-auto differ.')
+        errors = True
+    else:
+        shutil.copyfile(
+            os.path.join(repo_path, 'certbot-auto'), 
+            os.path.join(temp_cwd, 'local-auto'))
+        shutil.copy(os.path.normpath(os.path.join(
+            repo_path, 
+            'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd)
+
+        # Compare file against current version in the target branch
+        branch = os.environ.get('TRAVIS_BRANCH', 'master')
+        url = (
+            'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto'
+            .format(branch))
+        urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto'))
+
+        if compare_files(
+                os.path.join(temp_cwd, 'certbot-auto'),
+                os.path.join(temp_cwd, 'local-auto')):
+            print('Root *-auto were unchanged')
+        else:
+            # Compare file against the latest released version
+            latest_version = subprocess.check_output(
+                [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd)
+            subprocess.call(
+                [sys.executable, 'fetch.py', '--le-auto-script', 
+                 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd)
+            if compare_files(
+                    os.path.join(temp_cwd, 'letsencrypt-auto'),
+                    os.path.join(temp_cwd, 'local-auto')):
+                print('Root *-auto were updated to the latest version.')
+            else:
+                print('Root *-auto have unexpected changes.')
+                errors = True
+
+    return errors
+
+def main():
+    repo_path = find_repo_path()
+    temp_cwd = tempfile.mkdtemp()
+    errors = False
+
+    try:
+        errors = validate_scripts_content(repo_path, temp_cwd)
+
+        shutil.copyfile(
+            os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')),
+            os.path.join(temp_cwd, 'original-lea')
+        )
+        subprocess.call([sys.executable, os.path.normpath(os.path.join(
+            repo_path, 'letsencrypt-auto-source/build.py'))])
+        shutil.copyfile(
+            os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')),
+            os.path.join(temp_cwd, 'build-lea')
+        )
+        shutil.copyfile(
+            os.path.join(temp_cwd, 'original-lea'),
+            os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto'))
+        )
+
+        if not compare_files(
+                os.path.join(temp_cwd, 'original-lea'),
+                os.path.join(temp_cwd, 'build-lea')):
+            print('Script letsencrypt-auto-source/letsencrypt-auto '
+                  'doesn\'t match output of build.py.')
+            errors = True
+        else:
+            print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.')
+    finally:
+        shutil.rmtree(temp_cwd)
+
+    return errors
+
+if __name__ == '__main__':
+    if main():
+        sys.exit(1)
diff --git a/tests/modification-check.sh b/tests/modification-check.sh
deleted file mode 100755
index 0145b0228..000000000
--- a/tests/modification-check.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash -e
-
-temp_dir=`mktemp -d`
-trap "rm -rf $temp_dir" EXIT
-
-# cd to repo root
-cd $(dirname $(dirname $(readlink -f $0)))
-FLAG=false
-
-if ! cmp -s certbot-auto letsencrypt-auto; then
-    echo "Root certbot-auto and letsencrypt-auto differ."
-    FLAG=true
-else
-    cp certbot-auto "$temp_dir/local-auto"
-    cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py"
-    cd $temp_dir
-
-    # Compare file against current version in the target branch
-    BRANCH=${TRAVIS_BRANCH:-master}
-    URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto"
-    curl -sS $URL > certbot-auto
-    if cmp -s certbot-auto local-auto; then
-        echo "Root *-auto were unchanged."
-    else
-        # Compare file against the latest released version
-        python fetch.py --le-auto-script "v$(python fetch.py --latest-version)"
-        if cmp -s letsencrypt-auto local-auto; then
-            echo "Root *-auto were updated to the latest version."
-        else
-            echo "Root *-auto have unexpected changes."
-            FLAG=true
-        fi
-    fi
-    cd ~-
-fi
-
-# Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py
-
-cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea
-python letsencrypt-auto-source/build.py
-cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/build-lea
-cp ${temp_dir}/original-lea letsencrypt-auto-source/letsencrypt-auto
-
-cd $temp_dir
-
-if ! cmp -s original-lea build-lea; then
-	echo "letsencrypt-auto-source/letsencrypt-auto doesn't match output of \
-build.py."
-	FLAG=true
-else
-	echo "letsencrypt-auto-source/letsencrypt-auto matches output of \
-build.py."
-fi
-
-rm -rf $temp_dir
-
-if $FLAG ; then
-	exit 1
-fi
diff --git a/tools/_venv_common.py b/tools/_venv_common.py
new file mode 100755
index 000000000..0c24664b3
--- /dev/null
+++ b/tools/_venv_common.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import shutil
+import glob
+import time
+import subprocess
+import sys
+
+def subprocess_with_print(command):
+    print(command)
+    subprocess.call(command, shell=True)
+
+def get_venv_python(venv_path):
+    python_linux = os.path.join(venv_path, 'bin/python')
+    python_windows = os.path.join(venv_path, 'Scripts\\python.exe')
+    if os.path.isfile(python_linux):
+        return python_linux
+    if os.path.isfile(python_windows):
+        return python_windows
+
+    raise ValueError((
+        'Error, could not find python executable in venv path {0}: is it a valid venv ?'
+        .format(venv_path)))
+
+def main(venv_name, venv_args, args):
+    for path in glob.glob('*.egg-info'):
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+    if os.path.isdir(venv_name):
+        os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time())))
+
+    subprocess_with_print(' '.join([
+        sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools',
+        venv_name, venv_args]))
+
+    python_executable = get_venv_python(venv_name)
+
+    subprocess_with_print(' '.join([
+        python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')]))
+    command = [python_executable, os.path.normpath('./tools/pip_install.py')]
+    command.extend(args)
+    subprocess_with_print(' '.join(command))
+
+    if os.path.isdir(os.path.join(venv_name, 'bin')):
+        # Linux/OSX specific
+        print('-------------------------------------------------------------------')
+        print('Please run the following command to activate developer environment:')
+        print('source {0}/bin/activate'.format(venv_name))
+        print('-------------------------------------------------------------------')
+    elif os.path.isdir(os.path.join(venv_args, 'Scripts')):
+        # Windows specific
+        print('---------------------------------------------------------------------------')
+        print('Please run one of the following commands to activate developer environment:')
+        print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name))
+        print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name))
+        print('---------------------------------------------------------------------------')
+    else:
+        raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name))
+
+if __name__ == '__main__':
+    main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:])
diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh
deleted file mode 100755
index 0f0ff7e28..000000000
--- a/tools/_venv_common.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/sh -xe
-
-VENV_NAME=${VENV_NAME:-venv}
-
-# .egg-info directories tend to cause bizarre problems (e.g. `pip -e
-# .` might unexpectedly install letshelp-certbot only, in case
-# `python letshelp-certbot/setup.py build` has been called
-# earlier)
-rm -rf *.egg-info
-
-# virtualenv setup is NOT idempotent: shutil.Error:
-# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and
-# `venv/bin/python2` are the same file
-mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
-virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
-. ./$VENV_NAME/bin/activate
-
-# Use pipstrap to update Python packaging tools to only update to a well tested
-# version and to work around https://github.com/pypa/pip/issues/4817 on older
-# systems.
-python letsencrypt-auto-source/pieces/pipstrap.py
-./tools/pip_install.sh "$@"
-
-set +x
-echo "Please run the following command to activate developer environment:"
-echo "source $VENV_NAME/bin/activate"
diff --git a/tools/install_and_test.py b/tools/install_and_test.py
new file mode 100755
index 000000000..b16181aa5
--- /dev/null
+++ b/tools/install_and_test.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# pip installs the requested packages in editable mode and runs unit tests on
+# them. Each package is installed and tested in the order they are provided
+# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.py for more information on the versions pinned
+# to.
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+import shutil
+import subprocess
+import re
+
+SKIP_PROJECTS_ON_WINDOWS = [
+    'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot']
+
+def call_with_print(command, cwd=None):
+    print(command)
+    subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    if os.environ.get('CERTBOT_NO_PIN') == '1':
+        command = [sys.executable, '-m', 'pip', '-q', '-e']
+    else:
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
+
+    new_args = []
+    for arg in args:
+        if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS:
+            print((
+                'Info: currently {0} is not supported on Windows and will not be tested.'
+                .format(arg)))
+        else:
+            new_args.append(arg)
+
+    for requirement in new_args:
+        current_command = command[:]
+        current_command.append(requirement)
+        call_with_print(' '.join(current_command))
+        pkg = re.sub(r'\[\w+\]', '', requirement)
+
+        if pkg == '.':
+            pkg = 'certbot'
+
+        temp_cwd = tempfile.mkdtemp()
+        try:
+            call_with_print(' '.join([
+                sys.executable, '-m', 'pytest', '--numprocesses', 'auto',
+                '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd)
+        finally:
+            shutil.rmtree(temp_cwd)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh
deleted file mode 100755
index 819f683aa..000000000
--- a/tools/install_and_test.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh -e
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using pinned versions of all of our
-# dependencies. See pip_install.sh for more information on the versions pinned
-# to.
-
-if [ "$CERTBOT_NO_PIN" = 1 ]; then
-  pip_install="pip install -q -e"
-else
-  pip_install="$(dirname $0)/pip_install_editable.sh"
-fi
-
-temp_cwd=$(mktemp -d)
-trap "rm -rf $temp_cwd" EXIT
-
-set -x
-for requirement in "$@" ; do
-  $pip_install $requirement
-  pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
-  pkg=$(echo "$pkg" | tr - _ )  # convert package names to Python import names
-  if [ $pkg = "." ]; then
-    pkg="certbot"
-  fi
-  cd "$temp_cwd"
-  pytest --numprocesses auto --quiet --pyargs $pkg
-  cd -
-done
diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py
index c8fb95351..ad44a55d0 100755
--- a/tools/merge_requirements.py
+++ b/tools/merge_requirements.py
@@ -10,7 +10,6 @@ from __future__ import print_function
 
 import sys
 
-
 def read_file(file_path):
     """Reads in a Python requirements file.
 
@@ -32,17 +31,17 @@ def read_file(file_path):
     return d
 
 
-def print_requirements(requirements):
-    """Prints requirements to stdout.
+def output_requirements(requirements):
+    """Prepare print requirements to stdout.
 
     :param dict requirements: mapping from a project to its pinned version
 
     """
-    print('\n'.join('{0}=={1}'.format(k, v)
-          for k, v in sorted(requirements.items())))
+    return '\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items()))
 
 
-def merge_requirements_files(*files):
+def main(*files):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
@@ -54,8 +53,9 @@ def merge_requirements_files(*files):
     d = {}
     for f in files:
         d.update(read_file(f))
-    print_requirements(d)
+    return output_requirements(d)
 
 
 if __name__ == '__main__':
-    merge_requirements_files(*sys.argv[1:])
+    merged_requirements = main(*sys.argv[1:])
+    print(merged_requirements)
diff --git a/tools/pip_install.py b/tools/pip_install.py
new file mode 100755
index 000000000..d09997bf5
--- /dev/null
+++ b/tools/pip_install.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt,
+# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# top level of the package's directory is used, otherwise, a combination of
+# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
+# other file always takes precedence over tools/dev_constraints.txt. If
+# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
+# no other arguments.
+
+from __future__ import print_function, absolute_import
+
+import subprocess
+import os
+import sys
+import re
+import shutil
+import tempfile
+
+import merge_requirements as merge_module
+import readlink
+
+def find_tools_path():
+    return os.path.dirname(readlink.main(__file__))
+
+def certbot_oldest_processing(tools_path, args, test_constraints):
+    if args[0] != '-e' or len(args) != 2:
+        raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
+                         'with a single -e <path> argument.')
+    # remove any extras such as [dev]
+    pkg_dir = re.sub(r'\[\w+\]', '', args[1])
+    requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
+    # packages like acme don't have any local oldest requirements
+    if not os.path.isfile(requirements):
+        requirements = None
+    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
+
+    return requirements
+
+def certbot_normal_processing(tools_path, test_constraints):
+    repo_path = os.path.dirname(tools_path)
+    certbot_requirements = os.path.normpath(os.path.join(
+        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
+    with open(certbot_requirements, 'r') as fd:
+        data = fd.readlines()
+    with open(test_constraints, 'w') as fd:
+        for line in data:
+            search = re.search(r'^(\S*==\S*).*$', line)
+            if search:
+                fd.write('{0}{1}'.format(search.group(1), os.linesep))
+
+def merge_requirements(tools_path, test_constraints, all_constraints):
+    merged_requirements = merge_module.main(
+        os.path.join(tools_path, 'dev_constraints.txt'),
+        test_constraints
+    )
+    with open(all_constraints, 'w') as fd:
+        fd.write(merged_requirements)
+
+def call_with_print(command, cwd=None):
+    print(command)
+    subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    tools_path = find_tools_path()
+    working_dir = tempfile.mkdtemp()
+    try:
+        test_constraints = os.path.join(working_dir, 'test_constraints.txt')
+        all_constraints = os.path.join(working_dir, 'all_constraints.txt')
+
+        requirements = None
+        if os.environ.get('CERTBOT_OLDEST') == '1':
+            requirements = certbot_oldest_processing(tools_path, args, test_constraints)
+        else:
+            certbot_normal_processing(tools_path, test_constraints)
+
+        merge_requirements(tools_path, test_constraints, all_constraints)
+        if requirements:
+            call_with_print(' '.join([
+                sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints,
+                '--requirement', requirements]))
+
+        command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints]
+        command.extend(args)
+        call_with_print(' '.join(command))
+    finally:
+        shutil.rmtree(working_dir)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/tools/pip_install.sh b/tools/pip_install.sh
deleted file mode 100755
index 78e2afa17..000000000
--- a/tools/pip_install.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/sh -e
-# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
-# to 1, a combination of tools/oldest_constraints.txt,
-# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
-# top level of the package's directory is used, otherwise, a combination of
-# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
-# other file always takes precedence over tools/dev_constraints.txt. If
-# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
-# no other arguments.
-
-# get the root of the Certbot repo
-tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
-all_constraints=$(mktemp)
-test_constraints=$(mktemp)
-trap "rm -f $all_constraints $test_constraints" EXIT
-
-if [ "$CERTBOT_OLDEST" = 1 ]; then
-    if [ "$1" != "-e" -o "$#" -ne "2" ]; then
-        echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument."
-        exit 1
-    fi
-    pkg_dir=$(echo $2 | cut -f1 -d\[)  # remove any extras such as [dev]
-    requirements="$pkg_dir/local-oldest-requirements.txt"
-    # packages like acme don't have any local oldest requirements
-    if [ ! -f "$requirements" ]; then
-        unset requirements
-    fi
-    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
-else
-    repo_root=$(dirname "$tools_dir")
-    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
-    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
-fi
-
-"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \
-                                   "$test_constraints" > "$all_constraints"
-
-set -x
-
-# install the requested packages using the pinned requirements as constraints
-if [ -n "$requirements" ]; then
-    pip install -q --constraint "$all_constraints" --requirement "$requirements"
-fi
-pip install -q --constraint "$all_constraints" "$@"
diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py
new file mode 100755
index 000000000..bdbdcadc5
--- /dev/null
+++ b/tools/pip_install_editable.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# pip installs packages in editable mode using certbot-auto's requirements file
+# as constraints
+
+from __future__ import absolute_import
+
+import sys
+
+import pip_install
+
+def main(args):
+    new_args = []
+    for arg in args:
+        new_args.append('-e')
+        new_args.append(arg)
+    pip_install.main(new_args)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh
deleted file mode 100755
index 6130bf6e7..000000000
--- a/tools/pip_install_editable.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh -e
-# pip installs packages in editable mode using certbot-auto's requirements file
-# as constraints
-
-args=""
-for requirement in "$@" ; do
-  args="$args -e $requirement"
-done
-
-"$(dirname $0)/pip_install.sh" $args
diff --git a/tools/readlink.py b/tools/readlink.py
index 02c74c48d..0199ce184 100755
--- a/tools/readlink.py
+++ b/tools/readlink.py
@@ -7,7 +7,12 @@ platforms.
 
 """
 from __future__ import print_function
+
 import os
 import sys
 
-print(os.path.realpath(sys.argv[1]))
+def main(link):
+    return os.path.realpath(link)
+
+if __name__ == '__main__':
+    print(main(sys.argv[1]))
diff --git a/tools/venv.py b/tools/venv.py
new file mode 100755
index 000000000..849c49119
--- /dev/null
+++ b/tools/venv.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        command_python2_st_code = subprocess.call(
+            'command -v python2', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python2_st_code:
+            return '--python python2'
+
+        command_python27_st_code = subprocess.call(
+            'command -v python2.7', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python27_st_code:
+            return '--python python2.7'
+
+    raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    if os.name == 'nt':
+        raise ValueError('Certbot for Windows is not supported on Python 2.x.')
+
+    venv_args = get_venv_args()
+
+    _venv_common.main('venv', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/venv.sh b/tools/venv.sh
deleted file mode 100755
index 5692f9ebf..000000000
--- a/tools/venv.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh -xe
-# Developer virtualenv setup for Certbot client
-
-if command -v python2; then
-    export VENV_ARGS="--python python2"
-elif command -v python2.7; then
-    export VENV_ARGS="--python python2.7"
-else
-    echo "Couldn't find python2 or python2.7 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test
diff --git a/tools/venv3.py b/tools/venv3.py
new file mode 100755
index 000000000..15db9495a
--- /dev/null
+++ b/tools/venv3.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        where_python3_st_code = subprocess.call(
+            'where python3', shell=True, stdout=fnull, stderr=fnull)
+        command_python3_st_code = subprocess.call(
+            'command -v python3', shell=True, stdout=fnull, stderr=fnull)
+
+    if not where_python3_st_code or not command_python3_st_code:
+        return '--python python3'
+
+    raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    venv_args = get_venv_args()
+
+    _venv_common.main('venv3', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/venv3.sh b/tools/venv3.sh
deleted file mode 100755
index 07512f370..000000000
--- a/tools/venv3.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh -xe
-# Developer Python3 virtualenv setup for Certbot
-
-if command -v python3; then
-    export VENV_NAME="${VENV_NAME:-venv3}"
-    export VENV_ARGS="--python python3"
-else
-    echo "Couldn't find python3 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test
diff --git a/tox-win.ini b/tox-win.ini
deleted file mode 100644
index fe063c264..000000000
--- a/tox-win.ini
+++ /dev/null
@@ -1,13 +0,0 @@
-[tox]
-skipsdist = True
-envlist = py{34,35,36,37}-cover
-
-[testenv]
-deps = -e acme[dev]
-       -e .[dev]
-commands = pytest -n auto --pyargs acme
-           pytest -n auto --pyargs certbot
-
-[testenv:cover]
-commands = pytest -n auto --cov acme --pyargs acme
-           pytest -n auto --cov certbot --cov-append --pyargs certbot
diff --git a/tox.cover.py b/tox.cover.py
new file mode 100755
index 000000000..8bbce2d09
--- /dev/null
+++ b/tox.cover.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+import argparse
+import subprocess
+import os
+import sys
+
+DEFAULT_PACKAGES = [
+    'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns',
+    'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy',
+    'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns',
+    'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53',
+    'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot']
+
+COVER_THRESHOLDS = {
+    'certbot': 98,
+    'acme': 100,
+    'certbot_apache': 100,
+    'certbot_dns_cloudflare': 98,
+    'certbot_dns_cloudxns': 99,
+    'certbot_dns_digitalocean': 98,
+    'certbot_dns_dnsimple': 98,
+    'certbot_dns_dnsmadeeasy': 99,
+    'certbot_dns_gehirn': 97,
+    'certbot_dns_google': 99,
+    'certbot_dns_linode': 98,
+    'certbot_dns_luadns': 98,
+    'certbot_dns_nsone': 99,
+    'certbot_dns_ovh': 97,
+    'certbot_dns_rfc2136': 99,
+    'certbot_dns_route53': 92,
+    'certbot_dns_sakuracloud': 97,
+    'certbot_nginx': 97,
+    'certbot_postfix': 100,
+    'letshelp_certbot': 100
+}
+
+SKIP_PROJECTS_ON_WINDOWS = [
+    'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot']
+
+def cover(package):
+    threshold = COVER_THRESHOLDS.get(package)
+
+    if not threshold:
+        raise ValueError('Unrecognized package: {0}'.format(package))
+
+    pkg_dir = package.replace('_', '-')
+
+    if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS:
+        print((
+            'Info: currently {0} is not supported on Windows and will not be tested/covered.'
+            .format(pkg_dir)))
+        return
+
+    subprocess.call([
+        sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=',
+        '--numprocesses', 'auto', '--pyargs', package])
+    subprocess.call([
+        sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include',
+        '{0}/*'.format(pkg_dir), '--show-missing'])
+
+def main():
+    description = """
+This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order
+to generate separate stats for each package. It should be removed once those
+packages are moved to a separate repo.
+
+Option -e makes sure we fail fast and don't submit to codecov."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--packages', nargs='+')
+
+    args = parser.parse_args()
+
+    packages = args.packages or DEFAULT_PACKAGES
+
+    # --cov-append is on, make sure stats are correct
+    try:
+        os.remove('.coverage')
+    except OSError:
+        pass
+
+    for package in packages:
+        cover(package)
+
+if __name__ == '__main__':
+    main()
diff --git a/tox.cover.sh b/tox.cover.sh
deleted file mode 100755
index c68e757de..000000000
--- a/tox.cover.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/sh -xe
-
-# USAGE: ./tox.cover.sh [package]
-#
-# This script is used by tox.ini (and thus Travis CI) in order to
-# generate separate stats for each package. It should be removed once
-# those packages are moved to separate repo.
-#
-# -e makes sure we fail fast and don't submit to codecov
-
-if [ "xxx$1" = "xxx" ]; then
-  pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot"
-else
-  pkgs="$@"
-fi
-
-cover () {
-  if [ "$1" = "certbot" ]; then
-    min=98
-  elif [ "$1" = "acme" ]; then
-    min=100
-  elif [ "$1" = "certbot_apache" ]; then
-    min=100
-  elif [ "$1" = "certbot_dns_cloudflare" ]; then
-    min=98
-  elif [ "$1" = "certbot_dns_cloudxns" ]; then
-    min=99
-  elif [ "$1" = "certbot_dns_digitalocean" ]; then
-    min=98
-  elif [ "$1" = "certbot_dns_dnsimple" ]; then
-    min=98
-  elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then
-    min=99
-  elif [ "$1" = "certbot_dns_gehirn" ]; then
-    min=97
-  elif [ "$1" = "certbot_dns_google" ]; then
-    min=99
-  elif [ "$1" = "certbot_dns_linode" ]; then
-    min=98
-  elif [ "$1" = "certbot_dns_luadns" ]; then
-    min=98
-  elif [ "$1" = "certbot_dns_nsone" ]; then
-    min=99
-  elif [ "$1" = "certbot_dns_ovh" ]; then
-    min=97
-  elif [ "$1" = "certbot_dns_rfc2136" ]; then
-    min=99
-  elif [ "$1" = "certbot_dns_route53" ]; then
-    min=92
-  elif [ "$1" = "certbot_dns_sakuracloud" ]; then
-    min=97
-  elif [ "$1" = "certbot_nginx" ]; then
-    min=97
-  elif [ "$1" = "certbot_postfix" ]; then
-    min=100
-  elif [ "$1" = "letshelp_certbot" ]; then
-    min=100
-  else
-    echo "Unrecognized package: $1"
-    exit 1
-  fi
-
-  pkg_dir=$(echo "$1" | tr _ -)
-  pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1"
-  coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing
-}
-
-rm -f .coverage  # --cov-append is on, make sure stats are correct
-for pkg in $pkgs
-do
-  cover $pkg
-done
diff --git a/tox.ini b/tox.ini
index 9db06f78c..e38f1311e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,16 +4,16 @@
 
 [tox]
 skipsdist = true
-envlist = modification,py{34,35,36},cover,lint
+envlist = modification,py{34,35,36},py27-cover,lint
 
 [base]
 # pip installs the requested packages in editable mode
-pip_install = {toxinidir}/tools/pip_install_editable.sh
+pip_install = python {toxinidir}/tools/pip_install_editable.py
 # pip installs the requested packages in editable mode and runs unit tests on
 # them. Each package is installed and tested in the order they are provided
 # before the script moves on to the next package. All dependencies are pinned
 # to a specific version for increased stability for developers.
-install_and_test = {toxinidir}/tools/install_and_test.sh
+install_and_test = python {toxinidir}/tools/install_and_test.py
 dns_packages =
     certbot-dns-cloudflare \
     certbot-dns-cloudxns \
@@ -38,7 +38,7 @@ all_packages =
     certbot-postfix \
     letshelp-certbot
 install_packages =
-    {toxinidir}/tools/pip_install_editable.sh {[base]all_packages}
+    python {toxinidir}/tools/pip_install_editable.py {[base]all_packages}
 source_paths =
     acme/acme
     certbot
@@ -64,7 +64,9 @@ source_paths =
     tests/lock_test.py
 
 [testenv]
-passenv = TRAVIS
+passenv = 
+    TRAVIS
+    APPVEYOR
 commands =
     {[base]install_and_test} {[base]all_packages}
     python tests/lock_test.py
@@ -120,11 +122,17 @@ basepython = python2.7
 commands =
     {[base]install_packages}
 
-[testenv:cover]
+[testenv:py27-cover]
 basepython = python2.7
 commands =
     {[base]install_packages}
-    ./tox.cover.sh
+    python tox.cover.py
+
+[testenv:py37-cover]
+basepython = python3.7
+commands =
+    {[base]install_packages}
+    python tox.cover.py
 
 [testenv:lint]
 basepython = python2.7
@@ -133,7 +141,7 @@ basepython = python2.7
 # continue, but tox return code will reflect previous error
 commands =
     {[base]install_packages}
-    pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
+    python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
 
 [testenv:mypy]
 basepython = python3
@@ -157,7 +165,7 @@ commands =
 # allow users to run the modification check by running `tox`
 [testenv:modification]
 commands =
-    {toxinidir}/tests/modification-check.sh
+    python {toxinidir}/tests/modification-check.py
 
 [testenv:apache_compat]
 commands =
@@ -197,7 +205,7 @@ passenv =
 # At the moment, this tests under Python 2.7 only, as only that version is
 # readily available on the Trusty Docker image.
 commands =
-    {toxinidir}/tests/modification-check.sh
+    python {toxinidir}/tests/modification-check.py
     docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source
     docker run --rm -t -i lea
 whitelist_externals =