Use tools/pipstrap.py

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -79,13 +79,9 @@ jobs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
-      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
-      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
-      # a recent version of pip, but we also to disable the isolated feature as described in
-      # https://github.com/certbot/certbot/issues/8256
       - script: |
           py -3 -m venv venv
-          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
+          venv\Scripts\python tools\pipstrap.py
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         env:
           PIP_NO_BUILD_ISOLATION: no
@@ -155,7 +151,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
           python3 -m venv venv
-          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -U tox
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
@@ -193,7 +189,7 @@ jobs:
       - script: |
           set -e
           python3 -m venv venv
-          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |

.azure-pipelines/templates/steps/tox-steps.yml

@@ -32,7 +32,7 @@ steps:
     # problems with its lack of real dependency resolution.
   - bash: |
       set -e
-      python letsencrypt-auto-source/pieces/pipstrap.py
+      python tools/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1

tools/_venv_common.py

@@ -200,7 +200,7 @@ def install_packages(venv_name, pip_args):
     """
     # Using the python executable from venv, we ensure to execute following commands in this venv.
     py_venv = get_venv_python_path(venv_name)
-    subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')])
+    subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')])
     # We only use this value during pip install because:
     # 1) We're really only adding it for installing cryptography, which happens here, and
     # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the

tools/docker/core/Dockerfile

@@ -14,16 +14,11 @@ WORKDIR /opt/certbot
 
 # Copy certbot code
 COPY CHANGELOG.md README.rst src/
-COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
-COPY letsencrypt-auto-source/pieces/pipstrap.py .
+COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/
 COPY tools tools
 COPY acme src/acme
 COPY certbot src/certbot
 
-# Generate constraints file to pin dependency versions
-RUN cat dependency-requirements.txt | tools/strip_hashes.py > unhashed_requirements.txt \
- && cat tools/dev_constraints.txt unhashed_requirements.txt | tools/merge_requirements.py > docker_constraints.txt
-
 # Install certbot runtime dependencies
 RUN apk add --no-cache --virtual .certbot-deps \
         libffi \
@@ -33,15 +28,20 @@ RUN apk add --no-cache --virtual .certbot-deps \
         binutils
 
 # Install certbot from sources
+#
+# We don't use tools/pip_install.py below so the hashes in
+# dependency-requirements.txt can be used when installing packages for extra
+# security.
 RUN apk add --no-cache --virtual .build-deps \
         gcc \
         linux-headers \
         openssl-dev \
         musl-dev \
         libffi-dev \
-    && python pipstrap.py \
-    && pip install -r dependency-requirements.txt \
-    && pip install --no-cache-dir --no-deps \
+    && python tools/pipstrap.py \
+    && pip install --no-build-isolation \
+        -r letsencrypt-auto-source/pieces/dependency-requirements.txt \
+    && pip install --no-build-isolation --no-cache-dir --no-deps \
         --editable src/acme \
         --editable src/certbot \
 && apk del .build-deps

tools/docker/plugin/Dockerfile

@@ -11,4 +11,4 @@ COPY qemu-${QEMU_ARCH}-static /usr/bin/
 COPY . /opt/certbot/src/plugin
 
 # Install the DNS plugin
-RUN pip install --constraint /opt/certbot/docker_constraints.txt --no-cache-dir --editable /opt/certbot/src/plugin
+RUN tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin

tools/pip_install.py

@@ -75,13 +75,15 @@ def call_with_print(command):
     subprocess.check_call(command, shell=True)
 
 
-def pip_install_with_print(args_str):
-    command = '"{0}" -m pip install --disable-pip-version-check {1}'.format(sys.executable,
-                                                                            args_str)
-    call_with_print(command)
+def pip_install_with_print(args_str, disable_build_isolation):
+    command = ['"', sys.executable, '" -m pip install --disable-pip-version-check ']
+    if disable_build_isolation:
+        command.append('--no-build-isolation ')
+    command.append(args_str)
+    call_with_print(''.join(command))
 
 
-def main(args):
+def main(args, disable_build_isolation=True):
     tools_path = find_tools_path()
     working_dir = tempfile.mkdtemp()
 
@@ -96,7 +98,7 @@ def main(args):
 
         if os.environ.get('CERTBOT_NO_PIN') == '1':
             # With unpinned dependencies, there is no constraint
-            pip_install_with_print(' '.join(args))
+            pip_install_with_print(' '.join(args), disable_build_isolation)
         else:
             # Otherwise, we merge requirements to build the constraints and pin dependencies
             requirements = None
@@ -110,15 +112,17 @@ def main(args):
                 # First step, install the transitive dependencies of oldest requirements
                 # in respect with oldest constraints.
                 pip_install_with_print('--constraint "{0}" --requirement "{1}"'
-                                       .format(all_constraints, requirements))
+                                       .format(all_constraints, requirements),
+                                       disable_build_isolation)
                 # Second step, ensure that oldest requirements themselves are effectively
                 # installed using --force-reinstall, and avoid corner cases like the one described
                 # in https://github.com/certbot/certbot/issues/7014.
                 pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"'
-                                       .format(requirements))
+                                       .format(requirements),
+                                       disable_build_isolation)
 
             pip_install_with_print('--constraint "{0}" {1}'.format(
-                all_constraints, ' '.join(args)))
+                all_constraints, ' '.join(args)), disable_build_isolation)
     finally:
         if os.environ.get('TRAVIS'):
             print('travis_fold:end:install_certbot_deps')

tools/pipstrap.py

@@ -12,7 +12,11 @@ import pip_install
 
 def main():
     pkgs = 'pip setuptools wheel'.split()
-    pip_install.main(pkgs)
+    # We don't disable build isolation because we may have an older version of
+    # pip that doesn't support the flag disabling it. We expect these packages
+    # to already have usable wheels available anyway so no building should be
+    # required.
+    pip_install.main(pkgs, disable_build_isolation=False)
 
 
 if __name__ == '__main__':

tools/run_oldest_tests.sh

@@ -16,21 +16,22 @@ DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX)
 
 cat << "EOF" >> "${DOCKERFILE}"
 FROM ubuntu:16.04
-COPY pipstrap.py /tmp/pipstrap.py
+COPY letsencrypt-auto-source/pieces/dependency-requirements.txt /tmp/letsencrypt-auto-source/pieces/
+COPY tools/ /tmp/tools/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
         python-dev python-pip python-setuptools \
         gcc libaugeas0 libssl-dev libffi-dev \
         git ca-certificates nginx-light openssl curl \
  && curl -fsSL https://get.docker.com | bash /dev/stdin \
- && python /tmp/pipstrap.py \
- && python -m pip install tox \
+ && python /tmp/tools/pipstrap.py \
+ && python /tmp/tools/pip_install.py tox \
  && rm -rf /var/lib/apt/lists/*
 EOF
 
-docker build -f "${DOCKERFILE}" -t oldest-worker ./letsencrypt-auto-source/pieces
-docker run --rm --network=host -w "${PWD}" \
-  -v /var/run/docker.sock:/var/run/docker.sock \
-  -v "${PWD}:${PWD}" -v /tmp:/tmp \
-  -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \
-  oldest-worker python -m tox
+docker build -f "${DOCKERFILE}" -t oldest-worker .
+#docker run --rm --network=host -w "${PWD}" \
+#  -v /var/run/docker.sock:/var/run/docker.sock \
+#  -v "${PWD}:${PWD}" -v /tmp:/tmp \
+#  -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \
+#  oldest-worker python -m tox

tox.ini

@@ -62,7 +62,7 @@ source_paths =
 [testenv]
 passenv =
     CERTBOT_NO_PIN
-commands_pre = python {toxinidir}/letsencrypt-auto-source/pieces/pipstrap.py
+commands_pre = python {toxinidir}/tools/pipstrap.py
 commands =
     !cover: {[base]install_and_test} {[base]all_packages}
     !cover: python tests/lock_test.py

windows-installer/construct.py

@@ -54,7 +54,7 @@ def _compile_wheels(repo_path, build_path, venv_python):
 def _prepare_build_tools(venv_path, venv_python, repo_path):
     print('Prepare build tools')
     subprocess.check_call([sys.executable, '-m', 'venv', venv_path])
-    subprocess.check_call([venv_python, os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'pipstrap.py')])
+    subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')])
     subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist'])
     subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION])