From aeb09095c3883bfa2ef0c8c6afa6c509f2836daf Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Oct 2020 13:57:41 -0700 Subject: [PATCH] Use tools/pipstrap.py --- .../templates/jobs/packaging-jobs.yml | 10 +++------ .../templates/steps/tox-steps.yml | 2 +- tools/_venv_common.py | 2 +- tools/docker/core/Dockerfile | 18 +++++++-------- tools/docker/plugin/Dockerfile | 2 +- tools/pip_install.py | 22 +++++++++++-------- tools/pipstrap.py | 6 ++++- tools/run_oldest_tests.sh | 19 ++++++++-------- tox.ini | 2 +- windows-installer/construct.py | 2 +- 10 files changed, 45 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index 8da30b1f5..eca5faa2f 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -79,13 +79,9 @@ jobs: artifact: windows-installer path: $(Build.SourcesDirectory)/bin displayName: Retrieve Windows installer - # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency - # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install - # a recent version of pip, but we also to disable the isolated feature as described in - # https://github.com/certbot/certbot/issues/8256 - script: | py -3 -m venv venv - venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1 + venv\Scripts\python tools\pipstrap.py venv\Scripts\python tools\pip_install.py -e certbot-ci env: PIP_NO_BUILD_ISOLATION: no @@ -155,7 +151,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends nginx-light snapd python3 -m venv venv - venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py + venv/bin/python tools/pipstrap.py venv/bin/python tools/pip_install.py -U tox displayName: Install dependencies - task: DownloadPipelineArtifact@2 @@ -193,7 +189,7 @@ jobs: - script: | set -e python3 -m venv venv - venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py + venv/bin/python tools/pipstrap.py venv/bin/python tools/pip_install.py -e certbot-ci displayName: Prepare Certbot-CI - script: | diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml index 14b27b08f..a9f78d36b 100644 --- a/.azure-pipelines/templates/steps/tox-steps.yml +++ b/.azure-pipelines/templates/steps/tox-steps.yml @@ -32,7 +32,7 @@ steps: # problems with its lack of real dependency resolution. - bash: | set -e - python letsencrypt-auto-source/pieces/pipstrap.py + python tools/pipstrap.py python tools/pip_install.py -I tox virtualenv displayName: Install runtime dependencies - task: DownloadSecureFile@1 diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 2b3014cce..58c05ed09 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -200,7 +200,7 @@ def install_packages(venv_name, pip_args): """ # Using the python executable from venv, we ensure to execute following commands in this venv. py_venv = get_venv_python_path(venv_name) - subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + subprocess_with_print([py_venv, os.path.abspath('tools/pipstrap.py')]) # We only use this value during pip install because: # 1) We're really only adding it for installing cryptography, which happens here, and # 2) There are issues with calling it along with VIRTUALENV_NO_DOWNLOAD, which applies at the diff --git a/tools/docker/core/Dockerfile b/tools/docker/core/Dockerfile index ff8c6386c..91b90a4a2 100644 --- a/tools/docker/core/Dockerfile +++ b/tools/docker/core/Dockerfile @@ -14,16 +14,11 @@ WORKDIR /opt/certbot # Copy certbot code COPY CHANGELOG.md README.rst src/ -COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . -COPY letsencrypt-auto-source/pieces/pipstrap.py . +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt letsencrypt-auto-source/pieces/ COPY tools tools COPY acme src/acme COPY certbot src/certbot -# Generate constraints file to pin dependency versions -RUN cat dependency-requirements.txt | tools/strip_hashes.py > unhashed_requirements.txt \ - && cat tools/dev_constraints.txt unhashed_requirements.txt | tools/merge_requirements.py > docker_constraints.txt - # Install certbot runtime dependencies RUN apk add --no-cache --virtual .certbot-deps \ libffi \ @@ -33,15 +28,20 @@ RUN apk add --no-cache --virtual .certbot-deps \ binutils # Install certbot from sources +# +# We don't use tools/pip_install.py below so the hashes in +# dependency-requirements.txt can be used when installing packages for extra +# security. RUN apk add --no-cache --virtual .build-deps \ gcc \ linux-headers \ openssl-dev \ musl-dev \ libffi-dev \ - && python pipstrap.py \ - && pip install -r dependency-requirements.txt \ - && pip install --no-cache-dir --no-deps \ + && python tools/pipstrap.py \ + && pip install --no-build-isolation \ + -r letsencrypt-auto-source/pieces/dependency-requirements.txt \ + && pip install --no-build-isolation --no-cache-dir --no-deps \ --editable src/acme \ --editable src/certbot \ && apk del .build-deps diff --git a/tools/docker/plugin/Dockerfile b/tools/docker/plugin/Dockerfile index 6bbbae7c1..5a6673e5b 100644 --- a/tools/docker/plugin/Dockerfile +++ b/tools/docker/plugin/Dockerfile @@ -11,4 +11,4 @@ COPY qemu-${QEMU_ARCH}-static /usr/bin/ COPY . /opt/certbot/src/plugin # Install the DNS plugin -RUN pip install --constraint /opt/certbot/docker_constraints.txt --no-cache-dir --editable /opt/certbot/src/plugin +RUN tools/pip_install.py --no-cache-dir --editable /opt/certbot/src/plugin diff --git a/tools/pip_install.py b/tools/pip_install.py index 0a3961384..80ecc0e59 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -75,13 +75,15 @@ def call_with_print(command): subprocess.check_call(command, shell=True) -def pip_install_with_print(args_str): - command = '"{0}" -m pip install --disable-pip-version-check {1}'.format(sys.executable, - args_str) - call_with_print(command) +def pip_install_with_print(args_str, disable_build_isolation): + command = ['"', sys.executable, '" -m pip install --disable-pip-version-check '] + if disable_build_isolation: + command.append('--no-build-isolation ') + command.append(args_str) + call_with_print(''.join(command)) -def main(args): +def main(args, disable_build_isolation=True): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() @@ -96,7 +98,7 @@ def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint - pip_install_with_print(' '.join(args)) + pip_install_with_print(' '.join(args), disable_build_isolation) else: # Otherwise, we merge requirements to build the constraints and pin dependencies requirements = None @@ -110,15 +112,17 @@ def main(args): # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. pip_install_with_print('--constraint "{0}" --requirement "{1}"' - .format(all_constraints, requirements)) + .format(all_constraints, requirements), + disable_build_isolation) # Second step, ensure that oldest requirements themselves are effectively # installed using --force-reinstall, and avoid corner cases like the one described # in https://github.com/certbot/certbot/issues/7014. pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' - .format(requirements)) + .format(requirements), + disable_build_isolation) pip_install_with_print('--constraint "{0}" {1}'.format( - all_constraints, ' '.join(args))) + all_constraints, ' '.join(args)), disable_build_isolation) finally: if os.environ.get('TRAVIS'): print('travis_fold:end:install_certbot_deps') diff --git a/tools/pipstrap.py b/tools/pipstrap.py index 2567ae4e7..2e6b89049 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -12,7 +12,11 @@ import pip_install def main(): pkgs = 'pip setuptools wheel'.split() - pip_install.main(pkgs) + # We don't disable build isolation because we may have an older version of + # pip that doesn't support the flag disabling it. We expect these packages + # to already have usable wheels available anyway so no building should be + # required. + pip_install.main(pkgs, disable_build_isolation=False) if __name__ == '__main__': diff --git a/tools/run_oldest_tests.sh b/tools/run_oldest_tests.sh index 2d007888d..16b784ddd 100755 --- a/tools/run_oldest_tests.sh +++ b/tools/run_oldest_tests.sh @@ -16,21 +16,22 @@ DOCKERFILE=$(mktemp /tmp/Dockerfile.XXXXXX) cat << "EOF" >> "${DOCKERFILE}" FROM ubuntu:16.04 -COPY pipstrap.py /tmp/pipstrap.py +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt /tmp/letsencrypt-auto-source/pieces/ +COPY tools/ /tmp/tools/ RUN apt-get update \ && apt-get install -y --no-install-recommends \ python-dev python-pip python-setuptools \ gcc libaugeas0 libssl-dev libffi-dev \ git ca-certificates nginx-light openssl curl \ && curl -fsSL https://get.docker.com | bash /dev/stdin \ - && python /tmp/pipstrap.py \ - && python -m pip install tox \ + && python /tmp/tools/pipstrap.py \ + && python /tmp/tools/pip_install.py tox \ && rm -rf /var/lib/apt/lists/* EOF -docker build -f "${DOCKERFILE}" -t oldest-worker ./letsencrypt-auto-source/pieces -docker run --rm --network=host -w "${PWD}" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v "${PWD}:${PWD}" -v /tmp:/tmp \ - -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \ - oldest-worker python -m tox +docker build -f "${DOCKERFILE}" -t oldest-worker . +#docker run --rm --network=host -w "${PWD}" \ +# -v /var/run/docker.sock:/var/run/docker.sock \ +# -v "${PWD}:${PWD}" -v /tmp:/tmp \ +# -e TOXENV -e ACME_SERVER -e PYTEST_ADDOPTS \ +# oldest-worker python -m tox diff --git a/tox.ini b/tox.ini index 9412a2349..5dcc55d3f 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,7 @@ source_paths = [testenv] passenv = CERTBOT_NO_PIN -commands_pre = python {toxinidir}/letsencrypt-auto-source/pieces/pipstrap.py +commands_pre = python {toxinidir}/tools/pipstrap.py commands = !cover: {[base]install_and_test} {[base]all_packages} !cover: python tests/lock_test.py diff --git a/windows-installer/construct.py b/windows-installer/construct.py index b5be69fd2..14f770959 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -54,7 +54,7 @@ def _compile_wheels(repo_path, build_path, venv_python): def _prepare_build_tools(venv_path, venv_python, repo_path): print('Prepare build tools') subprocess.check_call([sys.executable, '-m', 'venv', venv_path]) - subprocess.check_call([venv_python, os.path.join(repo_path, 'letsencrypt-auto-source', 'pieces', 'pipstrap.py')]) + subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pipstrap.py')]) subprocess.check_call([venv_python, os.path.join(repo_path, 'tools', 'pip_install.py'), 'pynsist']) subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION])