From 451e2202de82639f46fb8c2e5e8eee6fd03fe52a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 22 Jun 2021 16:16:33 -0400 Subject: [PATCH] refactor pinning script --- .../common/export-pinned-dependencies.sh | 38 +++++++++++++++++++ tools/pinning/{pin.sh => normal/repin.sh} | 32 +++------------- tools/requirements.txt | 15 ++++---- 3 files changed, 51 insertions(+), 34 deletions(-) create mode 100755 tools/pinning/common/export-pinned-dependencies.sh rename tools/pinning/{pin.sh => normal/repin.sh} (52%) diff --git a/tools/pinning/common/export-pinned-dependencies.sh b/tools/pinning/common/export-pinned-dependencies.sh new file mode 100755 index 000000000..70477077f --- /dev/null +++ b/tools/pinning/common/export-pinned-dependencies.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# This script accepts a directory containing a pyproject.toml file configured +# for use with poetry and generates and prints the pinned dependencies of that +# file. Any dependencies on acme or those referencing certbot will be removed +# from the output. The exported requirements are printed to stdout. +set -euo pipefail + +if [ -z ${1+x} ]; then + echo "Usage:" >&2 + echo "$0 PYPROJECT_TOML_DIRECTORY" >&2 + exit 1 +fi + +REPO_ROOT="$(git rev-parse --show-toplevel)" +WORK_DIR="$1" + +if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then + echo "Please install poetry 1.2+." >&2 + echo "You may need to recreate Certbot's virtual environment and activate it." >&2 + exit 1 +fi + +# Old eggs can cause outdated dependency information to be used by poetry so we +# delete them before generating the lock file. See +# https://github.com/python-poetry/poetry/issues/4103 for more info. +rm -rf ${REPO_ROOT}/*.egg-info + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock >&2 +trap 'rm poetry.lock' EXIT + +# We need to remove local packages from the output. +poetry export --without-hashes | sed '/^acme @/d; /certbot/d;' diff --git a/tools/pinning/pin.sh b/tools/pinning/normal/repin.sh similarity index 52% rename from tools/pinning/pin.sh rename to tools/pinning/normal/repin.sh index 1c8111e1b..3984c5a3c 100755 --- a/tools/pinning/pin.sh +++ b/tools/pinning/normal/repin.sh @@ -5,36 +5,16 @@ set -euo pipefail WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" +COMMON_DIR="$(dirname "${WORK_DIR}")/common" +REPO_ROOT="$(git rev-parse --show-toplevel)" RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" -if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then - echo "Please install poetry 1.2+." - echo "You may need to recreate Certbot's virtual environment and activate it." - exit 1 -fi - -# Old eggs can cause outdated dependency information to be used by poetry so we -# delete them before generating the lock file. See -# https://github.com/python-poetry/poetry/issues/4103 for more info. -cd "${REPO_ROOT}" -rm -rf */*.egg-info - -cd "${WORK_DIR}" - -if [ -f poetry.lock ]; then - rm poetry.lock -fi - -poetry lock - +# We first write requirements to a temporary file to avoid modifying the +# requirements file if the repinning process fails. TEMP_REQUIREMENTS=$(mktemp) -trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT - -poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes -# We need to remove local packages from the requirements file. -sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" +trap 'rm $TEMP_REQUIREMENTS' EXIT +"${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}" > "${TEMP_REQUIREMENTS}" cat << EOF > "$REQUIREMENTS_FILE" # This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using diff --git a/tools/requirements.txt b/tools/requirements.txt index 7190c9c81..38ca2aaa1 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,4 +1,4 @@ -# This file was generated by tools/pinning/pin.sh and can be updated using +# This file was generated by tools/pinning/normal/repin.sh and can be updated using # that script. # # It is normally used as constraints to pip, however, it has the name @@ -7,21 +7,20 @@ # for more info. alabaster==0.7.12; python_version >= "3.6" apacheconfig==0.3.2; python_version >= "3.6" -apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" appnope==0.1.2; python_version == "3.6" and sys_platform == "darwin" or python_version >= "3.7" and sys_platform == "darwin" astroid==2.5.6; python_version >= "3.6" and python_version < "4.0" atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -awscli==1.19.91; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +awscli==1.19.95; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") azure-devops==6.0.0b4; python_version >= "3.6" babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" backcall==0.2.0; python_version == "3.6" or python_version >= "3.7" bcrypt==3.2.0; python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.91; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.91; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +boto3==1.17.95; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.95; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -51,11 +50,11 @@ dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or p docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" docutils==0.15.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.3.0" entrypoints==0.3; python_version >= "3.6" and python_version < "4.0" -execnet==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" fabric==2.6.0; python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0" google-api-core==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.8.0; python_version >= "3.6" +google-api-python-client==2.9.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" google-auth==1.31.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" @@ -167,7 +166,7 @@ tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0" or python_version > "3.6" and python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" tomlkit==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -tqdm==4.61.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +tqdm==4.61.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" typed-ast==1.4.3; python_version >= "3.6" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"