diff --git a/Makefile b/Makefile index bb20435..ffae6ed 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ endif -include ./conf/make.documentation.Makefile -include ./conf/make.image_db.Makefile -include ./conf/make.installation.Makefile +-include ./conf/make.installation.poetry.Makefile -include ./conf/make.test.Makefile -include ./conf/make.todo.Makefile @@ -177,13 +178,6 @@ format_script_isort_only: log_show_test: vim ${LOG_FILE} -########## -# poetry # -########## -.PHONY: poetry_update -poetry_update: - ./script/poetry/poetry_update.py - ########### # clean # ########### diff --git a/conf/make.installation.poetry.Makefile b/conf/make.installation.poetry.Makefile new file mode 100644 index 0000000..6421b60 --- /dev/null +++ b/conf/make.installation.poetry.Makefile @@ -0,0 +1,22 @@ +########### +# INSTALL # +########### + +.PHONY: install_venv_with_minimal_pip +install_venv_with_minimal_pip: + # This install .venv.odoo.version with poetry without complete pip installation + # This is interested to create new pip installation + WITH_POETRY_INSTALLATION=0 ./script/install/install_locally.sh + +########## +# poetry # +########## +.PHONY: poetry_update +poetry_update: + ./script/poetry/poetry_update.py + +.PHONY: poetry_update_force +poetry_update_force: + # Use this to recreate pip dependency from new installation, this will ignore freeze versions + ./script/poetry/poetry_update.py -f + diff --git a/script/install/install_locally.sh b/script/install/install_locally.sh index 7b68242..cfbee34 100755 --- a/script/install/install_locally.sh +++ b/script/install/install_locally.sh @@ -38,6 +38,7 @@ VENV_REPO_PATH=${VENV_PATH}/repo VENV_MULTILINGUAL_MARKDOWN_PATH=${VENV_PATH}/multilang_md.py #POETRY_PATH=~/.local/bin/poetry POETRY_PATH=${VENV_PATH}/bin/poetry +export WITH_POETRY_INSTALLATION=1 if [[ ! -n "${DOCKER_BUILD}" ]]; then echo "Python path version home" @@ -129,7 +130,9 @@ if [[ ! -f "${POETRY_PATH}" ]]; then # ${VENV_PATH}/bin/poetry lock --no-update # To fix keyring problem when installation is blocked, use export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring - ${VENV_PATH}/bin/poetry install --no-root -vvv + if [[ ${WITH_POETRY_INSTALLATION} -ne 0 ]]; then + ${VENV_PATH}/bin/poetry install --no-root -vvv + fi retVal=$? if [[ $retVal -ne 0 ]]; then echo "Poetry installation error with status ${retVal}" diff --git a/script/poetry/iscompatible.py b/script/poetry/iscompatible.py index 3eb5b56..d35253c 100644 --- a/script/poetry/iscompatible.py +++ b/script/poetry/iscompatible.py @@ -2,6 +2,9 @@ Python versioning with requirements.txt syntax ============================================== +# Copied from project iscompatible with support packaging Version +# from packaging.version import Version + :mod:`iscompatible` gives you the power of the pip requirements.txt syntax for everyday python packages, modules, classes or arbitrary functions. @@ -47,8 +50,10 @@ version_info = (0, 1, 1) __version__ = "%s.%s.%s" % version_info -import re import operator +import re + +from packaging.version import Version def iscompatible(requirements, version): @@ -76,10 +81,12 @@ def iscompatible(requirements, version): results = list() - for operator_string, requirement_string in parse_requirements(requirements): + for operator_string, requirement_string in parse_requirements( + requirements + ): operator = operators[operator_string] required = string_to_tuple(requirement_string) - result = operator(version, required) + result = operator(version.release, required) results.append(result) @@ -117,9 +124,7 @@ def parse_requirements(line): while not LINE_END.match(line, p): match = VERSION.match(line, p) if not match: - raise ValueError( - "Expected version spec in", - line, "at", line[p:]) + raise ValueError("Expected version spec in", line, "at", line[p:]) specs.append(match.group(*(1, 2))) p = match.end() @@ -129,8 +134,8 @@ def parse_requirements(line): p = match.end() # Skip comma elif not LINE_END.match(line, p): raise ValueError( - "Expected ',' or end-of-list in", - line, "at", line[p:]) + "Expected ',' or end-of-list in", line, "at", line[p:] + ) return specs @@ -149,9 +154,11 @@ def string_to_tuple(version): return tuple(map(int, version.split("."))) -operators = {"<": operator.lt, - "<=": operator.le, - "==": operator.eq, - "!=": operator.ne, - ">=": operator.ge, - ">": operator.gt} +operators = { + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} diff --git a/script/poetry/poetry_update.py b/script/poetry/poetry_update.py index 6f14806..04b0a50 100755 --- a/script/poetry/poetry_update.py +++ b/script/poetry/poetry_update.py @@ -6,14 +6,19 @@ import argparse import ast import logging import os +import re from collections import OrderedDict, defaultdict from pathlib import Path import iscompatible import toml from colorama import Fore, Style +from packaging.markers import default_environment +from packaging.requirements import Requirement +from packaging.version import Version _logger = logging.getLogger(__name__) +_logger.setLevel(logging.INFO) def get_config(): @@ -56,33 +61,82 @@ def get_config(): action="store_true", help="Don't apply change, only show warning.", ) + parser.add_argument( + "--set_version_poetry", + help="Force to change poetry version, default is from file './.poetry-version'.", + ) + parser.add_argument( + "--set_version_odoo", + help="Force to change odoo version, default is from file './.odoo-version'.", + ) + parser.add_argument( + "--set_version_python", + help="Force to change python version, default is from file './.python-version'.", + ) + parser.add_argument( + "--set_version_erplibre", + help="Force to change erplibre version, default is from file './.erplibre-version'.", + ) args = parser.parse_args() + if not args.set_version_poetry: + with open("./.poetry-version", "r", encoding="utf-8") as fichier: + args.set_version_poetry = fichier.read().strip() + if not args.set_version_odoo: + with open("./.odoo-version", "r", encoding="utf-8") as fichier: + args.set_version_odoo = fichier.read().strip() + if not args.set_version_python: + with open("./.python-version", "r", encoding="utf-8") as fichier: + args.set_version_python = fichier.read().strip() + if not args.set_version_erplibre: + with open("./.erplibre-version", "r", encoding="utf-8") as fichier: + args.set_version_erplibre = fichier.read().strip() return args -def get_lst_requirements_txt(): - return get_file_from_glob("requirements.[tT][xX][tT]") +def get_lst_requirements_txt( + config, ignore_dir_startswith: list = None, force_add_item: list = None +): + return get_file_from_glob( + config, + "requirements.[tT][xX][tT]", + ignore_dir_startswith=ignore_dir_startswith, + force_add_item=force_add_item, + ) -def get_lst_manifest_py(): - return get_file_from_glob("__manifest__.py") +def get_lst_manifest_py(config, ignore_dir_startswith: list = None): + return get_file_from_glob( + config, "__manifest__.py", ignore_dir_startswith=ignore_dir_startswith + ) -def get_file_from_glob(glob_txt): - with open(".odoo-version") as txt: - odoo_version = txt.read() - # with open(".erplibre-version") as txt: - # erplibre_version = txt.read() +def get_file_from_glob( + config, + glob_txt, + ignore_dir_startswith: list = None, + force_add_item: list = None, +): lst_v = [] for a in Path(".").rglob(glob_txt): a_dirname = os.path.dirname(a) if a_dirname.startswith(".repo/") or a_dirname.startswith(".venv"): continue + if ignore_dir_startswith: + ignore_it = False + for item_ignore_dir in ignore_dir_startswith: + if a_dirname.startswith(item_ignore_dir): + ignore_it = True + break + if ignore_it: + continue if a_dirname.startswith("addons") and not a_dirname.startswith( - f"addons.odoo{odoo_version}" + f"addons.odoo{config.set_version_odoo}" ): continue lst_v.append(a) + if force_add_item: + for item in force_add_item: + lst_v.append(Path(item)) return lst_v @@ -94,21 +148,31 @@ def combine_requirements(config): :param config: :return: """ - priority_filename_requirement = "requirements.txt" - second_priority_filename_requirement = "odoo/requirements.txt" + priority_filename_requirement = ( + f"requirement/requirements.{config.set_version_erplibre}.txt" + ) + second_priority_filename_requirement = ( + f"odoo{config.set_version_odoo}/odoo/requirements.txt" + ) except_sign = ";" lst_sign = ("==", "!=", ">=", "<=", "<", ">", except_sign, "~=") lst_requirements = [] - lst_replace_special_sign = {} # the lib iscompatible doesn't support ~= + lst_replace_special_sign = {} ignore_requirements = ["sys_platform == 'win32'", "python_version < '3.7'"] dct_requirements = defaultdict(set) - lst_requirements_file = get_lst_requirements_txt() + lst_requirements_file = get_lst_requirements_txt( + config, + ignore_dir_startswith=["script/"], + force_add_item=[priority_filename_requirement], + ) # lst_requirements_with_condition = set() # dct_special_condition = defaultdict(list) if not os.path.isfile(priority_filename_requirement): _logger.error(f"File {priority_filename_requirement} not found.") + print(f"Find {len(lst_requirements_file)} requirements.txt files.") + dct_requirements_module_filename = defaultdict(list) for requirements_filename in lst_requirements_file: with open(requirements_filename, "r") as f: @@ -124,11 +188,29 @@ def combine_requirements(config): b = b[: b.index("#")].strip() comment_depend = "" if except_sign in b: - # TODO support python_version into comment_depend, check odoo/requirements.txt - b, comment_depend = b.split(except_sign) - b = b.strip() - comment_depend = comment_depend.strip() + # current_python_version = platform.python_version() + # current_platform = sys.platform + current_platform_2 = default_environment() + current_platform_2["python_version"] = ".".join( + config.set_version_python.split(".")[:2] + ) + current_platform_2["python_full_version"] = ( + config.set_version_python + ) + current_platform_2["implementation_version"] = ( + config.set_version_python + ) + req = Requirement(b) + is_compatible = req.marker.evaluate(current_platform_2) + if not is_compatible: + continue + # this support python_version into comment_depend, check odoo/requirements.txt + # b, comment_depend = b.split(except_sign) + # b = b.strip() + # comment_depend = comment_depend.strip() # print(comment_depend) + # Rewrite dependancy : module==version + b = req.name + str(req.specifier) # Regroup requirement for sign in lst_sign: @@ -169,7 +251,7 @@ def combine_requirements(config): lst_requirements.append(b) dct_requirements = get_manifest_external_dependencies( - dct_requirements, lst_sign + config, dct_requirements, lst_sign ) # Merge all requirement by insensitive @@ -214,49 +296,47 @@ def combine_requirements(config): old_requirement = requirement requirement = requirement.replace("~=", "==") lst_replace_special_sign[requirement] = old_requirement - result_number = iscompatible.parse_requirements(requirement) - if not result_number: + + match = re.search(r"[=<>~!]{1,2}(.*)", requirement) + if not match: # Ignore empty version continue - # Exception of missing feature in iscompatible - # TODO support me in iscompatible lib - no_version = result_number[0][1] - if "b" in no_version: - result_number[0] = ( - result_number[0][0], - no_version[: no_version.find("b")], - ) - elif not no_version[no_version.rfind(".") + 1 :].isnumeric(): - result_number[0] = ( - result_number[0][0], - no_version[: no_version.rfind(".")], - ) - result_number = iscompatible.string_to_tuple( - result_number[0][1] - ) + match_version = match.group(1).strip() + match_sign = match.group(0).strip()[: -len(match_version)] + match_app_name = requirement[ + : -(len(match_sign) + len(match_version)) + ] + result_number = [(match_sign, Version(match_version))] + # result_number = iscompatible.parse_requirements(requirement) + if not result_number: + continue lst_version_requirement.append((requirement, result_number)) # Check compatibility with all possibility is_compatible = True if len(lst_version_requirement) > 1: highest_value = sorted( - lst_version_requirement, key=lambda tup: tup[1] + lst_version_requirement, key=lambda tup: tup[1][0][1] )[-1] for version_requirement in lst_version_requirement: # TODO support me in iscompatible lib # check after ., because b can appear in number - v_r_split = version_requirement[0].split(".", 1) - if len(v_r_split) > 1 and "b" in v_r_split[1]: - version_requirement_upd = version_requirement[0][ - : version_requirement[0].rindex("b") - ] - else: - version_requirement_upd = version_requirement[0] + # v_r_split = version_requirement[0].split(".", 1) + # if len(v_r_split) > 1 and "b" in v_r_split[1]: + # version_requirement_upd = version_requirement[0][ + # : version_requirement[0].rindex("b") + # ] + # else: + # version_requirement_upd = version_requirement[0] + version_requirement_upd = version_requirement[0] try: is_compatible &= iscompatible.iscompatible( - version_requirement_upd, highest_value[1] + version_requirement_upd, highest_value[1][0][1] ) except Exception as e: - print(e) + _logger.error(e) + is_compatible &= iscompatible.iscompatible( + version_requirement_upd, highest_value[1][0][1] + ) if is_compatible: result = highest_value[0] else: @@ -324,7 +404,8 @@ def combine_requirements(config): for key in lst_ignored_key: del dct_requirements[key] - with open("./.venv/build_dependency.txt", "w") as f: + venv_dir = f".venv.{config.set_version_erplibre}" + with open(f"./{venv_dir}/build_dependency.txt", "w") as f: # TODO remove all comment f.writelines([f"{list(a)[0]}\n" for a in dct_requirements.values()]) @@ -375,8 +456,11 @@ def delete_dependency_poetry(pyproject_filename): toml.dump(dct_pyproject, f) -def get_manifest_external_dependencies(dct_requirements, lst_sign): - lst_manifest_file = get_lst_manifest_py() +def get_manifest_external_dependencies(config, dct_requirements, lst_sign): + lst_manifest_file = get_lst_manifest_py( + config, ignore_dir_startswith=["script/"] + ) + print(f"Find {len(lst_manifest_file)} manifest.py files.") lst_dct_ext_depend = [] for manifest_file in lst_manifest_file: with open(manifest_file, "r") as f: @@ -427,6 +511,10 @@ def main(): # lst_repo = get_all_repo() config = get_config() + print( + f"Version: Poetry '{config.set_version_poetry}' : Odoo '{config.set_version_odoo}' : Python '{config.set_version_python}' : ERPLibre '{config.set_version_erplibre}'" + ) + if not config.dry: pyproject_toml_filename = f"{config.dir}pyproject.toml" delete_dependency_poetry(pyproject_toml_filename) diff --git a/script/version/update_env_version.py b/script/version/update_env_version.py index 4b1ba67..87a0c05 100755 --- a/script/version/update_env_version.py +++ b/script/version/update_env_version.py @@ -32,7 +32,7 @@ PYPROJECT_FILE = os.path.join("pyproject.toml") PYPROJECT_TEMPLATE_FILE = "pyproject.%s.toml" POETRY_LOCK_FILE = os.path.join("poetry.lock") POETRY_LOCK_TEMPLATE_FILE = "poetry.%s.lock" -PIP_REQUIREMENT_FILE = os.path.join("requirements.txt") +# PIP_REQUIREMENT_FILE = os.path.join("requirements.txt") PIP_REQUIREMENT_TEMPLATE_FILE = "requirements.%s.txt" PIP_IGNORE_REQUIREMENT_FILE = os.path.join( "requirement", "ignore_requirements.txt" @@ -364,12 +364,12 @@ class Update: self.expected_poetry_lock_path, do_delete_source=True, ) - status &= self.update_link_file( - "Pip requirement.txt", - PIP_REQUIREMENT_FILE, - self.expected_pip_requirement_path, - do_delete_source=True, - ) + # status &= self.update_link_file( + # "Pip requirement.txt", + # PIP_REQUIREMENT_FILE, + # self.expected_pip_requirement_path, + # do_delete_source=True, + # ) status &= self.update_link_file( "Pip ignore_requirement.txt", PIP_IGNORE_REQUIREMENT_FILE,