[UPD] makefile support poetry and refactor poetry update with new rep

Makefile

@@ -38,6 +38,7 @@ endif
 -include ./conf/make.documentation.Makefile
 -include ./conf/make.image_db.Makefile
 -include ./conf/make.installation.Makefile
+-include ./conf/make.installation.poetry.Makefile
 -include ./conf/make.test.Makefile
 -include ./conf/make.todo.Makefile
 
@@ -177,13 +178,6 @@ format_script_isort_only:
 log_show_test:
 	vim ${LOG_FILE}
 
-##########
-# poetry #
-##########
-.PHONY: poetry_update
-poetry_update:
-	./script/poetry/poetry_update.py
-
 ###########
 #  clean  #
 ###########

conf/make.installation.poetry.Makefile

@@ -0,0 +1,22 @@
+###########
+# INSTALL #
+###########
+
+.PHONY: install_venv_with_minimal_pip
+install_venv_with_minimal_pip:
+	# This install .venv.odoo.version with poetry without complete pip installation
+	# This is interested to create new pip installation
+	WITH_POETRY_INSTALLATION=0 ./script/install/install_locally.sh
+
+##########
+# poetry #
+##########
+.PHONY: poetry_update
+poetry_update:
+	./script/poetry/poetry_update.py
+
+.PHONY: poetry_update_force
+poetry_update_force:
+	# Use this to recreate pip dependency from new installation, this will ignore freeze versions
+	./script/poetry/poetry_update.py -f
+

script/install/install_locally.sh

@@ -38,6 +38,7 @@ VENV_REPO_PATH=${VENV_PATH}/repo
 VENV_MULTILINGUAL_MARKDOWN_PATH=${VENV_PATH}/multilang_md.py
 #POETRY_PATH=~/.local/bin/poetry
 POETRY_PATH=${VENV_PATH}/bin/poetry
+export WITH_POETRY_INSTALLATION=1
 
 if [[ ! -n "${DOCKER_BUILD}" ]]; then
   echo "Python path version home"
@@ -129,7 +130,9 @@ if [[ ! -f "${POETRY_PATH}" ]]; then
     #    ${VENV_PATH}/bin/poetry lock --no-update
     # To fix keyring problem when installation is blocked, use
     export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
-    ${VENV_PATH}/bin/poetry install --no-root -vvv
+    if [[ ${WITH_POETRY_INSTALLATION} -ne 0 ]]; then
+      ${VENV_PATH}/bin/poetry install --no-root -vvv
+    fi
     retVal=$?
     if [[ $retVal -ne 0 ]]; then
         echo "Poetry installation error with status ${retVal}"

script/poetry/iscompatible.py

@@ -2,6 +2,9 @@
 Python versioning with requirements.txt syntax
 ==============================================
 
+# Copied from project iscompatible with support packaging Version
+# from packaging.version import Version
+
 :mod:`iscompatible` gives you the power of the pip requirements.txt
 syntax for everyday python packages, modules, classes or arbitrary
 functions.
@@ -47,8 +50,10 @@ version_info = (0, 1, 1)
 __version__ = "%s.%s.%s" % version_info
 
 
-import re
 import operator
+import re
+
+from packaging.version import Version
 
 
 def iscompatible(requirements, version):
@@ -76,10 +81,12 @@ def iscompatible(requirements, version):
 
     results = list()
 
-    for operator_string, requirement_string in parse_requirements(requirements):
+    for operator_string, requirement_string in parse_requirements(
+        requirements
+    ):
         operator = operators[operator_string]
         required = string_to_tuple(requirement_string)
-        result = operator(version, required)
+        result = operator(version.release, required)
 
         results.append(result)
 
@@ -117,9 +124,7 @@ def parse_requirements(line):
     while not LINE_END.match(line, p):
         match = VERSION.match(line, p)
         if not match:
-            raise ValueError(
+            raise ValueError("Expected version spec in", line, "at", line[p:])
-                "Expected version spec in",
-                line, "at", line[p:])
 
         specs.append(match.group(*(1, 2)))
         p = match.end()
@@ -129,8 +134,8 @@ def parse_requirements(line):
             p = match.end()  # Skip comma
         elif not LINE_END.match(line, p):
             raise ValueError(
-                "Expected ',' or end-of-list in",
+                "Expected ',' or end-of-list in", line, "at", line[p:]
-                line, "at", line[p:])
+            )
 
     return specs
 
@@ -149,9 +154,11 @@ def string_to_tuple(version):
     return tuple(map(int, version.split(".")))
 
 
-operators = {"<":   operator.lt,
+operators = {
-             "<=":  operator.le,
+    "<": operator.lt,
-             "==":  operator.eq,
+    "<=": operator.le,
-             "!=":  operator.ne,
+    "==": operator.eq,
-             ">=":  operator.ge,
+    "!=": operator.ne,
-             ">":   operator.gt}
+    ">=": operator.ge,
+    ">": operator.gt,
+}

script/poetry/poetry_update.py

@@ -6,14 +6,19 @@ import argparse
 import ast
 import logging
 import os
+import re
 from collections import OrderedDict, defaultdict
 from pathlib import Path
 
 import iscompatible
 import toml
 from colorama import Fore, Style
+from packaging.markers import default_environment
+from packaging.requirements import Requirement
+from packaging.version import Version
 
 _logger = logging.getLogger(__name__)
+_logger.setLevel(logging.INFO)
 
 
 def get_config():
@@ -56,33 +61,82 @@ def get_config():
         action="store_true",
         help="Don't apply change, only show warning.",
     )
+    parser.add_argument(
+        "--set_version_poetry",
+        help="Force to change poetry version, default is from file './.poetry-version'.",
+    )
+    parser.add_argument(
+        "--set_version_odoo",
+        help="Force to change odoo version, default is from file './.odoo-version'.",
+    )
+    parser.add_argument(
+        "--set_version_python",
+        help="Force to change python version, default is from file './.python-version'.",
+    )
+    parser.add_argument(
+        "--set_version_erplibre",
+        help="Force to change erplibre version, default is from file './.erplibre-version'.",
+    )
     args = parser.parse_args()
+    if not args.set_version_poetry:
+        with open("./.poetry-version", "r", encoding="utf-8") as fichier:
+            args.set_version_poetry = fichier.read().strip()
+    if not args.set_version_odoo:
+        with open("./.odoo-version", "r", encoding="utf-8") as fichier:
+            args.set_version_odoo = fichier.read().strip()
+    if not args.set_version_python:
+        with open("./.python-version", "r", encoding="utf-8") as fichier:
+            args.set_version_python = fichier.read().strip()
+    if not args.set_version_erplibre:
+        with open("./.erplibre-version", "r", encoding="utf-8") as fichier:
+            args.set_version_erplibre = fichier.read().strip()
     return args
 
 
-def get_lst_requirements_txt():
+def get_lst_requirements_txt(
-    return get_file_from_glob("requirements.[tT][xX][tT]")
+    config, ignore_dir_startswith: list = None, force_add_item: list = None
+):
+    return get_file_from_glob(
+        config,
+        "requirements.[tT][xX][tT]",
+        ignore_dir_startswith=ignore_dir_startswith,
+        force_add_item=force_add_item,
+    )
 
 
-def get_lst_manifest_py():
+def get_lst_manifest_py(config, ignore_dir_startswith: list = None):
-    return get_file_from_glob("__manifest__.py")
+    return get_file_from_glob(
+        config, "__manifest__.py", ignore_dir_startswith=ignore_dir_startswith
+    )
 
 
-def get_file_from_glob(glob_txt):
+def get_file_from_glob(
-    with open(".odoo-version") as txt:
+    config,
-        odoo_version = txt.read()
+    glob_txt,
-    # with open(".erplibre-version") as txt:
+    ignore_dir_startswith: list = None,
-    #     erplibre_version = txt.read()
+    force_add_item: list = None,
+):
     lst_v = []
     for a in Path(".").rglob(glob_txt):
         a_dirname = os.path.dirname(a)
         if a_dirname.startswith(".repo/") or a_dirname.startswith(".venv"):
             continue
+        if ignore_dir_startswith:
+            ignore_it = False
+            for item_ignore_dir in ignore_dir_startswith:
+                if a_dirname.startswith(item_ignore_dir):
+                    ignore_it = True
+                    break
+            if ignore_it:
+                continue
         if a_dirname.startswith("addons") and not a_dirname.startswith(
-            f"addons.odoo{odoo_version}"
+            f"addons.odoo{config.set_version_odoo}"
         ):
             continue
         lst_v.append(a)
+    if force_add_item:
+        for item in force_add_item:
+            lst_v.append(Path(item))
     return lst_v
 
 
@@ -94,21 +148,31 @@ def combine_requirements(config):
     :param config:
     :return:
     """
-    priority_filename_requirement = "requirements.txt"
+    priority_filename_requirement = (
-    second_priority_filename_requirement = "odoo/requirements.txt"
+        f"requirement/requirements.{config.set_version_erplibre}.txt"
+    )
+    second_priority_filename_requirement = (
+        f"odoo{config.set_version_odoo}/odoo/requirements.txt"
+    )
     except_sign = ";"
     lst_sign = ("==", "!=", ">=", "<=", "<", ">", except_sign, "~=")
     lst_requirements = []
-    lst_replace_special_sign = {}  # the lib iscompatible doesn't support ~=
+    lst_replace_special_sign = {}
     ignore_requirements = ["sys_platform == 'win32'", "python_version < '3.7'"]
     dct_requirements = defaultdict(set)
-    lst_requirements_file = get_lst_requirements_txt()
+    lst_requirements_file = get_lst_requirements_txt(
+        config,
+        ignore_dir_startswith=["script/"],
+        force_add_item=[priority_filename_requirement],
+    )
     # lst_requirements_with_condition = set()
     # dct_special_condition = defaultdict(list)
 
     if not os.path.isfile(priority_filename_requirement):
         _logger.error(f"File {priority_filename_requirement} not found.")
 
+    print(f"Find {len(lst_requirements_file)} requirements.txt files.")
+
     dct_requirements_module_filename = defaultdict(list)
     for requirements_filename in lst_requirements_file:
         with open(requirements_filename, "r") as f:
@@ -124,11 +188,29 @@ def combine_requirements(config):
                     b = b[: b.index("#")].strip()
                 comment_depend = ""
                 if except_sign in b:
-                    # TODO support python_version into comment_depend, check odoo/requirements.txt
+                    # current_python_version = platform.python_version()
-                    b, comment_depend = b.split(except_sign)
+                    # current_platform = sys.platform
-                    b = b.strip()
+                    current_platform_2 = default_environment()
-                    comment_depend = comment_depend.strip()
+                    current_platform_2["python_version"] = ".".join(
+                        config.set_version_python.split(".")[:2]
+                    )
+                    current_platform_2["python_full_version"] = (
+                        config.set_version_python
+                    )
+                    current_platform_2["implementation_version"] = (
+                        config.set_version_python
+                    )
+                    req = Requirement(b)
+                    is_compatible = req.marker.evaluate(current_platform_2)
+                    if not is_compatible:
+                        continue
+                    # this support python_version into comment_depend, check odoo/requirements.txt
+                    # b, comment_depend = b.split(except_sign)
+                    # b = b.strip()
+                    # comment_depend = comment_depend.strip()
                     # print(comment_depend)
+                    # Rewrite dependancy : module==version
+                    b = req.name + str(req.specifier)
 
                 # Regroup requirement
                 for sign in lst_sign:
@@ -169,7 +251,7 @@ def combine_requirements(config):
                 lst_requirements.append(b)
 
     dct_requirements = get_manifest_external_dependencies(
-        dct_requirements, lst_sign
+        config, dct_requirements, lst_sign
     )
 
     # Merge all requirement by insensitive
@@ -214,49 +296,47 @@ def combine_requirements(config):
                     old_requirement = requirement
                     requirement = requirement.replace("~=", "==")
                     lst_replace_special_sign[requirement] = old_requirement
-                result_number = iscompatible.parse_requirements(requirement)
+
-                if not result_number:
+                match = re.search(r"[=<>~!]{1,2}(.*)", requirement)
+                if not match:
                     # Ignore empty version
                     continue
-                # Exception of missing feature in iscompatible
+                match_version = match.group(1).strip()
-                # TODO support me in iscompatible lib
+                match_sign = match.group(0).strip()[: -len(match_version)]
-                no_version = result_number[0][1]
+                match_app_name = requirement[
-                if "b" in no_version:
+                    : -(len(match_sign) + len(match_version))
-                    result_number[0] = (
+                ]
-                        result_number[0][0],
+                result_number = [(match_sign, Version(match_version))]
-                        no_version[: no_version.find("b")],
+                # result_number = iscompatible.parse_requirements(requirement)
-                    )
+                if not result_number:
-                elif not no_version[no_version.rfind(".") + 1 :].isnumeric():
+                    continue
-                    result_number[0] = (
-                        result_number[0][0],
-                        no_version[: no_version.rfind(".")],
-                    )
-                result_number = iscompatible.string_to_tuple(
-                    result_number[0][1]
-                )
                 lst_version_requirement.append((requirement, result_number))
             # Check compatibility with all possibility
             is_compatible = True
             if len(lst_version_requirement) > 1:
                 highest_value = sorted(
-                    lst_version_requirement, key=lambda tup: tup[1]
+                    lst_version_requirement, key=lambda tup: tup[1][0][1]
                 )[-1]
                 for version_requirement in lst_version_requirement:
                     # TODO support me in iscompatible lib
                     # check after ., because b can appear in number
-                    v_r_split = version_requirement[0].split(".", 1)
+                    # v_r_split = version_requirement[0].split(".", 1)
-                    if len(v_r_split) > 1 and "b" in v_r_split[1]:
+                    # if len(v_r_split) > 1 and "b" in v_r_split[1]:
-                        version_requirement_upd = version_requirement[0][
+                    #     version_requirement_upd = version_requirement[0][
-                            : version_requirement[0].rindex("b")
+                    #         : version_requirement[0].rindex("b")
-                        ]
+                    #     ]
-                    else:
+                    # else:
-                        version_requirement_upd = version_requirement[0]
+                    #     version_requirement_upd = version_requirement[0]
+                    version_requirement_upd = version_requirement[0]
                     try:
                         is_compatible &= iscompatible.iscompatible(
-                            version_requirement_upd, highest_value[1]
+                            version_requirement_upd, highest_value[1][0][1]
                         )
                     except Exception as e:
-                        print(e)
+                        _logger.error(e)
+                        is_compatible &= iscompatible.iscompatible(
+                            version_requirement_upd, highest_value[1][0][1]
+                        )
                 if is_compatible:
                     result = highest_value[0]
                 else:
@@ -324,7 +404,8 @@ def combine_requirements(config):
     for key in lst_ignored_key:
         del dct_requirements[key]
 
-    with open("./.venv/build_dependency.txt", "w") as f:
+    venv_dir = f".venv.{config.set_version_erplibre}"
+    with open(f"./{venv_dir}/build_dependency.txt", "w") as f:
         # TODO remove all comment
         f.writelines([f"{list(a)[0]}\n" for a in dct_requirements.values()])
 
@@ -375,8 +456,11 @@ def delete_dependency_poetry(pyproject_filename):
         toml.dump(dct_pyproject, f)
 
 
-def get_manifest_external_dependencies(dct_requirements, lst_sign):
+def get_manifest_external_dependencies(config, dct_requirements, lst_sign):
-    lst_manifest_file = get_lst_manifest_py()
+    lst_manifest_file = get_lst_manifest_py(
+        config, ignore_dir_startswith=["script/"]
+    )
+    print(f"Find {len(lst_manifest_file)} manifest.py files.")
     lst_dct_ext_depend = []
     for manifest_file in lst_manifest_file:
         with open(manifest_file, "r") as f:
@@ -427,6 +511,10 @@ def main():
     # lst_repo = get_all_repo()
     config = get_config()
 
+    print(
+        f"Version: Poetry '{config.set_version_poetry}' : Odoo '{config.set_version_odoo}' : Python '{config.set_version_python}' : ERPLibre '{config.set_version_erplibre}'"
+    )
+
     if not config.dry:
         pyproject_toml_filename = f"{config.dir}pyproject.toml"
         delete_dependency_poetry(pyproject_toml_filename)

script/version/update_env_version.py

@@ -32,7 +32,7 @@ PYPROJECT_FILE = os.path.join("pyproject.toml")
 PYPROJECT_TEMPLATE_FILE = "pyproject.%s.toml"
 POETRY_LOCK_FILE = os.path.join("poetry.lock")
 POETRY_LOCK_TEMPLATE_FILE = "poetry.%s.lock"
-PIP_REQUIREMENT_FILE = os.path.join("requirements.txt")
+# PIP_REQUIREMENT_FILE = os.path.join("requirements.txt")
 PIP_REQUIREMENT_TEMPLATE_FILE = "requirements.%s.txt"
 PIP_IGNORE_REQUIREMENT_FILE = os.path.join(
     "requirement", "ignore_requirements.txt"
@@ -364,12 +364,12 @@ class Update:
             self.expected_poetry_lock_path,
             do_delete_source=True,
         )
-        status &= self.update_link_file(
+        # status &= self.update_link_file(
-            "Pip requirement.txt",
+        #     "Pip requirement.txt",
-            PIP_REQUIREMENT_FILE,
+        #     PIP_REQUIREMENT_FILE,
-            self.expected_pip_requirement_path,
+        #     self.expected_pip_requirement_path,
-            do_delete_source=True,
+        #     do_delete_source=True,
-        )
+        # )
         status &= self.update_link_file(
             "Pip ignore_requirement.txt",
             PIP_IGNORE_REQUIREMENT_FILE,