mathben/erplibre
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

[UPD] makefile support poetry and refactor poetry update with new rep

Browse source
This commit is contained in:
Mathieu Benoit 2025-07-14 03:44:48 -04:00
parent 9cafb3cead
commit 770e4b0966
6 changed files with 193 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Makefile
View file

@ -38,6 +38,7 @@ endif
-include ./conf/make.documentation.Makefile
-include ./conf/make.image_db.Makefile
-include ./conf/make.installation.Makefile
-include ./conf/make.installation.poetry.Makefile
-include ./conf/make.test.Makefile
-include ./conf/make.todo.Makefile
@ -177,13 +178,6 @@ format_script_isort_only:
log_show_test:
vim ${LOG_FILE}
##########
# poetry #
##########
.PHONY: poetry_update
poetry_update:
./script/poetry/poetry_update.py
###########
# clean #
###########

22
conf/make.installation.poetry.Makefile Normal file
View file

@ -0,0 +1,22 @@
###########
# INSTALL #
###########
.PHONY: install_venv_with_minimal_pip
install_venv_with_minimal_pip:
# This install .venv.odoo.version with poetry without complete pip installation
# This is interested to create new pip installation
WITH_POETRY_INSTALLATION=0 ./script/install/install_locally.sh
##########
# poetry #
##########
.PHONY: poetry_update
poetry_update:
./script/poetry/poetry_update.py
.PHONY: poetry_update_force
poetry_update_force:
# Use this to recreate pip dependency from new installation, this will ignore freeze versions
./script/poetry/poetry_update.py -f

5
script/install/install_locally.sh
View file

@ -38,6 +38,7 @@ VENV_REPO_PATH=${VENV_PATH}/repo
VENV_MULTILINGUAL_MARKDOWN_PATH=${VENV_PATH}/multilang_md.py
#POETRY_PATH=~/.local/bin/poetry
POETRY_PATH=${VENV_PATH}/bin/poetry
export WITH_POETRY_INSTALLATION=1
if [[ ! -n "${DOCKER_BUILD}" ]]; then
echo "Python path version home"
@ -129,7 +130,9 @@ if [[ ! -f "${POETRY_PATH}" ]]; then
# ${VENV_PATH}/bin/poetry lock --no-update
# To fix keyring problem when installation is blocked, use
export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
${VENV_PATH}/bin/poetry install --no-root -vvv
if [[ ${WITH_POETRY_INSTALLATION} -ne 0 ]]; then
${VENV_PATH}/bin/poetry install --no-root -vvv
fi
retVal=$?
if [[ $retVal -ne 0 ]]; then
echo "Poetry installation error with status ${retVal}"

35
script/poetry/iscompatible.py
View file

@ -2,6 +2,9 @@
Python versioning with requirements.txt syntax
==============================================
# Copied from project iscompatible with support packaging Version
# from packaging.version import Version
:mod:`iscompatible` gives you the power of the pip requirements.txt
syntax for everyday python packages, modules, classes or arbitrary
functions.
@ -47,8 +50,10 @@ version_info = (0, 1, 1)
__version__ = "%s.%s.%s" % version_info
import re
import operator
import re
from packaging.version import Version
def iscompatible(requirements, version):
@ -76,10 +81,12 @@ def iscompatible(requirements, version):
results = list()
for operator_string, requirement_string in parse_requirements(requirements):
for operator_string, requirement_string in parse_requirements(
requirements
):
operator = operators[operator_string]
required = string_to_tuple(requirement_string)
result = operator(version, required)
result = operator(version.release, required)
results.append(result)
@ -117,9 +124,7 @@ def parse_requirements(line):
while not LINE_END.match(line, p):
match = VERSION.match(line, p)
if not match:
raise ValueError(
"Expected version spec in",
line, "at", line[p:])
raise ValueError("Expected version spec in", line, "at", line[p:])
specs.append(match.group(*(1, 2)))
p = match.end()
@ -129,8 +134,8 @@ def parse_requirements(line):
p = match.end() # Skip comma
elif not LINE_END.match(line, p):
raise ValueError(
"Expected ',' or end-of-list in",
line, "at", line[p:])
"Expected ',' or end-of-list in", line, "at", line[p:]
)
return specs
@ -149,9 +154,11 @@ def string_to_tuple(version):
return tuple(map(int, version.split(".")))
operators = {"<": operator.lt,
"<=": operator.le,
"==": operator.eq,
"!=": operator.ne,
">=": operator.ge,
">": operator.gt}
operators = {
"<": operator.lt,
"<=": operator.le,
"==": operator.eq,
"!=": operator.ne,
">=": operator.ge,
">": operator.gt,
}

188
script/poetry/poetry_update.py
View file

@ -6,14 +6,19 @@ import argparse
import ast
import logging
import os
import re
from collections import OrderedDict, defaultdict
from pathlib import Path
import iscompatible
import toml
from colorama import Fore, Style
from packaging.markers import default_environment
from packaging.requirements import Requirement
from packaging.version import Version
_logger = logging.getLogger(__name__)
_logger.setLevel(logging.INFO)
def get_config():
@ -56,33 +61,82 @@ def get_config():
action="store_true",
help="Don't apply change, only show warning.",
)
parser.add_argument(
"--set_version_poetry",
help="Force to change poetry version, default is from file './.poetry-version'.",
)
parser.add_argument(
"--set_version_odoo",
help="Force to change odoo version, default is from file './.odoo-version'.",
)
parser.add_argument(
"--set_version_python",
help="Force to change python version, default is from file './.python-version'.",
)
parser.add_argument(
"--set_version_erplibre",
help="Force to change erplibre version, default is from file './.erplibre-version'.",
)
args = parser.parse_args()
if not args.set_version_poetry:
with open("./.poetry-version", "r", encoding="utf-8") as fichier:
args.set_version_poetry = fichier.read().strip()
if not args.set_version_odoo:
with open("./.odoo-version", "r", encoding="utf-8") as fichier:
args.set_version_odoo = fichier.read().strip()
if not args.set_version_python:
with open("./.python-version", "r", encoding="utf-8") as fichier:
args.set_version_python = fichier.read().strip()
if not args.set_version_erplibre:
with open("./.erplibre-version", "r", encoding="utf-8") as fichier:
args.set_version_erplibre = fichier.read().strip()
return args
def get_lst_requirements_txt():
return get_file_from_glob("requirements.[tT][xX][tT]")
def get_lst_requirements_txt(
config, ignore_dir_startswith: list = None, force_add_item: list = None
):
return get_file_from_glob(
config,
"requirements.[tT][xX][tT]",
ignore_dir_startswith=ignore_dir_startswith,
force_add_item=force_add_item,
)
def get_lst_manifest_py():
return get_file_from_glob("__manifest__.py")
def get_lst_manifest_py(config, ignore_dir_startswith: list = None):
return get_file_from_glob(
config, "__manifest__.py", ignore_dir_startswith=ignore_dir_startswith
)
def get_file_from_glob(glob_txt):
with open(".odoo-version") as txt:
odoo_version = txt.read()
# with open(".erplibre-version") as txt:
# erplibre_version = txt.read()
def get_file_from_glob(
config,
glob_txt,
ignore_dir_startswith: list = None,
force_add_item: list = None,
):
lst_v = []
for a in Path(".").rglob(glob_txt):
a_dirname = os.path.dirname(a)
if a_dirname.startswith(".repo/") or a_dirname.startswith(".venv"):
continue
if ignore_dir_startswith:
ignore_it = False
for item_ignore_dir in ignore_dir_startswith:
if a_dirname.startswith(item_ignore_dir):
ignore_it = True
break
if ignore_it:
continue
if a_dirname.startswith("addons") and not a_dirname.startswith(
f"addons.odoo{odoo_version}"
f"addons.odoo{config.set_version_odoo}"
):
continue
lst_v.append(a)
if force_add_item:
for item in force_add_item:
lst_v.append(Path(item))
return lst_v
@ -94,21 +148,31 @@ def combine_requirements(config):
:param config:
:return:
"""
priority_filename_requirement = "requirements.txt"
second_priority_filename_requirement = "odoo/requirements.txt"
priority_filename_requirement = (
f"requirement/requirements.{config.set_version_erplibre}.txt"
)
second_priority_filename_requirement = (
f"odoo{config.set_version_odoo}/odoo/requirements.txt"
)
except_sign = ";"
lst_sign = ("==", "!=", ">=", "<=", "<", ">", except_sign, "~=")
lst_requirements = []
lst_replace_special_sign = {} # the lib iscompatible doesn't support ~=
lst_replace_special_sign = {}
ignore_requirements = ["sys_platform == 'win32'", "python_version < '3.7'"]
dct_requirements = defaultdict(set)
lst_requirements_file = get_lst_requirements_txt()
lst_requirements_file = get_lst_requirements_txt(
config,
ignore_dir_startswith=["script/"],
force_add_item=[priority_filename_requirement],
)
# lst_requirements_with_condition = set()
# dct_special_condition = defaultdict(list)
if not os.path.isfile(priority_filename_requirement):
_logger.error(f"File {priority_filename_requirement} not found.")
print(f"Find {len(lst_requirements_file)} requirements.txt files.")
dct_requirements_module_filename = defaultdict(list)
for requirements_filename in lst_requirements_file:
with open(requirements_filename, "r") as f:
@ -124,11 +188,29 @@ def combine_requirements(config):
b = b[: b.index("#")].strip()
comment_depend = ""
if except_sign in b:
# TODO support python_version into comment_depend, check odoo/requirements.txt
b, comment_depend = b.split(except_sign)
b = b.strip()
comment_depend = comment_depend.strip()
# current_python_version = platform.python_version()
# current_platform = sys.platform
current_platform_2 = default_environment()
current_platform_2["python_version"] = ".".join(
config.set_version_python.split(".")[:2]
)
current_platform_2["python_full_version"] = (
config.set_version_python
)
current_platform_2["implementation_version"] = (
config.set_version_python
)
req = Requirement(b)
is_compatible = req.marker.evaluate(current_platform_2)
if not is_compatible:
continue
# this support python_version into comment_depend, check odoo/requirements.txt
# b, comment_depend = b.split(except_sign)
# b = b.strip()
# comment_depend = comment_depend.strip()
# print(comment_depend)
# Rewrite dependancy : module==version
b = req.name + str(req.specifier)
# Regroup requirement
for sign in lst_sign:
@ -169,7 +251,7 @@ def combine_requirements(config):
lst_requirements.append(b)
dct_requirements = get_manifest_external_dependencies(
dct_requirements, lst_sign
config, dct_requirements, lst_sign
)
# Merge all requirement by insensitive
@ -214,49 +296,47 @@ def combine_requirements(config):
old_requirement = requirement
requirement = requirement.replace("~=", "==")
lst_replace_special_sign[requirement] = old_requirement
result_number = iscompatible.parse_requirements(requirement)
if not result_number:
match = re.search(r"[=<>~!]{1,2}(.*)", requirement)
if not match:
# Ignore empty version
continue
# Exception of missing feature in iscompatible
# TODO support me in iscompatible lib
no_version = result_number[0][1]
if "b" in no_version:
result_number[0] = (
result_number[0][0],
no_version[: no_version.find("b")],
)
elif not no_version[no_version.rfind(".") + 1 :].isnumeric():
result_number[0] = (
result_number[0][0],
no_version[: no_version.rfind(".")],
)
result_number = iscompatible.string_to_tuple(
result_number[0][1]
)
match_version = match.group(1).strip()
match_sign = match.group(0).strip()[: -len(match_version)]
match_app_name = requirement[
: -(len(match_sign) + len(match_version))
]
result_number = [(match_sign, Version(match_version))]
# result_number = iscompatible.parse_requirements(requirement)
if not result_number:
continue
lst_version_requirement.append((requirement, result_number))
# Check compatibility with all possibility
is_compatible = True
if len(lst_version_requirement) > 1:
highest_value = sorted(
lst_version_requirement, key=lambda tup: tup[1]
lst_version_requirement, key=lambda tup: tup[1][0][1]
)[-1]
for version_requirement in lst_version_requirement:
# TODO support me in iscompatible lib
# check after ., because b can appear in number
v_r_split = version_requirement[0].split(".", 1)
if len(v_r_split) > 1 and "b" in v_r_split[1]:
version_requirement_upd = version_requirement[0][
: version_requirement[0].rindex("b")
]
else:
version_requirement_upd = version_requirement[0]
# v_r_split = version_requirement[0].split(".", 1)
# if len(v_r_split) > 1 and "b" in v_r_split[1]:
# version_requirement_upd = version_requirement[0][
# : version_requirement[0].rindex("b")
# ]
# else:
# version_requirement_upd = version_requirement[0]
version_requirement_upd = version_requirement[0]
try:
is_compatible &= iscompatible.iscompatible(
version_requirement_upd, highest_value[1]
version_requirement_upd, highest_value[1][0][1]
)
except Exception as e:
print(e)
_logger.error(e)
is_compatible &= iscompatible.iscompatible(
version_requirement_upd, highest_value[1][0][1]
)
if is_compatible:
result = highest_value[0]
else:
@ -324,7 +404,8 @@ def combine_requirements(config):
for key in lst_ignored_key:
del dct_requirements[key]
with open("./.venv/build_dependency.txt", "w") as f:
venv_dir = f".venv.{config.set_version_erplibre}"
with open(f"./{venv_dir}/build_dependency.txt", "w") as f:
# TODO remove all comment
f.writelines([f"{list(a)[0]}\n" for a in dct_requirements.values()])
@ -375,8 +456,11 @@ def delete_dependency_poetry(pyproject_filename):
toml.dump(dct_pyproject, f)
def get_manifest_external_dependencies(dct_requirements, lst_sign):
lst_manifest_file = get_lst_manifest_py()
def get_manifest_external_dependencies(config, dct_requirements, lst_sign):
lst_manifest_file = get_lst_manifest_py(
config, ignore_dir_startswith=["script/"]
)
print(f"Find {len(lst_manifest_file)} manifest.py files.")
lst_dct_ext_depend = []
for manifest_file in lst_manifest_file:
with open(manifest_file, "r") as f:
@ -427,6 +511,10 @@ def main():
# lst_repo = get_all_repo()
config = get_config()
print(
f"Version: Poetry '{config.set_version_poetry}' : Odoo '{config.set_version_odoo}' : Python '{config.set_version_python}' : ERPLibre '{config.set_version_erplibre}'"
)
if not config.dry:
pyproject_toml_filename = f"{config.dir}pyproject.toml"
delete_dependency_poetry(pyproject_toml_filename)

14
script/version/update_env_version.py
View file

@ -32,7 +32,7 @@ PYPROJECT_FILE = os.path.join("pyproject.toml")
PYPROJECT_TEMPLATE_FILE = "pyproject.%s.toml"
POETRY_LOCK_FILE = os.path.join("poetry.lock")
POETRY_LOCK_TEMPLATE_FILE = "poetry.%s.lock"
PIP_REQUIREMENT_FILE = os.path.join("requirements.txt")
# PIP_REQUIREMENT_FILE = os.path.join("requirements.txt")
PIP_REQUIREMENT_TEMPLATE_FILE = "requirements.%s.txt"
PIP_IGNORE_REQUIREMENT_FILE = os.path.join(
"requirement", "ignore_requirements.txt"
@ -364,12 +364,12 @@ class Update:
self.expected_poetry_lock_path,
do_delete_source=True,
)
status &= self.update_link_file(
"Pip requirement.txt",
PIP_REQUIREMENT_FILE,
self.expected_pip_requirement_path,
do_delete_source=True,
)
# status &= self.update_link_file(
# "Pip requirement.txt",
# PIP_REQUIREMENT_FILE,
# self.expected_pip_requirement_path,
# do_delete_source=True,
# )
status &= self.update_link_file(
"Pip ignore_requirement.txt",
PIP_IGNORE_REQUIREMENT_FILE,