diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
index 1940626..f66958e 100644
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,6 +1,6 @@
# These are supported funding model platforms
-github: mathben963 # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+github: mathben # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
diff --git a/.gitignore b/.gitignore
index 03104fb..c484315 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ config.conf
*.DS_Store
*.pyc
.coverage*
+coverage*
.repo
!addons/
diff --git a/Makefile b/Makefile
index ec480b5..73af3c3 100644
--- a/Makefile
+++ b/Makefile
@@ -616,12 +616,27 @@ test_full_fast_coverage:
./.venv/bin/coverage erase
./script/test/run_parallel_test.py --coverage
# TODO This test is broken in parallel
- ./script/make.sh test_code_generator_hello_world
+ ./script/make.sh test_coverage_code_generator_hello_world
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage
+
+.PHONY: test_cg_demo
+test_cg_demo:
+ ./script/make.sh clean
+ # Need to create a BD to create cache _cache_erplibre_base
+ ./script/database/db_restore.py --database test
+ ./.venv/bin/coverage erase
+ ./script/addons/coverage_install_addons_dev.sh test code_generator_demo
+ ./.venv/bin/coverage combine -a
+ ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
+
+
.PHONY: test_base
test_base:
./script/make.sh test_format
@@ -645,6 +660,10 @@ test_format:
test_code_generator_hello_world:
./test/code_generator/hello_world.sh
+.PHONY: test_coverage_code_generator_hello_world
+test_coverage_code_generator_hello_world:
+ ./test/code_generator/coverage_hello_world.sh
+
.PHONY: test_installation_demo
test_installation_demo:
./script/code_generator/check_git_change_code_generator.sh ./addons/TechnoLibre_odoo-code-generator-template
@@ -752,6 +771,7 @@ test_addons_sale:
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m
./.venv/bin/coverage html
+ ./.venv/bin/coverage json
.PHONY: test_addons_helpdesk
test_addons_helpdesk:
@@ -761,16 +781,18 @@ test_addons_helpdesk:
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m
./.venv/bin/coverage html
+ ./.venv/bin/coverage json
.PHONY: test_addons_code_generator
test_addons_code_generator:
./.venv/bin/coverage erase
./.venv/bin/python3 ./odoo/odoo-bin db --drop --database test_addons_code_generator
# TODO missing test in code_generator
- ./test.sh --dev all -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator
+ ./test.sh --dev cg -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage
.PHONY: test_addons_code_generator_code_generator
@@ -778,10 +800,24 @@ test_addons_code_generator_code_generator:
# TODO this test only generation, not test
./.venv/bin/coverage erase
./script/database/db_restore.py --database test_addons_code_generator_code_generator
- ./test.sh --dev all -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator
+ ./test.sh --dev cg -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
+ # run: make open_test_coverage
+
+.PHONY: test_addons_code_generator_template_code_generator
+test_addons_code_generator_template_code_generator:
+ # TODO this test only generation, not test
+ ./.venv/bin/coverage erase
+ ./script/database/db_restore.py --database test_addons_code_generator_template_code_generator
+ ./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator
+ ./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator_template_code_generator
+ ./.venv/bin/coverage combine -a
+ ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
+ ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage
.PHONY: open_test_coverage
@@ -1037,6 +1073,17 @@ i18n_generate_demo_portal:
clean:
find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+###############
+# Statistic #
+###############
+.PHONY: stat_module_evolution_per_year
+stat_module_evolution_per_year:
+ ./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7
+
+.PHONY: stat_module_evolution_per_year_OCA
+stat_module_evolution_per_year_OCA:
+ ./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7
+
###################
# Documentation #
###################
diff --git a/doc/FAQ.md b/doc/FAQ.md
index ea13d20..f94565a 100644
--- a/doc/FAQ.md
+++ b/doc/FAQ.md
@@ -124,3 +124,19 @@ docker-machine create --driver virtualbox default
docker-machine env default
eval "$(docker-machine env default)"
```
+
+### Run error `current limit exceeds maximum limit`
+
+When you got
+
+```
+File "/Users/test/Desktop/ERPLibre/odoo/odoo/service/server.py", line 83, in set_limit_memory_hard
+resource.setrlimit(rlimit, (config['limit_memory_hard'], hard))
+ValueError: current limit exceeds maximum limit
+```
+
+Add line at the end of config.conf
+
+```
+limit_memory_hard = 0
+```
diff --git a/manifest/default.dev.xml b/manifest/default.dev.xml
index 19471ec..50c6782 100644
--- a/manifest/default.dev.xml
+++ b/manifest/default.dev.xml
@@ -93,7 +93,7 @@
-
+
diff --git a/poetry.lock b/poetry.lock
index b908285..f4964eb 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -5210,6 +5210,27 @@ files = [
{file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"},
]
+[[package]]
+name = "tqdm"
+version = "4.64.1"
+description = "Fast, Extensible Progress Meter"
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
+files = [
+ {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"},
+ {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["py-make (>=0.1.0)", "twine", "wheel"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
[[package]]
name = "typed-ast"
version = "1.5.4"
@@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+[[package]]
+name = "uvloop"
+version = "0.17.0"
+description = "Fast implementation of asyncio event loop on top of libuv"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"},
+ {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"},
+ {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"},
+ {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"},
+ {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"},
+ {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"},
+ {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"},
+ {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"},
+ {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"},
+ {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"},
+ {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"},
+ {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"},
+ {file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"},
+ {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"},
+ {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"},
+ {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"},
+ {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"},
+ {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"},
+ {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"},
+ {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"},
+ {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"},
+ {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"},
+ {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"},
+ {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"},
+ {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"},
+ {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"},
+ {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"},
+ {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"},
+ {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"},
+ {file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"},
+]
+
+[package.extras]
+dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
+docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
+test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"]
+
[[package]]
name = "validators"
version = "0.20.0"
@@ -5879,4 +5945,4 @@ files = [
[metadata]
lock-version = "2.0"
python-versions = "^3.7.16"
-content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e"
+content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368"
diff --git a/pyproject.toml b/pyproject.toml
index 6935dd9..6a696c1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -185,6 +185,8 @@ xmltodict = "^0.13.0"
zeep = "^4.2.1"
zipp = ">=3.6.0"
zxcvbn = "^4.4.28"
+tqdm = "^4.64.1"
+uvloop = "^0.17.0"
[tool.poetry.dev-dependencies]
websocket_client = "^1.1.0"
diff --git a/requirements.txt b/requirements.txt
index 441cfc7..04cc620 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,6 @@ openupgradelib
unidiff
colorama
wheel
-aioshutil<1.2
# For OSX
cython
@@ -80,6 +79,10 @@ numpy==1.21.1
feedparser==6.0.10
geojson<3.0.0
+# asyncio
+aioshutil<1.2
+tqdm
+uvloop
# Solve pcodedmp, oletools, rtfde and extract-msg
win_unicode_console
diff --git a/script/addons/update_prod_to_dev.sh b/script/addons/update_prod_to_dev.sh
index 071c701..6b948eb 100755
--- a/script/addons/update_prod_to_dev.sh
+++ b/script/addons/update_prod_to_dev.sh
@@ -2,7 +2,7 @@
# This script will remove mail configuration, remove backup configuration, and force admin user to test/test
echo "Update prod to dev on BD '$1'"
-./script/addons/install_addons.sh "$1" user_test,disable_mail_server,disable_auto_backup
+./script/addons/install_addons_dev.sh "$1" user_test,disable_mail_server,disable_auto_backup
retVal=$?
if [[ $retVal -ne 0 ]]; then
diff --git a/script/code_generator/technical/transform_xml_data_website_page_to_controller.py b/script/code_generator/technical/transform_xml_data_website_page_to_controller.py
new file mode 100644
index 0000000..9a30636
--- /dev/null
+++ b/script/code_generator/technical/transform_xml_data_website_page_to_controller.py
@@ -0,0 +1,149 @@
+#!./.venv/bin/python
+import argparse
+import logging
+import os
+import sys
+
+import black
+import xmltodict
+
+logging.basicConfig(level=logging.DEBUG)
+_logger = logging.getLogger(__name__)
+
+
+def get_config():
+ """Parse command line arguments, extracting the config file name,
+ returning the union of config file and command line arguments
+
+ :return: dict of config file settings and command line arguments
+ """
+
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description="""\
+ Transform an XML into Python
+""",
+ epilog="""\
+""",
+ )
+ parser.add_argument(
+ "--source",
+ dest="source",
+ required=True,
+ help="File to transform (XML)",
+ )
+ # parser.add_argument(
+ # "--destination",
+ # dest="destination",
+ # help=(
+ # "Result, output (Python). If not set, output will be console."
+ # ),
+ # )
+ # parser.add_argument(
+ # "-q",
+ # "--quiet",
+ # action="store_true",
+ # help="Don't show output of found model.",
+ # )
+ args = parser.parse_args()
+ if not os.path.isfile(args.source):
+ raise Exception("Argument --source need to be a valid path to a file.")
+ return args
+
+
+def get_module_name(filepath):
+ manifest_file = "__manifest__.py"
+ current_path = os.path.abspath(os.path.dirname(filepath))
+ while current_path != "/":
+ manifest_path = os.path.join(current_path, manifest_file)
+ if os.path.isfile(manifest_path):
+ return os.path.basename(current_path)
+ current_path = os.path.dirname(current_path)
+ raise FileNotFoundError(f"No {manifest_file} found for {filepath}")
+
+
+def get_value_from_field(lst_field, by_attr, name, record_id, get_attr):
+ field_info = [a for a in lst_field if a.get(by_attr) == name]
+ if not field_info:
+ raise ValueError(
+ f"Missing from record {record_id}"
+ )
+ field_info = field_info[0]
+ return field_info.get(get_attr)
+
+
+def main():
+ config = get_config()
+ with open(config.source) as xml:
+ xml_as_string = xml.read()
+ xml_dict = xmltodict.parse(xml_as_string)
+ # TODO replace this by get_value(xml_dict, "odoo.data.record")
+ xml_dict_odoo = xml_dict.get("odoo")
+ if not xml_dict_odoo:
+ raise ValueError(f"Missing into {config.source}")
+ xml_dict_data = xml_dict_odoo.get("data")
+ if not xml_dict_data:
+ raise ValueError(f"Missing into {config.source}")
+ xml_dict_record = xml_dict_data.get("record")
+ if not xml_dict_record:
+ raise ValueError(f"Missing into {config.source}")
+ if type(xml_dict_record) is not list:
+ lst_xml_record = [xml_dict_record]
+ else:
+ lst_xml_record = xml_dict_record
+ lst_output = []
+ for record in lst_xml_record:
+ lst_field = record.get("field")
+ if type(lst_field) is not list:
+ lst_field = [lst_field]
+
+ # Condition unique
+ try:
+ get_value_from_field(
+ lst_field, "@name", "groups_id", record.get("@id"), "@eval"
+ )
+ except:
+ continue
+
+ url = get_value_from_field(
+ lst_field, "@name", "url", record.get("@id"), "#text"
+ )
+ view_id = get_value_from_field(
+ lst_field, "@name", "view_id", record.get("@id"), "@ref"
+ )
+ module_name = get_module_name(config.source)
+ method_name = (
+ f'get_{"_".join(url.replace("-", "_").strip("/").split("/"))}'
+ )
+
+# template = f"""
+# @http.route(
+# ["{url}"],
+# type="http",
+# auth="user",
+# website=True,
+# )
+# def {method_name}(self, **kw):
+# return request.env["ir.ui.view"].render_template(
+# "{module_name}.{view_id}",
+# )
+ template = f"""
+@http.route(
+ ["{url}"],
+ type="http",
+ auth="user",
+ website=True,
+)
+def {method_name}(self, **kw):
+ return request.env.ref('{module_name}.{view_id}').render()
+"""
+ lst_output.append(template)
+ str_output = "\n".join(lst_output)
+ formated_str_output = black.format_str(str_output, mode=black.Mode())
+ print(formated_str_output)
+ return 0
+
+
+if __name__ == "__main__":
+ status = main()
+ sys.exit(status)
diff --git a/script/install/install_OSX_dependency.sh b/script/install/install_OSX_dependency.sh
index 3480572..8319814 100755
--- a/script/install/install_OSX_dependency.sh
+++ b/script/install/install_OSX_dependency.sh
@@ -25,23 +25,21 @@ sudo su - postgres -c "CREATE EXTENSION postgis;\nCREATE EXTENSION postgis_topol
# Install Dependencies
#--------------------------------------------------
echo "\n--- Installing Python 3 + pip3 --"
-brew install git python3 wget pyenv
+brew install git python3 wget
brew link git
brew link wget
+
+echo "\n--- Installing extra --"
+brew install parallel
echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----"
brew install nodejs npm openssl
sudo npm install -g rtlcss
sudo npm install -g less
sudo npm install -g prettier
sudo npm install -g prettier @prettier/plugin-xml
-yes n|pyenv install 3.7.16
-pyenv local 3.7.16
echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.zshrc
-echo -e "\n---- Installing poetry for reliable python package ----"
-curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
-
#--------------------------------------------------
# Install Wkhtmltopdf if needed
#--------------------------------------------------
diff --git a/script/lib_asyncio.py b/script/lib_asyncio.py
new file mode 100644
index 0000000..7b64e90
--- /dev/null
+++ b/script/lib_asyncio.py
@@ -0,0 +1,152 @@
+#!./.venv/bin/python
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
+import asyncio
+from collections import deque
+
+import uvloop
+
+
+async def run_in_serial(task_list):
+ q = asyncio.Queue()
+ for task in task_list:
+ await q.put(task)
+ lst_result = []
+ for i in range(len(task_list)):
+ co = await q.get()
+ result = await co
+ lst_result.append(result)
+ return lst_result
+
+
+async def run_command_get_output(*args, cwd=None):
+ if cwd is not None:
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ # stdout must a pipe to be accessible as process.stdout
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ cwd=cwd,
+ )
+ else:
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ # stdout must a pipe to be accessible as process.stdout
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ # Wait for the subprocess to finish
+ stdout, stderr = await process.communicate()
+ return stdout.decode()
+
+
+async def run_command_get_output_and_status(*args, cwd=None):
+ if cwd is not None:
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ # stdout must a pipe to be accessible as process.stdout
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ cwd=cwd,
+ )
+ else:
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ # stdout must a pipe to be accessible as process.stdout
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ # Wait for the subprocess to finish
+ stdout, stderr = await process.communicate()
+ return stdout.decode(), stderr.decode(), process.returncode
+
+
+def print_summary_task(task_list):
+ for task in task_list:
+ print(task.cr_code.co_name)
+
+
+def execute(config, lst_task, use_uvloop=False):
+
+ if not config.no_parallel and asyncio.get_event_loop().is_closed():
+ asyncio.set_event_loop(asyncio.new_event_loop())
+
+ if config.no_parallel:
+ return_value = asyncio.run(run_in_serial(lst_task))
+ elif config.max_process:
+ pool = AsyncioPool(config.max_process)
+ for task in lst_task:
+ pool.add_coro(task)
+ try:
+ return_value = pool.run_until_complete()
+ finally:
+ pool.close()
+ else:
+ # Use maximal resource
+ if use_uvloop:
+ uvloop.install()
+ loop = asyncio.get_event_loop()
+ if config.debug:
+ loop.set_debug(True)
+ try:
+ commands = asyncio.gather(*lst_task)
+ return_value = loop.run_until_complete(commands)
+ finally:
+ loop.close()
+ return return_value
+
+
+class AsyncioPool:
+ # TODO check to replace this pool by https://github.com/dano/aioprocessing
+ def __init__(self, concurrency, loop=None):
+ """
+ @param loop: asyncio loop
+ @param concurrency: Maximum number of concurrently running tasks
+ """
+ self._loop = loop or asyncio.get_event_loop()
+ self._concurrency = concurrency
+ self._coros = deque([]) # All coroutines queued for execution
+ self._futures = [] # All currently running coroutines
+ self._lst_result = []
+
+ def close(self):
+ self._loop.close()
+
+ def add_coro(self, coro):
+ """
+ @param coro: coroutine to add
+ """
+ self._coros.append(coro)
+ self.print_status()
+
+ def run_until_complete(self):
+ self._loop.run_until_complete(self._wait_for_futures())
+ return self._lst_result
+
+ def print_status(self):
+ print(
+ " Status: coros:%s - futures:%s"
+ % (len(self._coros), len(self._futures))
+ )
+
+ def _start_futures(self):
+ num_to_start = self._concurrency - len(self._futures)
+ num_to_start = min(num_to_start, len(self._coros))
+ for _ in range(num_to_start):
+ coro = self._coros.popleft()
+ future = asyncio.ensure_future(coro, loop=self._loop)
+ self._futures.append(future)
+ self.print_status()
+
+ async def _wait_for_futures(self):
+ while len(self._coros) > 0 or len(self._futures) > 0:
+ self._start_futures()
+ futures_completed, futures_pending = await asyncio.wait(
+ self._futures,
+ loop=self._loop,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+
+ for future in futures_completed:
+ self._lst_result.append(future.result())
+ self._futures.remove(future)
+ self._start_futures()
diff --git a/script/statistic/code_count.sh b/script/statistic/code_count.sh
index 23ef2d8..4eeb285 100755
--- a/script/statistic/code_count.sh
+++ b/script/statistic/code_count.sh
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
# all argument is directory/file separate by space
-./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,css,scss --format=summary "$@"
+./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,csv,css,scss --format=summary "$@"
diff --git a/script/statistic/show_evolution_module.py b/script/statistic/show_evolution_module.py
new file mode 100755
index 0000000..5b06779
--- /dev/null
+++ b/script/statistic/show_evolution_module.py
@@ -0,0 +1,519 @@
+#!./.venv/bin/python
+import argparse
+import datetime
+import logging
+import os
+import shutil
+import sys
+import csv
+from collections import defaultdict
+
+from dateutil.relativedelta import relativedelta
+
+new_path = os.path.normpath(
+ os.path.join(os.path.dirname(__file__), "..", "..")
+)
+sys.path.append(new_path)
+
+from script import lib_asyncio
+
+FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT)
+
+_logger = logging.getLogger(__name__)
+
+PROJECT_NAME = os.path.basename(os.getcwd())
+VENV_PATH = "./.venv"
+CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo")
+CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv"
+MAX_COROUTINE = 300
+IGNORE_REPO_LIST = (
+ "https://github.com/OCA/odoo-module-migrator.git",
+ "https://github.com/OCA/maintainer-tools.git",
+ "https://github.com/itpp-labs/odoo-development.git",
+ "https://github.com/itpp-labs/odoo-port-docs.git",
+ "https://github.com/itpp-labs/odoo-test-docs.git",
+ "https://github.com/odoo/documentation.git",
+ "https://github.com/ERPLibre/ERPLibre_image_db.git",
+ "https://github.com/muk-it/muk_docs.git",
+)
+DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": ["/addons/", "/odoo/addons/"]}
+DCT_VERSION_RELEASE = {
+ "1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(),
+ "2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(),
+ "3.0": datetime.datetime.strptime("2005-09-01", "%Y-%m-%d").date(),
+ "4.0": datetime.datetime.strptime("2006-12-01", "%Y-%m-%d").date(),
+ "5.0": datetime.datetime.strptime("2009-04-01", "%Y-%m-%d").date(),
+ "6.0": datetime.datetime.strptime("2009-10-01", "%Y-%m-%d").date(),
+ "6.1": datetime.datetime.strptime("2012-02-27", "%Y-%m-%d").date(),
+ "7.0": datetime.datetime.strptime("2012-12-01", "%Y-%m-%d").date(),
+ "8.0": datetime.datetime.strptime("2014-09-01", "%Y-%m-%d").date(),
+ "9.0": datetime.datetime.strptime("2015-11-01", "%Y-%m-%d").date(),
+ "10.0": datetime.datetime.strptime("2016-10-01", "%Y-%m-%d").date(),
+ "11.0": datetime.datetime.strptime("2017-10-01", "%Y-%m-%d").date(),
+ "12.0": datetime.datetime.strptime("2018-10-01", "%Y-%m-%d").date(),
+ "13.0": datetime.datetime.strptime("2019-10-01", "%Y-%m-%d").date(),
+ "14.0": datetime.datetime.strptime("2020-10-01", "%Y-%m-%d").date(),
+ "15.0": datetime.datetime.strptime("2021-10-01", "%Y-%m-%d").date(),
+ "16.0": datetime.datetime.strptime("2022-10-01", "%Y-%m-%d").date(),
+}
+CSV_HEADER_MODULE_NAME = "Nom technique"
+# TODO use rich progress bar when clone
+# TODO support conflict repo name
+
+
+def get_config():
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ description="""\
+ Clone all OCA repo from file 'source_repo_addons.csv', checkout branch from param --branches
+""",
+ epilog="""\
+""",
+ )
+ parser.add_argument(
+ "--clean_all",
+ action="store_true",
+ help=f"Wipe all data from path '{CACHE_CLONE_PATH}'",
+ )
+ parser.add_argument(
+ "--verbose",
+ action="store_true",
+ help=f"More information",
+ )
+ parser.add_argument(
+ "--debug",
+ action="store_true",
+ help="Enable asyncio debugging",
+ )
+ parser.add_argument(
+ "--ignore_release_date",
+ action="store_true",
+ help=(
+ f"Give result without release date logic, this means it gives"
+ f" result in past of release version."
+ ),
+ )
+ parser.add_argument(
+ "--no_parallel",
+ action="store_true",
+ help=(
+ f"By default run in parallel with all CPU, this disable"
+ f" parallelism with asyncio"
+ ),
+ )
+ parser.add_argument(
+ "--force_git_fetch",
+ action="store_true",
+ help=f"Force git fetch to update it all remote.",
+ )
+ parser.add_argument(
+ "-p",
+ "--max_process",
+ type=int,
+ default=0,
+ help="Max processor to use. If 0, use max.",
+ )
+ parser.add_argument(
+ "--history_length",
+ default=0,
+ type=int,
+ help=(
+ f"Set to 0 to clone all history, default 0. Can be another number."
+ f" This broke other version."
+ ),
+ )
+ parser.add_argument(
+ "--more_year",
+ default=0,
+ type=int,
+ help=(
+ f"At 0, search only present before_date, > 0, add year to"
+ f" before_date for multiple stat. Depend on param 'before_date'"
+ ),
+ )
+ parser.add_argument(
+ "--before_date",
+ type=lambda d: datetime.datetime.strptime(d, "%Y-%m-%d").date(),
+ default=None,
+ help=f"Set a check date before, example '2020-01-01'",
+ )
+ parser.add_argument(
+ "--filter",
+ default="",
+ help=f"keyword to ignore separate by ','. Suggest /OCA/",
+ )
+ parser.add_argument(
+ "-b",
+ "--branches",
+ default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0",
+ help="Branch to analyse, separate by ','",
+ )
+ parser.add_argument(
+ "--compare_csv",
+ help=(
+ "Path of CSV, search column 'Nom technique' and give list of"
+ " difference module, missing from CSV. Need only 1 branches and no"
+ " more_year."
+ ),
+ )
+ args = parser.parse_args()
+
+ die(
+ args.history_length < 0,
+ f"Argument 'history_length' need to be positive",
+ )
+ die(
+ args.more_year < 0,
+ f"Argument 'more_year' need to be positive",
+ )
+ die(
+ args.before_date is None and args.more_year > 0,
+ f"Cannot specify more_year when before_date is not set.",
+ )
+ # clean parameter
+ args.branches = ",".join([a for a in args.branches.split(",") if a])
+ die(
+ any(
+ [
+ a
+ for a in args.branches.split(",")
+ if a not in DCT_VERSION_RELEASE.keys()
+ ]
+ ),
+ "The params branches contain not supported version, check"
+ f" DCT_VERSION_RELEASE : {list(DCT_VERSION_RELEASE.keys())}",
+ )
+ die(
+ args.compare_csv
+ and (args.more_year > 0 or len(args.branches.split(",")) > 1),
+ f"When use param compare_csv, cannot use more_year and only 1 branch.",
+ )
+ if args.compare_csv:
+ die(
+ not os.path.isfile(args.compare_csv),
+ f"Path of {args.compare_csv} need to be a file path.",
+ )
+ return args
+
+
+def main():
+ config = get_config()
+ lst_branch = config.branches.split(",")
+ before_date = config.before_date
+ lst_unique_module = set()
+ lst_unique_uninstallable_module = set()
+
+ # Check
+ die(
+ not os.path.isdir(VENV_PATH),
+ f"Missing {VENV_PATH} venv path, did you install ERPLibre?",
+ )
+
+ # Setup environment
+ _logger.info(f"Setup {CACHE_CLONE_PATH}")
+ if config.clean_all and os.path.isdir(CACHE_CLONE_PATH):
+ _logger.info(f"Delete directory {CACHE_CLONE_PATH}")
+ shutil.rmtree(CACHE_CLONE_PATH)
+
+ if not os.path.isdir(CACHE_CLONE_PATH):
+ _logger.info(f"Create directory {CACHE_CLONE_PATH}")
+ os.mkdir(CACHE_CLONE_PATH)
+
+ # Get list of Repo
+ lst_repo_url = get_OCA_repo_list(config)
+
+ # Clone all repo
+ _logger.info(f"Clone repo")
+ lst_task_clone = [
+ clone_repo(i, a, config.force_git_fetch)
+ for i, a in enumerate(lst_repo_url)
+ ]
+ lst_repo_path = lib_asyncio.execute(
+ config, lst_task_clone, use_uvloop=True
+ )
+
+ # Extract module list
+ _logger.info("Extract module")
+ lst_task = []
+ for nb_year in range(config.more_year + 1):
+ # key is odoo version, value is set of module name
+ lst_branch_adapt = lst_branch
+ from_date = before_date
+ if before_date:
+ from_date += relativedelta(years=nb_year)
+ if not config.ignore_release_date:
+ lst_branch_adapt = get_branches_supported_date(
+ lst_branch, from_date
+ )
+
+ # for branch in lst_branch_adapt:
+ for repo_name, repo_path in lst_repo_path:
+ lst_task.append(
+ extract_module(
+ lst_branch_adapt, repo_name, repo_path, from_date
+ )
+ )
+ if config.debug:
+ _logger.info(
+ "Setup extract module:"
+ f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}"
+ )
+
+ _logger.info(f"Execute extract module with {len(lst_task)} coroutine.")
+ # Need to set a max, or crash
+ if len(lst_task) / MAX_COROUTINE > 1:
+ lst_result = []
+ for i in range(int(len(lst_task) / MAX_COROUTINE) + 1):
+ min_i = i * MAX_COROUTINE
+ max_i = min((i + 1) * MAX_COROUTINE, len(lst_task))
+ _logger.info(f"Partial execution {min_i} to {max_i}")
+ tpl_result = lib_asyncio.execute(
+ config, lst_task[min_i:max_i], use_uvloop=True
+ )
+ lst_result += tpl_result
+ tpl_result = lst_result
+ else:
+ tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True)
+ _logger.info("Analyse information")
+ dct_result = defaultdict(lambda: defaultdict(int))
+ dct_result_unique = defaultdict(set)
+ for lst_result_stack in tpl_result:
+ for dct_result_module in lst_result_stack:
+ # lst_module, branch, path, repo, before_date
+ before_date = dct_result_module.get("before_date")
+ lst_module = dct_result_module.get("lst_module")
+ lst_uninstallable_module = dct_result_module.get(
+ "lst_uninstallable_module"
+ )
+ branch = dct_result_module.get("branch")
+ # path = dct_result_module.get("path")
+ # repo = dct_result_module.get("repo")
+ dct_result[before_date][branch] += len(lst_module)
+ dct_result_unique[before_date].update(lst_module)
+ lst_unique_module.update(lst_module)
+ lst_unique_uninstallable_module.update(lst_uninstallable_module)
+
+ # Show result
+ _logger.info("Show result")
+ for before_date, dct_branch_result in dct_result.items():
+ str_extra = ""
+ if before_date:
+ str_extra = f" before {before_date}"
+ print(f"Stat nb module by branch name{str_extra}")
+ for branch_name, count_module in dct_branch_result.items():
+ print(f"{branch_name}\t{count_module}")
+ for before_date, set_result in dct_result_unique.items():
+ str_extra = ""
+ if before_date:
+ str_extra = f" before {before_date}"
+ print(f"Diff nb unique module{str_extra}, length {len(set_result)}")
+ print("end of stat")
+
+ if config.compare_csv:
+ index_header = -1
+ lst_unique_module_csv = set()
+ with open(config.compare_csv, "r") as csvfile:
+ for csv_module in csv.reader(csvfile):
+ if index_header == -1:
+ try:
+ index_header = csv_module.index(CSV_HEADER_MODULE_NAME)
+ except Exception as e:
+ _logger.error(
+ f"Missing header {CSV_HEADER_MODULE_NAME} in CSV."
+ )
+ raise e
+ else:
+ module_name = csv_module[index_header]
+ if module_name not in lst_unique_uninstallable_module:
+ lst_unique_module_csv.add(module_name)
+
+ lst_missing_module_repo = lst_unique_module_csv.difference(
+ lst_unique_module
+ )
+ lst_missing_module_csv = lst_unique_module.difference(
+ lst_unique_module_csv
+ )
+ print("CSV compare")
+ print(f"Module missing from CSV '{len(lst_missing_module_repo)}':")
+ print(lst_missing_module_repo)
+ print(f"Module missing into CSV '{len(lst_missing_module_csv)}':")
+ print(lst_missing_module_csv)
+
+
+def die(cond, message, code=1):
+ if cond:
+ print(message, file=sys.stderr)
+ sys.exit(code)
+
+
+def get_branches_supported_date(lst_branch, from_date):
+ return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date]
+
+
+async def extract_module(lst_branch, repo, path, before_date):
+ # print(f"{branch} - {path} - {before_date}")
+ lst_result = []
+ for branch in lst_branch:
+ return_value = {
+ "lst_module": [],
+ "branch": branch,
+ "path": path,
+ "repo": repo,
+ "before_date": before_date,
+ }
+ remote_branch_name = f"origin/{branch}"
+
+ # Detect commit to check
+ # git rev-list -n 1 --before 2022-01-01 origin/12.0
+ lst_args = ["git", "rev-list", "-n", "1"]
+ if before_date:
+ lst_args += ["--before", str(before_date)]
+ lst_args += [remote_branch_name]
+ (
+ commit,
+ _,
+ status,
+ ) = await lib_asyncio.run_command_get_output_and_status(
+ *lst_args, cwd=path
+ )
+
+ if status:
+ # The branch doesn't exist
+ # _logger.error(f"Receive status 'git rev-list' {status}")
+ continue
+
+ # TODO send msg when no result
+ if not commit:
+ continue
+
+ commit = commit.strip()
+
+ lst_module_installable = []
+ lst_module_uninstallable = []
+ # List directory
+ lst_suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "")
+ if type(lst_suffix_path) is str:
+ lst_suffix_path = [lst_suffix_path]
+ for suffix_path in lst_suffix_path:
+ adapt_path = path + suffix_path
+ lst_args = ["git", "ls-tree", "-d", "--name-only", commit]
+ (
+ str_folders,
+ _,
+ status,
+ ) = await lib_asyncio.run_command_get_output_and_status(
+ *lst_args, cwd=adapt_path
+ )
+ if status:
+ _logger.error(f"Receive status 'git ls-tree' {status}")
+ continue
+
+ # TODO send msg when no result
+ if not str_folders:
+ continue
+ lst_module = str_folders.split()
+ if not lst_module:
+ continue
+
+ if suffix_path:
+ suffix_path = suffix_path[1:]
+
+ for module in lst_module:
+ # List directory
+ lst_args = [
+ "git",
+ "cat-file",
+ "-p",
+ f"{commit}:{suffix_path}{module}/__manifest__.py",
+ ]
+ (
+ manifest,
+ _,
+ status,
+ ) = await lib_asyncio.run_command_get_output_and_status(
+ *lst_args, cwd=path
+ )
+ if status:
+ # Maybe __openerp__.py
+ lst_args = [
+ "git",
+ "cat-file",
+ "-p",
+ f"{commit}:{module}/__openerp__.py",
+ ]
+ (
+ manifest,
+ _,
+ status,
+ ) = await lib_asyncio.run_command_get_output_and_status(
+ *lst_args, cwd=adapt_path
+ )
+ if status:
+ # Ignore, it's not a module
+ # _logger.error(f"Receive status 'git cat-file' {status}")
+ # return return_value
+ continue
+ dct_manifest = eval(manifest + "\n")
+ if dct_manifest.get("installable", True):
+ lst_module_installable.append(module)
+ else:
+ lst_module_uninstallable.append(module)
+
+ return_value["lst_module"] = lst_module_installable
+ return_value["lst_uninstallable_module"] = lst_module_uninstallable
+ lst_result.append(return_value)
+ return lst_result
+
+
+async def clone_repo(i, repo_url, force_fetch):
+ split_repo_url = repo_url.split("/")
+ repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}"
+ repo_path = os.path.join(CACHE_CLONE_PATH, repo_name)
+ if not os.path.isdir(repo_path):
+ await lib_asyncio.run_command_get_output(
+ "git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH
+ )
+ _logger.info(f"[{i}]New clone {repo_path} with {repo_url}")
+ elif force_fetch:
+ await lib_asyncio.run_command_get_output(
+ "git", "fetch", "--all", cwd=CACHE_CLONE_PATH
+ )
+ _logger.info(f"[{i}]Fetch {repo_path} with {repo_url}")
+
+ return repo_name, repo_path
+
+
+def get_OCA_repo_list(config):
+ with open(CST_FILE_SOURCE_REPO_ADDONS) as file:
+ all_lines = file.readlines()
+ if all_lines:
+ # Validate first line is supported column
+ expected_header = "url,path,revision,clone-depth\n"
+ if all_lines[0] != expected_header:
+ raise Exception(
+ "Not supported csv, please validate"
+ f" {CST_FILE_SOURCE_REPO_ADDONS} with first line"
+ f" {expected_header}"
+ )
+ # Ignore first line
+ all_lines = all_lines[1:]
+
+ # Be sure empty endline at the end of file
+ if all_lines[-1][-1] != "\n":
+ all_lines[-1] = all_lines[-1] + "\n"
+ all_lines = [a.split(",")[0] for a in all_lines]
+ # TODO support split(",")
+ if config.filter:
+ all_lines = [
+ a
+ for a in all_lines
+ if config.filter in a and a not in IGNORE_REPO_LIST
+ ]
+ else:
+ all_lines = [a for a in all_lines if a not in IGNORE_REPO_LIST]
+ return all_lines
+
+
+if __name__ == "__main__":
+ main()
diff --git a/script/test/run_parallel_test.py b/script/test/run_parallel_test.py
index dd999a7..7891dc1 100755
--- a/script/test/run_parallel_test.py
+++ b/script/test/run_parallel_test.py
@@ -9,13 +9,19 @@ import sys
import tempfile
import time
import uuid
-from collections import deque
from typing import Tuple
import aioshutil
import git
from colorama import Fore
+new_path = os.path.normpath(
+ os.path.join(os.path.dirname(__file__), "..", "..")
+)
+sys.path.append(new_path)
+
+from script import lib_asyncio
+
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(__name__)
@@ -172,27 +178,6 @@ def print_log(lst_task, tpl_result):
f.write("\n")
-async def run_command_get_output(*args, cwd=None):
- if cwd is not None:
- process = await asyncio.create_subprocess_exec(
- *args,
- # stdout must a pipe to be accessible as process.stdout
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- cwd=cwd,
- )
- else:
- process = await asyncio.create_subprocess_exec(
- *args,
- # stdout must a pipe to be accessible as process.stdout
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- # Wait for the subprocess to finish
- stdout, stderr = await process.communicate()
- return stdout.decode()
-
-
async def run_command(*args, test_name=None):
# Create subprocess
start_time = time.time()
@@ -378,8 +363,10 @@ async def test_exec(
lst_module_to_test = tested_module.split(",")
for module_name in lst_module_to_test:
# Update path to change new emplacement
- s_lst_path_tested_module = await run_command_get_output(
- "find", ".", "-name", module_name
+ s_lst_path_tested_module = (
+ await lib_asyncio.run_command_get_output(
+ "find", "./addons/", "-name", module_name
+ )
)
if not s_lst_path_tested_module:
return (
@@ -420,12 +407,14 @@ async def test_exec(
os.path.join(temp_dir_name, s_first_path)
)
- s_lst_path_generated_module = await run_command_get_output(
- "find",
- ".",
- "-name",
- generated_module,
- cwd=temp_dir_name,
+ s_lst_path_generated_module = (
+ await lib_asyncio.run_command_get_output(
+ "find",
+ "./addons/",
+ "-name",
+ generated_module,
+ cwd=temp_dir_name,
+ )
)
if s_lst_path_generated_module:
lst_path_generated_module = (
@@ -507,15 +496,19 @@ async def test_exec(
hook.write(new_hook_line)
# Format editing code before commit
- await run_command_get_output(
+ await lib_asyncio.run_command_get_output(
"./script/maintenance/black.sh", temp_dir_name
)
# init repo with git
for dir_to_git in lst_path_to_add_config:
- await run_command_get_output("git", "init", ".", cwd=dir_to_git)
- await run_command_get_output("git", "add", ".", cwd=dir_to_git)
- await run_command_get_output(
+ await lib_asyncio.run_command_get_output(
+ "git", "init", ".", cwd=dir_to_git
+ )
+ await lib_asyncio.run_command_get_output(
+ "git", "add", ".", cwd=dir_to_git
+ )
+ await lib_asyncio.run_command_get_output(
"git", "commit", "-am", "'first commit'", cwd=dir_to_git
)
@@ -714,11 +707,6 @@ def check_git_change():
return not status
-def print_summary_task(task_list):
- for task in task_list:
- print(task.cr_code.co_name)
-
-
# START TEST
@@ -1115,6 +1103,29 @@ async def run_code_generator_template_demo_portal_test(
return test_result, test_status
+async def run_code_generator_template_demo_internal_test(
+ config,
+) -> Tuple[str, int]:
+ test_result = ""
+ test_status = 0
+ # Template
+ res, status = await test_exec(
+ config,
+ "./addons/TechnoLibre_odoo-code-generator-template",
+ generated_module="code_generator_demo_internal",
+ tested_module="code_generator_template_demo_internal",
+ search_class_module="demo_internal",
+ lst_init_module_name=[
+ "demo_internal",
+ ],
+ test_name="code_generator_template_demo_internal",
+ run_in_sandbox=True,
+ )
+ test_result += res
+ test_status += status
+
+ return test_result, test_status
+
async def run_code_generator_template_demo_internal_inherit_test(
config,
) -> Tuple[str, int]:
@@ -1173,75 +1184,6 @@ async def run_helloworld_test(config) -> Tuple[str, int]:
return res, status
-async def run_in_serial(task_list):
- q = asyncio.Queue()
- for task in task_list:
- await q.put(task)
- lst_result = []
- for i in range(len(task_list)):
- co = await q.get()
- result = await co
- lst_result.append(result)
- return lst_result
-
-
-class AsyncioPool:
- # TODO check to replace this pool by https://github.com/dano/aioprocessing
- def __init__(self, concurrency, loop=None):
- """
- @param loop: asyncio loop
- @param concurrency: Maximum number of concurrently running tasks
- """
- self._loop = loop or asyncio.get_event_loop()
- self._concurrency = concurrency
- self._coros = deque([]) # All coroutines queued for execution
- self._futures = [] # All currently running coroutines
- self._lst_result = []
-
- def close(self):
- self._loop.close()
-
- def add_coro(self, coro):
- """
- @param coro: coroutine to add
- """
- self._coros.append(coro)
- self.print_status()
-
- def run_until_complete(self):
- self._loop.run_until_complete(self._wait_for_futures())
- return self._lst_result
-
- def print_status(self):
- print(
- " Status: coros:%s - futures:%s"
- % (len(self._coros), len(self._futures))
- )
-
- def _start_futures(self):
- num_to_start = self._concurrency - len(self._futures)
- num_to_start = min(num_to_start, len(self._coros))
- for _ in range(num_to_start):
- coro = self._coros.popleft()
- future = asyncio.ensure_future(coro, loop=self._loop)
- self._futures.append(future)
- self.print_status()
-
- async def _wait_for_futures(self):
- while len(self._coros) > 0 or len(self._futures) > 0:
- self._start_futures()
- futures_completed, futures_pending = await asyncio.wait(
- self._futures,
- loop=self._loop,
- return_when=asyncio.FIRST_COMPLETED,
- )
-
- for future in futures_completed:
- self._lst_result.append(future.result())
- self._futures.remove(future)
- self._start_futures()
-
-
def run_all_test(config) -> None:
# low in time, at the end for more speed
task_list = [
@@ -1250,6 +1192,7 @@ def run_all_test(config) -> None:
run_code_generator_demo_mariadb_sql_example_1_test(config),
run_code_generator_auto_backup_test(config),
run_code_generator_template_demo_portal_test(config),
+ run_code_generator_template_demo_internal_test(config),
run_code_generator_template_demo_internal_inherit_test(config),
run_code_generator_template_demo_sysadmin_cron_test(config),
run_code_generator_demo_test(config),
@@ -1268,31 +1211,10 @@ def run_all_test(config) -> None:
run_demo_test(config),
]
- print_summary_task(task_list)
+ lib_asyncio.print_summary_task(task_list)
- if asyncio.get_event_loop().is_closed():
- asyncio.set_event_loop(asyncio.new_event_loop())
+ tpl_result = lib_asyncio.execute(config, task_list)
- if config.no_parallel:
- tpl_result = asyncio.run(run_in_serial(task_list))
- elif config.max_process:
- pool = AsyncioPool(config.max_process)
- for task in task_list:
- pool.add_coro(task)
- try:
- tpl_result = pool.run_until_complete()
- finally:
- pool.close()
- else:
- # Use maximal resource
- loop = asyncio.get_event_loop()
- if config.debug:
- loop.set_debug(True)
- try:
- commands = asyncio.gather(*task_list)
- tpl_result = loop.run_until_complete(commands)
- finally:
- loop.close()
print_log(task_list, tpl_result)
status = check_result(task_list, tpl_result)
if status:
diff --git a/test/code_generator/coverage_hello_world.sh b/test/code_generator/coverage_hello_world.sh
index 4b4bda2..883facf 100755
--- a/test/code_generator/coverage_hello_world.sh
+++ b/test/code_generator/coverage_hello_world.sh
@@ -6,7 +6,7 @@ Green='\033[0;32m' # Green
tmp_dir=$(mktemp -d -t "erplibre_code_generator_helloworld-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX")
-./script/code_generator/new_project.py -m test -d "${tmp_dir}"
+./script/code_generator/new_project.py -m test -d "${tmp_dir}" --coverage
MODULE_PATH="${tmp_dir}/test"
if [[ -d "${MODULE_PATH}" ]]; then