mathben/erplibre
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge branch 'develop'

Browse source 
- add statistic code tools
- fix test validate no change in addons for run_parallel_test.py
- upd osx installation
- fix test coverage
- missing test internal
- and more
This commit is contained in:
Mathieu Benoit 2023-04-05 00:03:52 -04:00
commit f3525e72ed
16 changed files with 1022 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/FUNDING.yml vendored
View file

@ -1,6 +1,6 @@
# These are supported funding model platforms # These are supported funding model platforms
github: mathben963 # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] github: mathben # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username ko_fi: # Replace with a single Ko-fi username

1
.gitignore vendored
View file

@ -9,6 +9,7 @@ config.conf
*.DS_Store *.DS_Store
*.pyc *.pyc
.coverage* .coverage*
coverage*
.repo .repo
!addons/ !addons/

53
Makefile
View file

@ -616,12 +616,27 @@ test_full_fast_coverage:
./.venv/bin/coverage erase ./.venv/bin/coverage erase
./script/test/run_parallel_test.py --coverage ./script/test/run_parallel_test.py --coverage
# TODO This test is broken in parallel # TODO This test is broken in parallel
./script/make.sh test_code_generator_hello_world ./script/make.sh test_coverage_code_generator_hello_world
./.venv/bin/coverage combine -a ./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage # run: make open_test_coverage
.PHONY: test_cg_demo
test_cg_demo:
./script/make.sh clean
# Need to create a BD to create cache _cache_erplibre_base
./script/database/db_restore.py --database test
./.venv/bin/coverage erase
./script/addons/coverage_install_addons_dev.sh test code_generator_demo
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
.PHONY: test_base .PHONY: test_base
test_base: test_base:
./script/make.sh test_format ./script/make.sh test_format
@ -645,6 +660,10 @@ test_format:
test_code_generator_hello_world: test_code_generator_hello_world:
./test/code_generator/hello_world.sh ./test/code_generator/hello_world.sh
.PHONY: test_coverage_code_generator_hello_world
test_coverage_code_generator_hello_world:
./test/code_generator/coverage_hello_world.sh
.PHONY: test_installation_demo .PHONY: test_installation_demo
test_installation_demo: test_installation_demo:
./script/code_generator/check_git_change_code_generator.sh ./addons/TechnoLibre_odoo-code-generator-template ./script/code_generator/check_git_change_code_generator.sh ./addons/TechnoLibre_odoo-code-generator-template
@ -752,6 +771,7 @@ test_addons_sale:
./.venv/bin/coverage combine -a ./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m ./.venv/bin/coverage report -m
./.venv/bin/coverage html ./.venv/bin/coverage html
./.venv/bin/coverage json
.PHONY: test_addons_helpdesk .PHONY: test_addons_helpdesk
test_addons_helpdesk: test_addons_helpdesk:
@ -761,16 +781,18 @@ test_addons_helpdesk:
./.venv/bin/coverage combine -a ./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m ./.venv/bin/coverage report -m
./.venv/bin/coverage html ./.venv/bin/coverage html
./.venv/bin/coverage json
.PHONY: test_addons_code_generator .PHONY: test_addons_code_generator
test_addons_code_generator: test_addons_code_generator:
./.venv/bin/coverage erase ./.venv/bin/coverage erase
./.venv/bin/python3 ./odoo/odoo-bin db --drop --database test_addons_code_generator ./.venv/bin/python3 ./odoo/odoo-bin db --drop --database test_addons_code_generator
# TODO missing test in code_generator # TODO missing test in code_generator
./test.sh --dev all -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator ./test.sh --dev cg -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator
./.venv/bin/coverage combine -a ./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage # run: make open_test_coverage
.PHONY: test_addons_code_generator_code_generator .PHONY: test_addons_code_generator_code_generator
@ -778,10 +800,24 @@ test_addons_code_generator_code_generator:
# TODO this test only generation, not test # TODO this test only generation, not test
./.venv/bin/coverage erase ./.venv/bin/coverage erase
./script/database/db_restore.py --database test_addons_code_generator_code_generator ./script/database/db_restore.py --database test_addons_code_generator_code_generator
./test.sh --dev all -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator ./test.sh --dev cg -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator
./.venv/bin/coverage combine -a ./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage
.PHONY: test_addons_code_generator_template_code_generator
test_addons_code_generator_template_code_generator:
# TODO this test only generation, not test
./.venv/bin/coverage erase
./script/database/db_restore.py --database test_addons_code_generator_template_code_generator
./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator
./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator_template_code_generator
./.venv/bin/coverage combine -a
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
# run: make open_test_coverage # run: make open_test_coverage
.PHONY: open_test_coverage .PHONY: open_test_coverage
@ -1037,6 +1073,17 @@ i18n_generate_demo_portal:
clean: clean:
find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
###############
# Statistic #
###############
.PHONY: stat_module_evolution_per_year
stat_module_evolution_per_year:
./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7
.PHONY: stat_module_evolution_per_year_OCA
stat_module_evolution_per_year_OCA:
./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7
################### ###################
# Documentation # # Documentation #
################### ###################

16
doc/FAQ.md
View file

@ -124,3 +124,19 @@ docker-machine create --driver virtualbox default
docker-machine env default docker-machine env default
eval "$(docker-machine env default)" eval "$(docker-machine env default)"
``` ```
### Run error `current limit exceeds maximum limit`
When you got
```
File "/Users/test/Desktop/ERPLibre/odoo/odoo/service/server.py", line 83, in set_limit_memory_hard
resource.setrlimit(rlimit, (config['limit_memory_hard'], hard))
ValueError: current limit exceeds maximum limit
```
Add line at the end of config.conf
```
limit_memory_hard = 0
```

2
manifest/default.dev.xml
View file

@ -93,7 +93,7 @@
<project name="muk_docs.git" path="addons/muk-it_muk_docs" remote="muk-it" groups="addons"/> <project name="muk_docs.git" path="addons/muk-it_muk_docs" remote="muk-it" groups="addons"/>
<project name="muk_misc.git" path="addons/muk-it_muk_misc" remote="muk-it" revision="12.0_dev" groups="addons,base"/> <project name="muk_misc.git" path="addons/muk-it_muk_misc" remote="muk-it" revision="12.0_dev" groups="addons,base"/>
<project name="muk_quality.git" path="addons/muk-it_muk_quality" remote="muk-it" groups="addons"/> <project name="muk_quality.git" path="addons/muk-it_muk_quality" remote="muk-it" groups="addons"/>
<project name="muk_web.git" path="addons/muk-it_muk_web" remote="muk-it" revision="12.0_dev" groups="addons,base"/> <project name="muk_web.git" path="addons/muk-it_muk_web" remote="muk-it" revision="12.0_dev" groups="addons,base,code_generator"/>
<project name="muk_website.git" path="addons/muk-it_muk_website" remote="muk-it" revision="12.0_remove_autoinstall" groups="addons,base"/> <project name="muk_website.git" path="addons/muk-it_muk_website" remote="muk-it" revision="12.0_remove_autoinstall" groups="addons,base"/>
<project name="multi-company.git" path="addons/OCA_multi-company" remote="OCA" groups="addons"/> <project name="multi-company.git" path="addons/OCA_multi-company" remote="OCA" groups="addons"/>
<project name="odoo-addons.git" path="addons/ajepe_odoo-addons" remote="ajepe" groups="addons"/> <project name="odoo-addons.git" path="addons/ajepe_odoo-addons" remote="ajepe" groups="addons"/>

68
poetry.lock generated
View file

@ -5210,6 +5210,27 @@ files = [
{file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"},
] ]
[[package]]
name = "tqdm"
version = "4.64.1"
description = "Fast, Extensible Progress Meter"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
files = [
{file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"},
{file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"},
]
[package.dependencies]
colorama = {version = "*", markers = "platform_system == \"Windows\""}
[package.extras]
dev = ["py-make (>=0.1.0)", "twine", "wheel"]
notebook = ["ipywidgets (>=6)"]
slack = ["slack-sdk"]
telegram = ["requests"]
[[package]] [[package]]
name = "typed-ast" name = "typed-ast"
version = "1.5.4" version = "1.5.4"
@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "uvloop"
version = "0.17.0"
description = "Fast implementation of asyncio event loop on top of libuv"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"},
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"},
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"},
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"},
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"},
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"},
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"},
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"},
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"},
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"},
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"},
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"},
{file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"},
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"},
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"},
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"},
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"},
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"},
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"},
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"},
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"},
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"},
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"},
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"},
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"},
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"},
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"},
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"},
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"},
{file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"},
]
[package.extras]
dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"]
[[package]] [[package]]
name = "validators" name = "validators"
version = "0.20.0" version = "0.20.0"
@ -5879,4 +5945,4 @@ files = [
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.7.16" python-versions = "^3.7.16"
content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e" content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368"

2
pyproject.toml
View file

@ -185,6 +185,8 @@ xmltodict = "^0.13.0"
zeep = "^4.2.1" zeep = "^4.2.1"
zipp = ">=3.6.0" zipp = ">=3.6.0"
zxcvbn = "^4.4.28" zxcvbn = "^4.4.28"
tqdm = "^4.64.1"
uvloop = "^0.17.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
websocket_client = "^1.1.0" websocket_client = "^1.1.0"

5
requirements.txt
View file

@ -8,7 +8,6 @@ openupgradelib
unidiff unidiff
colorama colorama
wheel wheel
aioshutil<1.2
# For OSX # For OSX
cython cython
@ -80,6 +79,10 @@ numpy==1.21.1
feedparser==6.0.10 feedparser==6.0.10
geojson<3.0.0 geojson<3.0.0
# asyncio
aioshutil<1.2
tqdm
uvloop
# Solve pcodedmp, oletools, rtfde and extract-msg # Solve pcodedmp, oletools, rtfde and extract-msg
win_unicode_console win_unicode_console

2
script/addons/update_prod_to_dev.sh
View file

@ -2,7 +2,7 @@
# This script will remove mail configuration, remove backup configuration, and force admin user to test/test # This script will remove mail configuration, remove backup configuration, and force admin user to test/test
echo "Update prod to dev on BD '$1'" echo "Update prod to dev on BD '$1'"
./script/addons/install_addons.sh "$1" user_test,disable_mail_server,disable_auto_backup ./script/addons/install_addons_dev.sh "$1" user_test,disable_mail_server,disable_auto_backup
retVal=$? retVal=$?
if [[ $retVal -ne 0 ]]; then if [[ $retVal -ne 0 ]]; then

149
script/code_generator/technical/transform_xml_data_website_page_to_controller.py Normal file
View file

@ -0,0 +1,149 @@
#!./.venv/bin/python
import argparse
import logging
import os
import sys
import black
import xmltodict
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(__name__)
def get_config():
"""Parse command line arguments, extracting the config file name,
returning the union of config file and command line arguments
:return: dict of config file settings and command line arguments
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""\
Transform an XML into Python
""",
epilog="""\
""",
)
parser.add_argument(
"--source",
dest="source",
required=True,
help="File to transform (XML)",
)
# parser.add_argument(
# "--destination",
# dest="destination",
# help=(
# "Result, output (Python). If not set, output will be console."
# ),
# )
# parser.add_argument(
# "-q",
# "--quiet",
# action="store_true",
# help="Don't show output of found model.",
# )
args = parser.parse_args()
if not os.path.isfile(args.source):
raise Exception("Argument --source need to be a valid path to a file.")
return args
def get_module_name(filepath):
manifest_file = "__manifest__.py"
current_path = os.path.abspath(os.path.dirname(filepath))
while current_path != "/":
manifest_path = os.path.join(current_path, manifest_file)
if os.path.isfile(manifest_path):
return os.path.basename(current_path)
current_path = os.path.dirname(current_path)
raise FileNotFoundError(f"No {manifest_file} found for {filepath}")
def get_value_from_field(lst_field, by_attr, name, record_id, get_attr):
field_info = [a for a in lst_field if a.get(by_attr) == name]
if not field_info:
raise ValueError(
f"Missing <field {by_attr[1:]}={name}'/> from record {record_id}"
)
field_info = field_info[0]
return field_info.get(get_attr)
def main():
config = get_config()
with open(config.source) as xml:
xml_as_string = xml.read()
xml_dict = xmltodict.parse(xml_as_string)
# TODO replace this by get_value(xml_dict, "odoo.data.record")
xml_dict_odoo = xml_dict.get("odoo")
if not xml_dict_odoo:
raise ValueError(f"Missing <odoo> into {config.source}")
xml_dict_data = xml_dict_odoo.get("data")
if not xml_dict_data:
raise ValueError(f"Missing <data> into {config.source}")
xml_dict_record = xml_dict_data.get("record")
if not xml_dict_record:
raise ValueError(f"Missing <record> into {config.source}")
if type(xml_dict_record) is not list:
lst_xml_record = [xml_dict_record]
else:
lst_xml_record = xml_dict_record
lst_output = []
for record in lst_xml_record:
lst_field = record.get("field")
if type(lst_field) is not list:
lst_field = [lst_field]
# Condition unique
try:
get_value_from_field(
lst_field, "@name", "groups_id", record.get("@id"), "@eval"
)
except:
continue
url = get_value_from_field(
lst_field, "@name", "url", record.get("@id"), "#text"
)
view_id = get_value_from_field(
lst_field, "@name", "view_id", record.get("@id"), "@ref"
)
module_name = get_module_name(config.source)
method_name = (
f'get_{"_".join(url.replace("-", "_").strip("/").split("/"))}'
)
# template = f"""
# @http.route(
# ["{url}"],
# type="http",
# auth="user",
# website=True,
# )
# def {method_name}(self, **kw):
# return request.env["ir.ui.view"].render_template(
# "{module_name}.{view_id}",
# )
template = f"""
@http.route(
["{url}"],
type="http",
auth="user",
website=True,
)
def {method_name}(self, **kw):
return request.env.ref('{module_name}.{view_id}').render()
"""
lst_output.append(template)
str_output = "\n".join(lst_output)
formated_str_output = black.format_str(str_output, mode=black.Mode())
print(formated_str_output)
return 0
if __name__ == "__main__":
status = main()
sys.exit(status)

10
script/install/install_OSX_dependency.sh
View file

@ -25,23 +25,21 @@ sudo su - postgres -c "CREATE EXTENSION postgis;\nCREATE EXTENSION postgis_topol
# Install Dependencies # Install Dependencies
#-------------------------------------------------- #--------------------------------------------------
echo "\n--- Installing Python 3 + pip3 --" echo "\n--- Installing Python 3 + pip3 --"
brew install git python3 wget pyenv brew install git python3 wget
brew link git brew link git
brew link wget brew link wget
echo "\n--- Installing extra --"
brew install parallel
echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----" echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----"
brew install nodejs npm openssl brew install nodejs npm openssl
sudo npm install -g rtlcss sudo npm install -g rtlcss
sudo npm install -g less sudo npm install -g less
sudo npm install -g prettier sudo npm install -g prettier
sudo npm install -g prettier @prettier/plugin-xml sudo npm install -g prettier @prettier/plugin-xml
yes n|pyenv install 3.7.16
pyenv local 3.7.16
echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.zshrc echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.zshrc
echo -e "\n---- Installing poetry for reliable python package ----"
curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
#-------------------------------------------------- #--------------------------------------------------
# Install Wkhtmltopdf if needed # Install Wkhtmltopdf if needed
#-------------------------------------------------- #--------------------------------------------------

152
script/lib_asyncio.py Normal file
View file

@ -0,0 +1,152 @@
#!./.venv/bin/python
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
import asyncio
from collections import deque
import uvloop
async def run_in_serial(task_list):
q = asyncio.Queue()
for task in task_list:
await q.put(task)
lst_result = []
for i in range(len(task_list)):
co = await q.get()
result = await co
lst_result.append(result)
return lst_result
async def run_command_get_output(*args, cwd=None):
if cwd is not None:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=cwd,
)
else:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
# Wait for the subprocess to finish
stdout, stderr = await process.communicate()
return stdout.decode()
async def run_command_get_output_and_status(*args, cwd=None):
if cwd is not None:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=cwd,
)
else:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
# Wait for the subprocess to finish
stdout, stderr = await process.communicate()
return stdout.decode(), stderr.decode(), process.returncode
def print_summary_task(task_list):
for task in task_list:
print(task.cr_code.co_name)
def execute(config, lst_task, use_uvloop=False):
if not config.no_parallel and asyncio.get_event_loop().is_closed():
asyncio.set_event_loop(asyncio.new_event_loop())
if config.no_parallel:
return_value = asyncio.run(run_in_serial(lst_task))
elif config.max_process:
pool = AsyncioPool(config.max_process)
for task in lst_task:
pool.add_coro(task)
try:
return_value = pool.run_until_complete()
finally:
pool.close()
else:
# Use maximal resource
if use_uvloop:
uvloop.install()
loop = asyncio.get_event_loop()
if config.debug:
loop.set_debug(True)
try:
commands = asyncio.gather(*lst_task)
return_value = loop.run_until_complete(commands)
finally:
loop.close()
return return_value
class AsyncioPool:
# TODO check to replace this pool by https://github.com/dano/aioprocessing
def __init__(self, concurrency, loop=None):
"""
@param loop: asyncio loop
@param concurrency: Maximum number of concurrently running tasks
"""
self._loop = loop or asyncio.get_event_loop()
self._concurrency = concurrency
self._coros = deque([]) # All coroutines queued for execution
self._futures = [] # All currently running coroutines
self._lst_result = []
def close(self):
self._loop.close()
def add_coro(self, coro):
"""
@param coro: coroutine to add
"""
self._coros.append(coro)
self.print_status()
def run_until_complete(self):
self._loop.run_until_complete(self._wait_for_futures())
return self._lst_result
def print_status(self):
print(
" Status: coros:%s - futures:%s"
% (len(self._coros), len(self._futures))
)
def _start_futures(self):
num_to_start = self._concurrency - len(self._futures)
num_to_start = min(num_to_start, len(self._coros))
for _ in range(num_to_start):
coro = self._coros.popleft()
future = asyncio.ensure_future(coro, loop=self._loop)
self._futures.append(future)
self.print_status()
async def _wait_for_futures(self):
while len(self._coros) > 0 or len(self._futures) > 0:
self._start_futures()
futures_completed, futures_pending = await asyncio.wait(
self._futures,
loop=self._loop,
return_when=asyncio.FIRST_COMPLETED,
)
for future in futures_completed:
self._lst_result.append(future.result())
self._futures.remove(future)
self._start_futures()

2
script/statistic/code_count.sh
View file

@ -1,4 +1,4 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# all argument is directory/file separate by space # all argument is directory/file separate by space
./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,css,scss --format=summary "$@" ./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,csv,css,scss --format=summary "$@"

519
script/statistic/show_evolution_module.py Executable file
View file

@ -0,0 +1,519 @@
#!./.venv/bin/python
import argparse
import datetime
import logging
import os
import shutil
import sys
import csv
from collections import defaultdict
from dateutil.relativedelta import relativedelta
new_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), "..", "..")
)
sys.path.append(new_path)
from script import lib_asyncio
FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT)
_logger = logging.getLogger(__name__)
PROJECT_NAME = os.path.basename(os.getcwd())
VENV_PATH = "./.venv"
CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo")
CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv"
MAX_COROUTINE = 300
IGNORE_REPO_LIST = (
"https://github.com/OCA/odoo-module-migrator.git",
"https://github.com/OCA/maintainer-tools.git",
"https://github.com/itpp-labs/odoo-development.git",
"https://github.com/itpp-labs/odoo-port-docs.git",
"https://github.com/itpp-labs/odoo-test-docs.git",
"https://github.com/odoo/documentation.git",
"https://github.com/ERPLibre/ERPLibre_image_db.git",
"https://github.com/muk-it/muk_docs.git",
)
DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": ["/addons/", "/odoo/addons/"]}
DCT_VERSION_RELEASE = {
"1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(),
"2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(),
"3.0": datetime.datetime.strptime("2005-09-01", "%Y-%m-%d").date(),
"4.0": datetime.datetime.strptime("2006-12-01", "%Y-%m-%d").date(),
"5.0": datetime.datetime.strptime("2009-04-01", "%Y-%m-%d").date(),
"6.0": datetime.datetime.strptime("2009-10-01", "%Y-%m-%d").date(),
"6.1": datetime.datetime.strptime("2012-02-27", "%Y-%m-%d").date(),
"7.0": datetime.datetime.strptime("2012-12-01", "%Y-%m-%d").date(),
"8.0": datetime.datetime.strptime("2014-09-01", "%Y-%m-%d").date(),
"9.0": datetime.datetime.strptime("2015-11-01", "%Y-%m-%d").date(),
"10.0": datetime.datetime.strptime("2016-10-01", "%Y-%m-%d").date(),
"11.0": datetime.datetime.strptime("2017-10-01", "%Y-%m-%d").date(),
"12.0": datetime.datetime.strptime("2018-10-01", "%Y-%m-%d").date(),
"13.0": datetime.datetime.strptime("2019-10-01", "%Y-%m-%d").date(),
"14.0": datetime.datetime.strptime("2020-10-01", "%Y-%m-%d").date(),
"15.0": datetime.datetime.strptime("2021-10-01", "%Y-%m-%d").date(),
"16.0": datetime.datetime.strptime("2022-10-01", "%Y-%m-%d").date(),
}
CSV_HEADER_MODULE_NAME = "Nom technique"
# TODO use rich progress bar when clone
# TODO support conflict repo name
def get_config():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""\
Clone all OCA repo from file 'source_repo_addons.csv', checkout branch from param --branches
""",
epilog="""\
""",
)
parser.add_argument(
"--clean_all",
action="store_true",
help=f"Wipe all data from path '{CACHE_CLONE_PATH}'",
)
parser.add_argument(
"--verbose",
action="store_true",
help=f"More information",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable asyncio debugging",
)
parser.add_argument(
"--ignore_release_date",
action="store_true",
help=(
f"Give result without release date logic, this means it gives"
f" result in past of release version."
),
)
parser.add_argument(
"--no_parallel",
action="store_true",
help=(
f"By default run in parallel with all CPU, this disable"
f" parallelism with asyncio"
),
)
parser.add_argument(
"--force_git_fetch",
action="store_true",
help=f"Force git fetch to update it all remote.",
)
parser.add_argument(
"-p",
"--max_process",
type=int,
default=0,
help="Max processor to use. If 0, use max.",
)
parser.add_argument(
"--history_length",
default=0,
type=int,
help=(
f"Set to 0 to clone all history, default 0. Can be another number."
f" This broke other version."
),
)
parser.add_argument(
"--more_year",
default=0,
type=int,
help=(
f"At 0, search only present before_date, > 0, add year to"
f" before_date for multiple stat. Depend on param 'before_date'"
),
)
parser.add_argument(
"--before_date",
type=lambda d: datetime.datetime.strptime(d, "%Y-%m-%d").date(),
default=None,
help=f"Set a check date before, example '2020-01-01'",
)
parser.add_argument(
"--filter",
default="",
help=f"keyword to ignore separate by ','. Suggest /OCA/",
)
parser.add_argument(
"-b",
"--branches",
default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0",
help="Branch to analyse, separate by ','",
)
parser.add_argument(
"--compare_csv",
help=(
"Path of CSV, search column 'Nom technique' and give list of"
" difference module, missing from CSV. Need only 1 branches and no"
" more_year."
),
)
args = parser.parse_args()
die(
args.history_length < 0,
f"Argument 'history_length' need to be positive",
)
die(
args.more_year < 0,
f"Argument 'more_year' need to be positive",
)
die(
args.before_date is None and args.more_year > 0,
f"Cannot specify more_year when before_date is not set.",
)
# clean parameter
args.branches = ",".join([a for a in args.branches.split(",") if a])
die(
any(
[
a
for a in args.branches.split(",")
if a not in DCT_VERSION_RELEASE.keys()
]
),
"The params branches contain not supported version, check"
f" DCT_VERSION_RELEASE : {list(DCT_VERSION_RELEASE.keys())}",
)
die(
args.compare_csv
and (args.more_year > 0 or len(args.branches.split(",")) > 1),
f"When use param compare_csv, cannot use more_year and only 1 branch.",
)
if args.compare_csv:
die(
not os.path.isfile(args.compare_csv),
f"Path of {args.compare_csv} need to be a file path.",
)
return args
def main():
config = get_config()
lst_branch = config.branches.split(",")
before_date = config.before_date
lst_unique_module = set()
lst_unique_uninstallable_module = set()
# Check
die(
not os.path.isdir(VENV_PATH),
f"Missing {VENV_PATH} venv path, did you install ERPLibre?",
)
# Setup environment
_logger.info(f"Setup {CACHE_CLONE_PATH}")
if config.clean_all and os.path.isdir(CACHE_CLONE_PATH):
_logger.info(f"Delete directory {CACHE_CLONE_PATH}")
shutil.rmtree(CACHE_CLONE_PATH)
if not os.path.isdir(CACHE_CLONE_PATH):
_logger.info(f"Create directory {CACHE_CLONE_PATH}")
os.mkdir(CACHE_CLONE_PATH)
# Get list of Repo
lst_repo_url = get_OCA_repo_list(config)
# Clone all repo
_logger.info(f"Clone repo")
lst_task_clone = [
clone_repo(i, a, config.force_git_fetch)
for i, a in enumerate(lst_repo_url)
]
lst_repo_path = lib_asyncio.execute(
config, lst_task_clone, use_uvloop=True
)
# Extract module list
_logger.info("Extract module")
lst_task = []
for nb_year in range(config.more_year + 1):
# key is odoo version, value is set of module name
lst_branch_adapt = lst_branch
from_date = before_date
if before_date:
from_date += relativedelta(years=nb_year)
if not config.ignore_release_date:
lst_branch_adapt = get_branches_supported_date(
lst_branch, from_date
)
# for branch in lst_branch_adapt:
for repo_name, repo_path in lst_repo_path:
lst_task.append(
extract_module(
lst_branch_adapt, repo_name, repo_path, from_date
)
)
if config.debug:
_logger.info(
"Setup extract module:"
f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}"
)
_logger.info(f"Execute extract module with {len(lst_task)} coroutine.")
# Need to set a max, or crash
if len(lst_task) / MAX_COROUTINE > 1:
lst_result = []
for i in range(int(len(lst_task) / MAX_COROUTINE) + 1):
min_i = i * MAX_COROUTINE
max_i = min((i + 1) * MAX_COROUTINE, len(lst_task))
_logger.info(f"Partial execution {min_i} to {max_i}")
tpl_result = lib_asyncio.execute(
config, lst_task[min_i:max_i], use_uvloop=True
)
lst_result += tpl_result
tpl_result = lst_result
else:
tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True)
_logger.info("Analyse information")
dct_result = defaultdict(lambda: defaultdict(int))
dct_result_unique = defaultdict(set)
for lst_result_stack in tpl_result:
for dct_result_module in lst_result_stack:
# lst_module, branch, path, repo, before_date
before_date = dct_result_module.get("before_date")
lst_module = dct_result_module.get("lst_module")
lst_uninstallable_module = dct_result_module.get(
"lst_uninstallable_module"
)
branch = dct_result_module.get("branch")
# path = dct_result_module.get("path")
# repo = dct_result_module.get("repo")
dct_result[before_date][branch] += len(lst_module)
dct_result_unique[before_date].update(lst_module)
lst_unique_module.update(lst_module)
lst_unique_uninstallable_module.update(lst_uninstallable_module)
# Show result
_logger.info("Show result")
for before_date, dct_branch_result in dct_result.items():
str_extra = ""
if before_date:
str_extra = f" before {before_date}"
print(f"Stat nb module by branch name{str_extra}")
for branch_name, count_module in dct_branch_result.items():
print(f"{branch_name}\t{count_module}")
for before_date, set_result in dct_result_unique.items():
str_extra = ""
if before_date:
str_extra = f" before {before_date}"
print(f"Diff nb unique module{str_extra}, length {len(set_result)}")
print("end of stat")
if config.compare_csv:
index_header = -1
lst_unique_module_csv = set()
with open(config.compare_csv, "r") as csvfile:
for csv_module in csv.reader(csvfile):
if index_header == -1:
try:
index_header = csv_module.index(CSV_HEADER_MODULE_NAME)
except Exception as e:
_logger.error(
f"Missing header {CSV_HEADER_MODULE_NAME} in CSV."
)
raise e
else:
module_name = csv_module[index_header]
if module_name not in lst_unique_uninstallable_module:
lst_unique_module_csv.add(module_name)
lst_missing_module_repo = lst_unique_module_csv.difference(
lst_unique_module
)
lst_missing_module_csv = lst_unique_module.difference(
lst_unique_module_csv
)
print("CSV compare")
print(f"Module missing from CSV '{len(lst_missing_module_repo)}':")
print(lst_missing_module_repo)
print(f"Module missing into CSV '{len(lst_missing_module_csv)}':")
print(lst_missing_module_csv)
def die(cond, message, code=1):
if cond:
print(message, file=sys.stderr)
sys.exit(code)
def get_branches_supported_date(lst_branch, from_date):
return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date]
async def extract_module(lst_branch, repo, path, before_date):
# print(f"{branch} - {path} - {before_date}")
lst_result = []
for branch in lst_branch:
return_value = {
"lst_module": [],
"branch": branch,
"path": path,
"repo": repo,
"before_date": before_date,
}
remote_branch_name = f"origin/{branch}"
# Detect commit to check
# git rev-list -n 1 --before 2022-01-01 origin/12.0
lst_args = ["git", "rev-list", "-n", "1"]
if before_date:
lst_args += ["--before", str(before_date)]
lst_args += [remote_branch_name]
(
commit,
_,
status,
) = await lib_asyncio.run_command_get_output_and_status(
*lst_args, cwd=path
)
if status:
# The branch doesn't exist
# _logger.error(f"Receive status 'git rev-list' {status}")
continue
# TODO send msg when no result
if not commit:
continue
commit = commit.strip()
lst_module_installable = []
lst_module_uninstallable = []
# List directory
lst_suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "")
if type(lst_suffix_path) is str:
lst_suffix_path = [lst_suffix_path]
for suffix_path in lst_suffix_path:
adapt_path = path + suffix_path
lst_args = ["git", "ls-tree", "-d", "--name-only", commit]
(
str_folders,
_,
status,
) = await lib_asyncio.run_command_get_output_and_status(
*lst_args, cwd=adapt_path
)
if status:
_logger.error(f"Receive status 'git ls-tree' {status}")
continue
# TODO send msg when no result
if not str_folders:
continue
lst_module = str_folders.split()
if not lst_module:
continue
if suffix_path:
suffix_path = suffix_path[1:]
for module in lst_module:
# List directory
lst_args = [
"git",
"cat-file",
"-p",
f"{commit}:{suffix_path}{module}/__manifest__.py",
]
(
manifest,
_,
status,
) = await lib_asyncio.run_command_get_output_and_status(
*lst_args, cwd=path
)
if status:
# Maybe __openerp__.py
lst_args = [
"git",
"cat-file",
"-p",
f"{commit}:{module}/__openerp__.py",
]
(
manifest,
_,
status,
) = await lib_asyncio.run_command_get_output_and_status(
*lst_args, cwd=adapt_path
)
if status:
# Ignore, it's not a module
# _logger.error(f"Receive status 'git cat-file' {status}")
# return return_value
continue
dct_manifest = eval(manifest + "\n")
if dct_manifest.get("installable", True):
lst_module_installable.append(module)
else:
lst_module_uninstallable.append(module)
return_value["lst_module"] = lst_module_installable
return_value["lst_uninstallable_module"] = lst_module_uninstallable
lst_result.append(return_value)
return lst_result
async def clone_repo(i, repo_url, force_fetch):
split_repo_url = repo_url.split("/")
repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}"
repo_path = os.path.join(CACHE_CLONE_PATH, repo_name)
if not os.path.isdir(repo_path):
await lib_asyncio.run_command_get_output(
"git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH
)
_logger.info(f"[{i}]New clone {repo_path} with {repo_url}")
elif force_fetch:
await lib_asyncio.run_command_get_output(
"git", "fetch", "--all", cwd=CACHE_CLONE_PATH
)
_logger.info(f"[{i}]Fetch {repo_path} with {repo_url}")
return repo_name, repo_path
def get_OCA_repo_list(config):
with open(CST_FILE_SOURCE_REPO_ADDONS) as file:
all_lines = file.readlines()
if all_lines:
# Validate first line is supported column
expected_header = "url,path,revision,clone-depth\n"
if all_lines[0] != expected_header:
raise Exception(
"Not supported csv, please validate"
f" {CST_FILE_SOURCE_REPO_ADDONS} with first line"
f" {expected_header}"
)
# Ignore first line
all_lines = all_lines[1:]
# Be sure empty endline at the end of file
if all_lines[-1][-1] != "\n":
all_lines[-1] = all_lines[-1] + "\n"
all_lines = [a.split(",")[0] for a in all_lines]
# TODO support split(",")
if config.filter:
all_lines = [
a
for a in all_lines
if config.filter in a and a not in IGNORE_REPO_LIST
]
else:
all_lines = [a for a in all_lines if a not in IGNORE_REPO_LIST]
return all_lines
if __name__ == "__main__":
main()

176
script/test/run_parallel_test.py
View file

@ -9,13 +9,19 @@ import sys
import tempfile import tempfile
import time import time
import uuid import uuid
from collections import deque
from typing import Tuple from typing import Tuple
import aioshutil import aioshutil
import git import git
from colorama import Fore from colorama import Fore
new_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), "..", "..")
)
sys.path.append(new_path)
from script import lib_asyncio
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
@ -172,27 +178,6 @@ def print_log(lst_task, tpl_result):
f.write("\n") f.write("\n")
async def run_command_get_output(*args, cwd=None):
if cwd is not None:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=cwd,
)
else:
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
# Wait for the subprocess to finish
stdout, stderr = await process.communicate()
return stdout.decode()
async def run_command(*args, test_name=None): async def run_command(*args, test_name=None):
# Create subprocess # Create subprocess
start_time = time.time() start_time = time.time()
@ -378,8 +363,10 @@ async def test_exec(
lst_module_to_test = tested_module.split(",") lst_module_to_test = tested_module.split(",")
for module_name in lst_module_to_test: for module_name in lst_module_to_test:
# Update path to change new emplacement # Update path to change new emplacement
s_lst_path_tested_module = await run_command_get_output( s_lst_path_tested_module = (
"find", ".", "-name", module_name await lib_asyncio.run_command_get_output(
"find", "./addons/", "-name", module_name
)
) )
if not s_lst_path_tested_module: if not s_lst_path_tested_module:
return ( return (
@ -420,13 +407,15 @@ async def test_exec(
os.path.join(temp_dir_name, s_first_path) os.path.join(temp_dir_name, s_first_path)
) )
s_lst_path_generated_module = await run_command_get_output( s_lst_path_generated_module = (
await lib_asyncio.run_command_get_output(
"find", "find",
".", "./addons/",
"-name", "-name",
generated_module, generated_module,
cwd=temp_dir_name, cwd=temp_dir_name,
) )
)
if s_lst_path_generated_module: if s_lst_path_generated_module:
lst_path_generated_module = ( lst_path_generated_module = (
s_lst_path_generated_module.strip().split("\n") s_lst_path_generated_module.strip().split("\n")
@ -507,15 +496,19 @@ async def test_exec(
hook.write(new_hook_line) hook.write(new_hook_line)
# Format editing code before commit # Format editing code before commit
await run_command_get_output( await lib_asyncio.run_command_get_output(
"./script/maintenance/black.sh", temp_dir_name "./script/maintenance/black.sh", temp_dir_name
) )
# init repo with git # init repo with git
for dir_to_git in lst_path_to_add_config: for dir_to_git in lst_path_to_add_config:
await run_command_get_output("git", "init", ".", cwd=dir_to_git) await lib_asyncio.run_command_get_output(
await run_command_get_output("git", "add", ".", cwd=dir_to_git) "git", "init", ".", cwd=dir_to_git
await run_command_get_output( )
await lib_asyncio.run_command_get_output(
"git", "add", ".", cwd=dir_to_git
)
await lib_asyncio.run_command_get_output(
"git", "commit", "-am", "'first commit'", cwd=dir_to_git "git", "commit", "-am", "'first commit'", cwd=dir_to_git
) )
@ -714,11 +707,6 @@ def check_git_change():
return not status return not status
def print_summary_task(task_list):
for task in task_list:
print(task.cr_code.co_name)
# START TEST # START TEST
@ -1115,6 +1103,29 @@ async def run_code_generator_template_demo_portal_test(
return test_result, test_status return test_result, test_status
async def run_code_generator_template_demo_internal_test(
config,
) -> Tuple[str, int]:
test_result = ""
test_status = 0
# Template
res, status = await test_exec(
config,
"./addons/TechnoLibre_odoo-code-generator-template",
generated_module="code_generator_demo_internal",
tested_module="code_generator_template_demo_internal",
search_class_module="demo_internal",
lst_init_module_name=[
"demo_internal",
],
test_name="code_generator_template_demo_internal",
run_in_sandbox=True,
)
test_result += res
test_status += status
return test_result, test_status
async def run_code_generator_template_demo_internal_inherit_test( async def run_code_generator_template_demo_internal_inherit_test(
config, config,
) -> Tuple[str, int]: ) -> Tuple[str, int]:
@ -1173,75 +1184,6 @@ async def run_helloworld_test(config) -> Tuple[str, int]:
return res, status return res, status
async def run_in_serial(task_list):
q = asyncio.Queue()
for task in task_list:
await q.put(task)
lst_result = []
for i in range(len(task_list)):
co = await q.get()
result = await co
lst_result.append(result)
return lst_result
class AsyncioPool:
# TODO check to replace this pool by https://github.com/dano/aioprocessing
def __init__(self, concurrency, loop=None):
"""
@param loop: asyncio loop
@param concurrency: Maximum number of concurrently running tasks
"""
self._loop = loop or asyncio.get_event_loop()
self._concurrency = concurrency
self._coros = deque([]) # All coroutines queued for execution
self._futures = [] # All currently running coroutines
self._lst_result = []
def close(self):
self._loop.close()
def add_coro(self, coro):
"""
@param coro: coroutine to add
"""
self._coros.append(coro)
self.print_status()
def run_until_complete(self):
self._loop.run_until_complete(self._wait_for_futures())
return self._lst_result
def print_status(self):
print(
" Status: coros:%s - futures:%s"
% (len(self._coros), len(self._futures))
)
def _start_futures(self):
num_to_start = self._concurrency - len(self._futures)
num_to_start = min(num_to_start, len(self._coros))
for _ in range(num_to_start):
coro = self._coros.popleft()
future = asyncio.ensure_future(coro, loop=self._loop)
self._futures.append(future)
self.print_status()
async def _wait_for_futures(self):
while len(self._coros) > 0 or len(self._futures) > 0:
self._start_futures()
futures_completed, futures_pending = await asyncio.wait(
self._futures,
loop=self._loop,
return_when=asyncio.FIRST_COMPLETED,
)
for future in futures_completed:
self._lst_result.append(future.result())
self._futures.remove(future)
self._start_futures()
def run_all_test(config) -> None: def run_all_test(config) -> None:
# low in time, at the end for more speed # low in time, at the end for more speed
task_list = [ task_list = [
@ -1250,6 +1192,7 @@ def run_all_test(config) -> None:
run_code_generator_demo_mariadb_sql_example_1_test(config), run_code_generator_demo_mariadb_sql_example_1_test(config),
run_code_generator_auto_backup_test(config), run_code_generator_auto_backup_test(config),
run_code_generator_template_demo_portal_test(config), run_code_generator_template_demo_portal_test(config),
run_code_generator_template_demo_internal_test(config),
run_code_generator_template_demo_internal_inherit_test(config), run_code_generator_template_demo_internal_inherit_test(config),
run_code_generator_template_demo_sysadmin_cron_test(config), run_code_generator_template_demo_sysadmin_cron_test(config),
run_code_generator_demo_test(config), run_code_generator_demo_test(config),
@ -1268,31 +1211,10 @@ def run_all_test(config) -> None:
run_demo_test(config), run_demo_test(config),
] ]
print_summary_task(task_list) lib_asyncio.print_summary_task(task_list)
if asyncio.get_event_loop().is_closed(): tpl_result = lib_asyncio.execute(config, task_list)
asyncio.set_event_loop(asyncio.new_event_loop())
if config.no_parallel:
tpl_result = asyncio.run(run_in_serial(task_list))
elif config.max_process:
pool = AsyncioPool(config.max_process)
for task in task_list:
pool.add_coro(task)
try:
tpl_result = pool.run_until_complete()
finally:
pool.close()
else:
# Use maximal resource
loop = asyncio.get_event_loop()
if config.debug:
loop.set_debug(True)
try:
commands = asyncio.gather(*task_list)
tpl_result = loop.run_until_complete(commands)
finally:
loop.close()
print_log(task_list, tpl_result) print_log(task_list, tpl_result)
status = check_result(task_list, tpl_result) status = check_result(task_list, tpl_result)
if status: if status:

2
test/code_generator/coverage_hello_world.sh
View file

@ -6,7 +6,7 @@ Green='\033[0;32m' # Green
tmp_dir=$(mktemp -d -t "erplibre_code_generator_helloworld-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX") tmp_dir=$(mktemp -d -t "erplibre_code_generator_helloworld-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX")
./script/code_generator/new_project.py -m test -d "${tmp_dir}" ./script/code_generator/new_project.py -m test -d "${tmp_dir}" --coverage
MODULE_PATH="${tmp_dir}/test" MODULE_PATH="${tmp_dir}/test"
if [[ -d "${MODULE_PATH}" ]]; then if [[ -d "${MODULE_PATH}" ]]; then