Merge branch 'develop'
- add statistic code tools - fix test validate no change in addons for run_parallel_test.py - upd osx installation - fix test coverage - missing test internal - and more
This commit is contained in:
commit
f3525e72ed
16 changed files with 1022 additions and 147 deletions
2
.github/FUNDING.yml
vendored
2
.github/FUNDING.yml
vendored
|
|
@ -1,6 +1,6 @@
|
|||
# These are supported funding model platforms
|
||||
|
||||
github: mathben963 # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
github: mathben # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: # Replace with a single Ko-fi username
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -9,6 +9,7 @@ config.conf
|
|||
*.DS_Store
|
||||
*.pyc
|
||||
.coverage*
|
||||
coverage*
|
||||
|
||||
.repo
|
||||
!addons/
|
||||
|
|
|
|||
53
Makefile
53
Makefile
|
|
@ -616,12 +616,27 @@ test_full_fast_coverage:
|
|||
./.venv/bin/coverage erase
|
||||
./script/test/run_parallel_test.py --coverage
|
||||
# TODO This test is broken in parallel
|
||||
./script/make.sh test_code_generator_hello_world
|
||||
./script/make.sh test_coverage_code_generator_hello_world
|
||||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
# run: make open_test_coverage
|
||||
|
||||
|
||||
.PHONY: test_cg_demo
|
||||
test_cg_demo:
|
||||
./script/make.sh clean
|
||||
# Need to create a BD to create cache _cache_erplibre_base
|
||||
./script/database/db_restore.py --database test
|
||||
./.venv/bin/coverage erase
|
||||
./script/addons/coverage_install_addons_dev.sh test code_generator_demo
|
||||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
|
||||
|
||||
.PHONY: test_base
|
||||
test_base:
|
||||
./script/make.sh test_format
|
||||
|
|
@ -645,6 +660,10 @@ test_format:
|
|||
test_code_generator_hello_world:
|
||||
./test/code_generator/hello_world.sh
|
||||
|
||||
.PHONY: test_coverage_code_generator_hello_world
|
||||
test_coverage_code_generator_hello_world:
|
||||
./test/code_generator/coverage_hello_world.sh
|
||||
|
||||
.PHONY: test_installation_demo
|
||||
test_installation_demo:
|
||||
./script/code_generator/check_git_change_code_generator.sh ./addons/TechnoLibre_odoo-code-generator-template
|
||||
|
|
@ -752,6 +771,7 @@ test_addons_sale:
|
|||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m
|
||||
./.venv/bin/coverage html
|
||||
./.venv/bin/coverage json
|
||||
|
||||
.PHONY: test_addons_helpdesk
|
||||
test_addons_helpdesk:
|
||||
|
|
@ -761,16 +781,18 @@ test_addons_helpdesk:
|
|||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m
|
||||
./.venv/bin/coverage html
|
||||
./.venv/bin/coverage json
|
||||
|
||||
.PHONY: test_addons_code_generator
|
||||
test_addons_code_generator:
|
||||
./.venv/bin/coverage erase
|
||||
./.venv/bin/python3 ./odoo/odoo-bin db --drop --database test_addons_code_generator
|
||||
# TODO missing test in code_generator
|
||||
./test.sh --dev all -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator
|
||||
./test.sh --dev cg -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator
|
||||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
# run: make open_test_coverage
|
||||
|
||||
.PHONY: test_addons_code_generator_code_generator
|
||||
|
|
@ -778,10 +800,24 @@ test_addons_code_generator_code_generator:
|
|||
# TODO this test only generation, not test
|
||||
./.venv/bin/coverage erase
|
||||
./script/database/db_restore.py --database test_addons_code_generator_code_generator
|
||||
./test.sh --dev all -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator
|
||||
./test.sh --dev cg -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator
|
||||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
# run: make open_test_coverage
|
||||
|
||||
.PHONY: test_addons_code_generator_template_code_generator
|
||||
test_addons_code_generator_template_code_generator:
|
||||
# TODO this test only generation, not test
|
||||
./.venv/bin/coverage erase
|
||||
./script/database/db_restore.py --database test_addons_code_generator_template_code_generator
|
||||
./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator
|
||||
./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator_template_code_generator
|
||||
./.venv/bin/coverage combine -a
|
||||
./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*"
|
||||
# run: make open_test_coverage
|
||||
|
||||
.PHONY: open_test_coverage
|
||||
|
|
@ -1037,6 +1073,17 @@ i18n_generate_demo_portal:
|
|||
clean:
|
||||
find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
|
||||
|
||||
###############
|
||||
# Statistic #
|
||||
###############
|
||||
.PHONY: stat_module_evolution_per_year
|
||||
stat_module_evolution_per_year:
|
||||
./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7
|
||||
|
||||
.PHONY: stat_module_evolution_per_year_OCA
|
||||
stat_module_evolution_per_year_OCA:
|
||||
./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7
|
||||
|
||||
###################
|
||||
# Documentation #
|
||||
###################
|
||||
|
|
|
|||
16
doc/FAQ.md
16
doc/FAQ.md
|
|
@ -124,3 +124,19 @@ docker-machine create --driver virtualbox default
|
|||
docker-machine env default
|
||||
eval "$(docker-machine env default)"
|
||||
```
|
||||
|
||||
### Run error `current limit exceeds maximum limit`
|
||||
|
||||
When you got
|
||||
|
||||
```
|
||||
File "/Users/test/Desktop/ERPLibre/odoo/odoo/service/server.py", line 83, in set_limit_memory_hard
|
||||
resource.setrlimit(rlimit, (config['limit_memory_hard'], hard))
|
||||
ValueError: current limit exceeds maximum limit
|
||||
```
|
||||
|
||||
Add line at the end of config.conf
|
||||
|
||||
```
|
||||
limit_memory_hard = 0
|
||||
```
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@
|
|||
<project name="muk_docs.git" path="addons/muk-it_muk_docs" remote="muk-it" groups="addons"/>
|
||||
<project name="muk_misc.git" path="addons/muk-it_muk_misc" remote="muk-it" revision="12.0_dev" groups="addons,base"/>
|
||||
<project name="muk_quality.git" path="addons/muk-it_muk_quality" remote="muk-it" groups="addons"/>
|
||||
<project name="muk_web.git" path="addons/muk-it_muk_web" remote="muk-it" revision="12.0_dev" groups="addons,base"/>
|
||||
<project name="muk_web.git" path="addons/muk-it_muk_web" remote="muk-it" revision="12.0_dev" groups="addons,base,code_generator"/>
|
||||
<project name="muk_website.git" path="addons/muk-it_muk_website" remote="muk-it" revision="12.0_remove_autoinstall" groups="addons,base"/>
|
||||
<project name="multi-company.git" path="addons/OCA_multi-company" remote="OCA" groups="addons"/>
|
||||
<project name="odoo-addons.git" path="addons/ajepe_odoo-addons" remote="ajepe" groups="addons"/>
|
||||
|
|
|
|||
68
poetry.lock
generated
68
poetry.lock
generated
|
|
@ -5210,6 +5210,27 @@ files = [
|
|||
{file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.64.1"
|
||||
description = "Fast, Extensible Progress Meter"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
|
||||
files = [
|
||||
{file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"},
|
||||
{file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[package.extras]
|
||||
dev = ["py-make (>=0.1.0)", "twine", "wheel"]
|
||||
notebook = ["ipywidgets (>=6)"]
|
||||
slack = ["slack-sdk"]
|
||||
telegram = ["requests"]
|
||||
|
||||
[[package]]
|
||||
name = "typed-ast"
|
||||
version = "1.5.4"
|
||||
|
|
@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
|
|||
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "uvloop"
|
||||
version = "0.17.0"
|
||||
description = "Fast implementation of asyncio event loop on top of libuv"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"},
|
||||
{file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "validators"
|
||||
version = "0.20.0"
|
||||
|
|
@ -5879,4 +5945,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.7.16"
|
||||
content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e"
|
||||
content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368"
|
||||
|
|
|
|||
|
|
@ -185,6 +185,8 @@ xmltodict = "^0.13.0"
|
|||
zeep = "^4.2.1"
|
||||
zipp = ">=3.6.0"
|
||||
zxcvbn = "^4.4.28"
|
||||
tqdm = "^4.64.1"
|
||||
uvloop = "^0.17.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
websocket_client = "^1.1.0"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ openupgradelib
|
|||
unidiff
|
||||
colorama
|
||||
wheel
|
||||
aioshutil<1.2
|
||||
|
||||
# For OSX
|
||||
cython
|
||||
|
|
@ -80,6 +79,10 @@ numpy==1.21.1
|
|||
feedparser==6.0.10
|
||||
geojson<3.0.0
|
||||
|
||||
# asyncio
|
||||
aioshutil<1.2
|
||||
tqdm
|
||||
uvloop
|
||||
|
||||
# Solve pcodedmp, oletools, rtfde and extract-msg
|
||||
win_unicode_console
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# This script will remove mail configuration, remove backup configuration, and force admin user to test/test
|
||||
echo "Update prod to dev on BD '$1'"
|
||||
|
||||
./script/addons/install_addons.sh "$1" user_test,disable_mail_server,disable_auto_backup
|
||||
./script/addons/install_addons_dev.sh "$1" user_test,disable_mail_server,disable_auto_backup
|
||||
|
||||
retVal=$?
|
||||
if [[ $retVal -ne 0 ]]; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
#!./.venv/bin/python
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import black
|
||||
import xmltodict
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_config():
|
||||
"""Parse command line arguments, extracting the config file name,
|
||||
returning the union of config file and command line arguments
|
||||
|
||||
:return: dict of config file settings and command line arguments
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description="""\
|
||||
Transform an XML into Python
|
||||
""",
|
||||
epilog="""\
|
||||
""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
dest="source",
|
||||
required=True,
|
||||
help="File to transform (XML)",
|
||||
)
|
||||
# parser.add_argument(
|
||||
# "--destination",
|
||||
# dest="destination",
|
||||
# help=(
|
||||
# "Result, output (Python). If not set, output will be console."
|
||||
# ),
|
||||
# )
|
||||
# parser.add_argument(
|
||||
# "-q",
|
||||
# "--quiet",
|
||||
# action="store_true",
|
||||
# help="Don't show output of found model.",
|
||||
# )
|
||||
args = parser.parse_args()
|
||||
if not os.path.isfile(args.source):
|
||||
raise Exception("Argument --source need to be a valid path to a file.")
|
||||
return args
|
||||
|
||||
|
||||
def get_module_name(filepath):
|
||||
manifest_file = "__manifest__.py"
|
||||
current_path = os.path.abspath(os.path.dirname(filepath))
|
||||
while current_path != "/":
|
||||
manifest_path = os.path.join(current_path, manifest_file)
|
||||
if os.path.isfile(manifest_path):
|
||||
return os.path.basename(current_path)
|
||||
current_path = os.path.dirname(current_path)
|
||||
raise FileNotFoundError(f"No {manifest_file} found for {filepath}")
|
||||
|
||||
|
||||
def get_value_from_field(lst_field, by_attr, name, record_id, get_attr):
|
||||
field_info = [a for a in lst_field if a.get(by_attr) == name]
|
||||
if not field_info:
|
||||
raise ValueError(
|
||||
f"Missing <field {by_attr[1:]}={name}'/> from record {record_id}"
|
||||
)
|
||||
field_info = field_info[0]
|
||||
return field_info.get(get_attr)
|
||||
|
||||
|
||||
def main():
|
||||
config = get_config()
|
||||
with open(config.source) as xml:
|
||||
xml_as_string = xml.read()
|
||||
xml_dict = xmltodict.parse(xml_as_string)
|
||||
# TODO replace this by get_value(xml_dict, "odoo.data.record")
|
||||
xml_dict_odoo = xml_dict.get("odoo")
|
||||
if not xml_dict_odoo:
|
||||
raise ValueError(f"Missing <odoo> into {config.source}")
|
||||
xml_dict_data = xml_dict_odoo.get("data")
|
||||
if not xml_dict_data:
|
||||
raise ValueError(f"Missing <data> into {config.source}")
|
||||
xml_dict_record = xml_dict_data.get("record")
|
||||
if not xml_dict_record:
|
||||
raise ValueError(f"Missing <record> into {config.source}")
|
||||
if type(xml_dict_record) is not list:
|
||||
lst_xml_record = [xml_dict_record]
|
||||
else:
|
||||
lst_xml_record = xml_dict_record
|
||||
lst_output = []
|
||||
for record in lst_xml_record:
|
||||
lst_field = record.get("field")
|
||||
if type(lst_field) is not list:
|
||||
lst_field = [lst_field]
|
||||
|
||||
# Condition unique
|
||||
try:
|
||||
get_value_from_field(
|
||||
lst_field, "@name", "groups_id", record.get("@id"), "@eval"
|
||||
)
|
||||
except:
|
||||
continue
|
||||
|
||||
url = get_value_from_field(
|
||||
lst_field, "@name", "url", record.get("@id"), "#text"
|
||||
)
|
||||
view_id = get_value_from_field(
|
||||
lst_field, "@name", "view_id", record.get("@id"), "@ref"
|
||||
)
|
||||
module_name = get_module_name(config.source)
|
||||
method_name = (
|
||||
f'get_{"_".join(url.replace("-", "_").strip("/").split("/"))}'
|
||||
)
|
||||
|
||||
# template = f"""
|
||||
# @http.route(
|
||||
# ["{url}"],
|
||||
# type="http",
|
||||
# auth="user",
|
||||
# website=True,
|
||||
# )
|
||||
# def {method_name}(self, **kw):
|
||||
# return request.env["ir.ui.view"].render_template(
|
||||
# "{module_name}.{view_id}",
|
||||
# )
|
||||
template = f"""
|
||||
@http.route(
|
||||
["{url}"],
|
||||
type="http",
|
||||
auth="user",
|
||||
website=True,
|
||||
)
|
||||
def {method_name}(self, **kw):
|
||||
return request.env.ref('{module_name}.{view_id}').render()
|
||||
"""
|
||||
lst_output.append(template)
|
||||
str_output = "\n".join(lst_output)
|
||||
formated_str_output = black.format_str(str_output, mode=black.Mode())
|
||||
print(formated_str_output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
status = main()
|
||||
sys.exit(status)
|
||||
|
|
@ -25,23 +25,21 @@ sudo su - postgres -c "CREATE EXTENSION postgis;\nCREATE EXTENSION postgis_topol
|
|||
# Install Dependencies
|
||||
#--------------------------------------------------
|
||||
echo "\n--- Installing Python 3 + pip3 --"
|
||||
brew install git python3 wget pyenv
|
||||
brew install git python3 wget
|
||||
brew link git
|
||||
brew link wget
|
||||
|
||||
echo "\n--- Installing extra --"
|
||||
brew install parallel
|
||||
echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----"
|
||||
brew install nodejs npm openssl
|
||||
sudo npm install -g rtlcss
|
||||
sudo npm install -g less
|
||||
sudo npm install -g prettier
|
||||
sudo npm install -g prettier @prettier/plugin-xml
|
||||
yes n|pyenv install 3.7.16
|
||||
pyenv local 3.7.16
|
||||
|
||||
echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.zshrc
|
||||
|
||||
echo -e "\n---- Installing poetry for reliable python package ----"
|
||||
curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
|
||||
|
||||
#--------------------------------------------------
|
||||
# Install Wkhtmltopdf if needed
|
||||
#--------------------------------------------------
|
||||
|
|
|
|||
152
script/lib_asyncio.py
Normal file
152
script/lib_asyncio.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
#!./.venv/bin/python
|
||||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
|
||||
import asyncio
|
||||
from collections import deque
|
||||
|
||||
import uvloop
|
||||
|
||||
|
||||
async def run_in_serial(task_list):
|
||||
q = asyncio.Queue()
|
||||
for task in task_list:
|
||||
await q.put(task)
|
||||
lst_result = []
|
||||
for i in range(len(task_list)):
|
||||
co = await q.get()
|
||||
result = await co
|
||||
lst_result.append(result)
|
||||
return lst_result
|
||||
|
||||
|
||||
async def run_command_get_output(*args, cwd=None):
|
||||
if cwd is not None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=cwd,
|
||||
)
|
||||
else:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
# Wait for the subprocess to finish
|
||||
stdout, stderr = await process.communicate()
|
||||
return stdout.decode()
|
||||
|
||||
|
||||
async def run_command_get_output_and_status(*args, cwd=None):
|
||||
if cwd is not None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=cwd,
|
||||
)
|
||||
else:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
# Wait for the subprocess to finish
|
||||
stdout, stderr = await process.communicate()
|
||||
return stdout.decode(), stderr.decode(), process.returncode
|
||||
|
||||
|
||||
def print_summary_task(task_list):
|
||||
for task in task_list:
|
||||
print(task.cr_code.co_name)
|
||||
|
||||
|
||||
def execute(config, lst_task, use_uvloop=False):
|
||||
|
||||
if not config.no_parallel and asyncio.get_event_loop().is_closed():
|
||||
asyncio.set_event_loop(asyncio.new_event_loop())
|
||||
|
||||
if config.no_parallel:
|
||||
return_value = asyncio.run(run_in_serial(lst_task))
|
||||
elif config.max_process:
|
||||
pool = AsyncioPool(config.max_process)
|
||||
for task in lst_task:
|
||||
pool.add_coro(task)
|
||||
try:
|
||||
return_value = pool.run_until_complete()
|
||||
finally:
|
||||
pool.close()
|
||||
else:
|
||||
# Use maximal resource
|
||||
if use_uvloop:
|
||||
uvloop.install()
|
||||
loop = asyncio.get_event_loop()
|
||||
if config.debug:
|
||||
loop.set_debug(True)
|
||||
try:
|
||||
commands = asyncio.gather(*lst_task)
|
||||
return_value = loop.run_until_complete(commands)
|
||||
finally:
|
||||
loop.close()
|
||||
return return_value
|
||||
|
||||
|
||||
class AsyncioPool:
|
||||
# TODO check to replace this pool by https://github.com/dano/aioprocessing
|
||||
def __init__(self, concurrency, loop=None):
|
||||
"""
|
||||
@param loop: asyncio loop
|
||||
@param concurrency: Maximum number of concurrently running tasks
|
||||
"""
|
||||
self._loop = loop or asyncio.get_event_loop()
|
||||
self._concurrency = concurrency
|
||||
self._coros = deque([]) # All coroutines queued for execution
|
||||
self._futures = [] # All currently running coroutines
|
||||
self._lst_result = []
|
||||
|
||||
def close(self):
|
||||
self._loop.close()
|
||||
|
||||
def add_coro(self, coro):
|
||||
"""
|
||||
@param coro: coroutine to add
|
||||
"""
|
||||
self._coros.append(coro)
|
||||
self.print_status()
|
||||
|
||||
def run_until_complete(self):
|
||||
self._loop.run_until_complete(self._wait_for_futures())
|
||||
return self._lst_result
|
||||
|
||||
def print_status(self):
|
||||
print(
|
||||
" Status: coros:%s - futures:%s"
|
||||
% (len(self._coros), len(self._futures))
|
||||
)
|
||||
|
||||
def _start_futures(self):
|
||||
num_to_start = self._concurrency - len(self._futures)
|
||||
num_to_start = min(num_to_start, len(self._coros))
|
||||
for _ in range(num_to_start):
|
||||
coro = self._coros.popleft()
|
||||
future = asyncio.ensure_future(coro, loop=self._loop)
|
||||
self._futures.append(future)
|
||||
self.print_status()
|
||||
|
||||
async def _wait_for_futures(self):
|
||||
while len(self._coros) > 0 or len(self._futures) > 0:
|
||||
self._start_futures()
|
||||
futures_completed, futures_pending = await asyncio.wait(
|
||||
self._futures,
|
||||
loop=self._loop,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
for future in futures_completed:
|
||||
self._lst_result.append(future.result())
|
||||
self._futures.remove(future)
|
||||
self._start_futures()
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
# all argument is directory/file separate by space
|
||||
|
||||
./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,css,scss --format=summary "$@"
|
||||
./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,csv,css,scss --format=summary "$@"
|
||||
|
|
|
|||
519
script/statistic/show_evolution_module.py
Executable file
519
script/statistic/show_evolution_module.py
Executable file
|
|
@ -0,0 +1,519 @@
|
|||
#!./.venv/bin/python
|
||||
import argparse
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import csv
|
||||
from collections import defaultdict
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
new_path = os.path.normpath(
|
||||
os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
)
|
||||
sys.path.append(new_path)
|
||||
|
||||
from script import lib_asyncio
|
||||
|
||||
FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT)
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_NAME = os.path.basename(os.getcwd())
|
||||
VENV_PATH = "./.venv"
|
||||
CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo")
|
||||
CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv"
|
||||
MAX_COROUTINE = 300
|
||||
IGNORE_REPO_LIST = (
|
||||
"https://github.com/OCA/odoo-module-migrator.git",
|
||||
"https://github.com/OCA/maintainer-tools.git",
|
||||
"https://github.com/itpp-labs/odoo-development.git",
|
||||
"https://github.com/itpp-labs/odoo-port-docs.git",
|
||||
"https://github.com/itpp-labs/odoo-test-docs.git",
|
||||
"https://github.com/odoo/documentation.git",
|
||||
"https://github.com/ERPLibre/ERPLibre_image_db.git",
|
||||
"https://github.com/muk-it/muk_docs.git",
|
||||
)
|
||||
DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": ["/addons/", "/odoo/addons/"]}
|
||||
DCT_VERSION_RELEASE = {
|
||||
"1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(),
|
||||
"2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(),
|
||||
"3.0": datetime.datetime.strptime("2005-09-01", "%Y-%m-%d").date(),
|
||||
"4.0": datetime.datetime.strptime("2006-12-01", "%Y-%m-%d").date(),
|
||||
"5.0": datetime.datetime.strptime("2009-04-01", "%Y-%m-%d").date(),
|
||||
"6.0": datetime.datetime.strptime("2009-10-01", "%Y-%m-%d").date(),
|
||||
"6.1": datetime.datetime.strptime("2012-02-27", "%Y-%m-%d").date(),
|
||||
"7.0": datetime.datetime.strptime("2012-12-01", "%Y-%m-%d").date(),
|
||||
"8.0": datetime.datetime.strptime("2014-09-01", "%Y-%m-%d").date(),
|
||||
"9.0": datetime.datetime.strptime("2015-11-01", "%Y-%m-%d").date(),
|
||||
"10.0": datetime.datetime.strptime("2016-10-01", "%Y-%m-%d").date(),
|
||||
"11.0": datetime.datetime.strptime("2017-10-01", "%Y-%m-%d").date(),
|
||||
"12.0": datetime.datetime.strptime("2018-10-01", "%Y-%m-%d").date(),
|
||||
"13.0": datetime.datetime.strptime("2019-10-01", "%Y-%m-%d").date(),
|
||||
"14.0": datetime.datetime.strptime("2020-10-01", "%Y-%m-%d").date(),
|
||||
"15.0": datetime.datetime.strptime("2021-10-01", "%Y-%m-%d").date(),
|
||||
"16.0": datetime.datetime.strptime("2022-10-01", "%Y-%m-%d").date(),
|
||||
}
|
||||
CSV_HEADER_MODULE_NAME = "Nom technique"
|
||||
# TODO use rich progress bar when clone
|
||||
# TODO support conflict repo name
|
||||
|
||||
|
||||
def get_config():
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description="""\
|
||||
Clone all OCA repo from file 'source_repo_addons.csv', checkout branch from param --branches
|
||||
""",
|
||||
epilog="""\
|
||||
""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean_all",
|
||||
action="store_true",
|
||||
help=f"Wipe all data from path '{CACHE_CLONE_PATH}'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help=f"More information",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
action="store_true",
|
||||
help="Enable asyncio debugging",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ignore_release_date",
|
||||
action="store_true",
|
||||
help=(
|
||||
f"Give result without release date logic, this means it gives"
|
||||
f" result in past of release version."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no_parallel",
|
||||
action="store_true",
|
||||
help=(
|
||||
f"By default run in parallel with all CPU, this disable"
|
||||
f" parallelism with asyncio"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force_git_fetch",
|
||||
action="store_true",
|
||||
help=f"Force git fetch to update it all remote.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--max_process",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Max processor to use. If 0, use max.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--history_length",
|
||||
default=0,
|
||||
type=int,
|
||||
help=(
|
||||
f"Set to 0 to clone all history, default 0. Can be another number."
|
||||
f" This broke other version."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--more_year",
|
||||
default=0,
|
||||
type=int,
|
||||
help=(
|
||||
f"At 0, search only present before_date, > 0, add year to"
|
||||
f" before_date for multiple stat. Depend on param 'before_date'"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--before_date",
|
||||
type=lambda d: datetime.datetime.strptime(d, "%Y-%m-%d").date(),
|
||||
default=None,
|
||||
help=f"Set a check date before, example '2020-01-01'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filter",
|
||||
default="",
|
||||
help=f"keyword to ignore separate by ','. Suggest /OCA/",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--branches",
|
||||
default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0",
|
||||
help="Branch to analyse, separate by ','",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compare_csv",
|
||||
help=(
|
||||
"Path of CSV, search column 'Nom technique' and give list of"
|
||||
" difference module, missing from CSV. Need only 1 branches and no"
|
||||
" more_year."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
die(
|
||||
args.history_length < 0,
|
||||
f"Argument 'history_length' need to be positive",
|
||||
)
|
||||
die(
|
||||
args.more_year < 0,
|
||||
f"Argument 'more_year' need to be positive",
|
||||
)
|
||||
die(
|
||||
args.before_date is None and args.more_year > 0,
|
||||
f"Cannot specify more_year when before_date is not set.",
|
||||
)
|
||||
# clean parameter
|
||||
args.branches = ",".join([a for a in args.branches.split(",") if a])
|
||||
die(
|
||||
any(
|
||||
[
|
||||
a
|
||||
for a in args.branches.split(",")
|
||||
if a not in DCT_VERSION_RELEASE.keys()
|
||||
]
|
||||
),
|
||||
"The params branches contain not supported version, check"
|
||||
f" DCT_VERSION_RELEASE : {list(DCT_VERSION_RELEASE.keys())}",
|
||||
)
|
||||
die(
|
||||
args.compare_csv
|
||||
and (args.more_year > 0 or len(args.branches.split(",")) > 1),
|
||||
f"When use param compare_csv, cannot use more_year and only 1 branch.",
|
||||
)
|
||||
if args.compare_csv:
|
||||
die(
|
||||
not os.path.isfile(args.compare_csv),
|
||||
f"Path of {args.compare_csv} need to be a file path.",
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
config = get_config()
|
||||
lst_branch = config.branches.split(",")
|
||||
before_date = config.before_date
|
||||
lst_unique_module = set()
|
||||
lst_unique_uninstallable_module = set()
|
||||
|
||||
# Check
|
||||
die(
|
||||
not os.path.isdir(VENV_PATH),
|
||||
f"Missing {VENV_PATH} venv path, did you install ERPLibre?",
|
||||
)
|
||||
|
||||
# Setup environment
|
||||
_logger.info(f"Setup {CACHE_CLONE_PATH}")
|
||||
if config.clean_all and os.path.isdir(CACHE_CLONE_PATH):
|
||||
_logger.info(f"Delete directory {CACHE_CLONE_PATH}")
|
||||
shutil.rmtree(CACHE_CLONE_PATH)
|
||||
|
||||
if not os.path.isdir(CACHE_CLONE_PATH):
|
||||
_logger.info(f"Create directory {CACHE_CLONE_PATH}")
|
||||
os.mkdir(CACHE_CLONE_PATH)
|
||||
|
||||
# Get list of Repo
|
||||
lst_repo_url = get_OCA_repo_list(config)
|
||||
|
||||
# Clone all repo
|
||||
_logger.info(f"Clone repo")
|
||||
lst_task_clone = [
|
||||
clone_repo(i, a, config.force_git_fetch)
|
||||
for i, a in enumerate(lst_repo_url)
|
||||
]
|
||||
lst_repo_path = lib_asyncio.execute(
|
||||
config, lst_task_clone, use_uvloop=True
|
||||
)
|
||||
|
||||
# Extract module list
|
||||
_logger.info("Extract module")
|
||||
lst_task = []
|
||||
for nb_year in range(config.more_year + 1):
|
||||
# key is odoo version, value is set of module name
|
||||
lst_branch_adapt = lst_branch
|
||||
from_date = before_date
|
||||
if before_date:
|
||||
from_date += relativedelta(years=nb_year)
|
||||
if not config.ignore_release_date:
|
||||
lst_branch_adapt = get_branches_supported_date(
|
||||
lst_branch, from_date
|
||||
)
|
||||
|
||||
# for branch in lst_branch_adapt:
|
||||
for repo_name, repo_path in lst_repo_path:
|
||||
lst_task.append(
|
||||
extract_module(
|
||||
lst_branch_adapt, repo_name, repo_path, from_date
|
||||
)
|
||||
)
|
||||
if config.debug:
|
||||
_logger.info(
|
||||
"Setup extract module:"
|
||||
f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}"
|
||||
)
|
||||
|
||||
_logger.info(f"Execute extract module with {len(lst_task)} coroutine.")
|
||||
# Need to set a max, or crash
|
||||
if len(lst_task) / MAX_COROUTINE > 1:
|
||||
lst_result = []
|
||||
for i in range(int(len(lst_task) / MAX_COROUTINE) + 1):
|
||||
min_i = i * MAX_COROUTINE
|
||||
max_i = min((i + 1) * MAX_COROUTINE, len(lst_task))
|
||||
_logger.info(f"Partial execution {min_i} to {max_i}")
|
||||
tpl_result = lib_asyncio.execute(
|
||||
config, lst_task[min_i:max_i], use_uvloop=True
|
||||
)
|
||||
lst_result += tpl_result
|
||||
tpl_result = lst_result
|
||||
else:
|
||||
tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True)
|
||||
_logger.info("Analyse information")
|
||||
dct_result = defaultdict(lambda: defaultdict(int))
|
||||
dct_result_unique = defaultdict(set)
|
||||
for lst_result_stack in tpl_result:
|
||||
for dct_result_module in lst_result_stack:
|
||||
# lst_module, branch, path, repo, before_date
|
||||
before_date = dct_result_module.get("before_date")
|
||||
lst_module = dct_result_module.get("lst_module")
|
||||
lst_uninstallable_module = dct_result_module.get(
|
||||
"lst_uninstallable_module"
|
||||
)
|
||||
branch = dct_result_module.get("branch")
|
||||
# path = dct_result_module.get("path")
|
||||
# repo = dct_result_module.get("repo")
|
||||
dct_result[before_date][branch] += len(lst_module)
|
||||
dct_result_unique[before_date].update(lst_module)
|
||||
lst_unique_module.update(lst_module)
|
||||
lst_unique_uninstallable_module.update(lst_uninstallable_module)
|
||||
|
||||
# Show result
|
||||
_logger.info("Show result")
|
||||
for before_date, dct_branch_result in dct_result.items():
|
||||
str_extra = ""
|
||||
if before_date:
|
||||
str_extra = f" before {before_date}"
|
||||
print(f"Stat nb module by branch name{str_extra}")
|
||||
for branch_name, count_module in dct_branch_result.items():
|
||||
print(f"{branch_name}\t{count_module}")
|
||||
for before_date, set_result in dct_result_unique.items():
|
||||
str_extra = ""
|
||||
if before_date:
|
||||
str_extra = f" before {before_date}"
|
||||
print(f"Diff nb unique module{str_extra}, length {len(set_result)}")
|
||||
print("end of stat")
|
||||
|
||||
if config.compare_csv:
|
||||
index_header = -1
|
||||
lst_unique_module_csv = set()
|
||||
with open(config.compare_csv, "r") as csvfile:
|
||||
for csv_module in csv.reader(csvfile):
|
||||
if index_header == -1:
|
||||
try:
|
||||
index_header = csv_module.index(CSV_HEADER_MODULE_NAME)
|
||||
except Exception as e:
|
||||
_logger.error(
|
||||
f"Missing header {CSV_HEADER_MODULE_NAME} in CSV."
|
||||
)
|
||||
raise e
|
||||
else:
|
||||
module_name = csv_module[index_header]
|
||||
if module_name not in lst_unique_uninstallable_module:
|
||||
lst_unique_module_csv.add(module_name)
|
||||
|
||||
lst_missing_module_repo = lst_unique_module_csv.difference(
|
||||
lst_unique_module
|
||||
)
|
||||
lst_missing_module_csv = lst_unique_module.difference(
|
||||
lst_unique_module_csv
|
||||
)
|
||||
print("CSV compare")
|
||||
print(f"Module missing from CSV '{len(lst_missing_module_repo)}':")
|
||||
print(lst_missing_module_repo)
|
||||
print(f"Module missing into CSV '{len(lst_missing_module_csv)}':")
|
||||
print(lst_missing_module_csv)
|
||||
|
||||
|
||||
def die(cond, message, code=1):
|
||||
if cond:
|
||||
print(message, file=sys.stderr)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def get_branches_supported_date(lst_branch, from_date):
|
||||
return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date]
|
||||
|
||||
|
||||
async def extract_module(lst_branch, repo, path, before_date):
|
||||
# print(f"{branch} - {path} - {before_date}")
|
||||
lst_result = []
|
||||
for branch in lst_branch:
|
||||
return_value = {
|
||||
"lst_module": [],
|
||||
"branch": branch,
|
||||
"path": path,
|
||||
"repo": repo,
|
||||
"before_date": before_date,
|
||||
}
|
||||
remote_branch_name = f"origin/{branch}"
|
||||
|
||||
# Detect commit to check
|
||||
# git rev-list -n 1 --before 2022-01-01 origin/12.0
|
||||
lst_args = ["git", "rev-list", "-n", "1"]
|
||||
if before_date:
|
||||
lst_args += ["--before", str(before_date)]
|
||||
lst_args += [remote_branch_name]
|
||||
(
|
||||
commit,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=path
|
||||
)
|
||||
|
||||
if status:
|
||||
# The branch doesn't exist
|
||||
# _logger.error(f"Receive status 'git rev-list' {status}")
|
||||
continue
|
||||
|
||||
# TODO send msg when no result
|
||||
if not commit:
|
||||
continue
|
||||
|
||||
commit = commit.strip()
|
||||
|
||||
lst_module_installable = []
|
||||
lst_module_uninstallable = []
|
||||
# List directory
|
||||
lst_suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "")
|
||||
if type(lst_suffix_path) is str:
|
||||
lst_suffix_path = [lst_suffix_path]
|
||||
for suffix_path in lst_suffix_path:
|
||||
adapt_path = path + suffix_path
|
||||
lst_args = ["git", "ls-tree", "-d", "--name-only", commit]
|
||||
(
|
||||
str_folders,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=adapt_path
|
||||
)
|
||||
if status:
|
||||
_logger.error(f"Receive status 'git ls-tree' {status}")
|
||||
continue
|
||||
|
||||
# TODO send msg when no result
|
||||
if not str_folders:
|
||||
continue
|
||||
lst_module = str_folders.split()
|
||||
if not lst_module:
|
||||
continue
|
||||
|
||||
if suffix_path:
|
||||
suffix_path = suffix_path[1:]
|
||||
|
||||
for module in lst_module:
|
||||
# List directory
|
||||
lst_args = [
|
||||
"git",
|
||||
"cat-file",
|
||||
"-p",
|
||||
f"{commit}:{suffix_path}{module}/__manifest__.py",
|
||||
]
|
||||
(
|
||||
manifest,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=path
|
||||
)
|
||||
if status:
|
||||
# Maybe __openerp__.py
|
||||
lst_args = [
|
||||
"git",
|
||||
"cat-file",
|
||||
"-p",
|
||||
f"{commit}:{module}/__openerp__.py",
|
||||
]
|
||||
(
|
||||
manifest,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=adapt_path
|
||||
)
|
||||
if status:
|
||||
# Ignore, it's not a module
|
||||
# _logger.error(f"Receive status 'git cat-file' {status}")
|
||||
# return return_value
|
||||
continue
|
||||
dct_manifest = eval(manifest + "\n")
|
||||
if dct_manifest.get("installable", True):
|
||||
lst_module_installable.append(module)
|
||||
else:
|
||||
lst_module_uninstallable.append(module)
|
||||
|
||||
return_value["lst_module"] = lst_module_installable
|
||||
return_value["lst_uninstallable_module"] = lst_module_uninstallable
|
||||
lst_result.append(return_value)
|
||||
return lst_result
|
||||
|
||||
|
||||
async def clone_repo(i, repo_url, force_fetch):
|
||||
split_repo_url = repo_url.split("/")
|
||||
repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}"
|
||||
repo_path = os.path.join(CACHE_CLONE_PATH, repo_name)
|
||||
if not os.path.isdir(repo_path):
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH
|
||||
)
|
||||
_logger.info(f"[{i}]New clone {repo_path} with {repo_url}")
|
||||
elif force_fetch:
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "fetch", "--all", cwd=CACHE_CLONE_PATH
|
||||
)
|
||||
_logger.info(f"[{i}]Fetch {repo_path} with {repo_url}")
|
||||
|
||||
return repo_name, repo_path
|
||||
|
||||
|
||||
def get_OCA_repo_list(config):
|
||||
with open(CST_FILE_SOURCE_REPO_ADDONS) as file:
|
||||
all_lines = file.readlines()
|
||||
if all_lines:
|
||||
# Validate first line is supported column
|
||||
expected_header = "url,path,revision,clone-depth\n"
|
||||
if all_lines[0] != expected_header:
|
||||
raise Exception(
|
||||
"Not supported csv, please validate"
|
||||
f" {CST_FILE_SOURCE_REPO_ADDONS} with first line"
|
||||
f" {expected_header}"
|
||||
)
|
||||
# Ignore first line
|
||||
all_lines = all_lines[1:]
|
||||
|
||||
# Be sure empty endline at the end of file
|
||||
if all_lines[-1][-1] != "\n":
|
||||
all_lines[-1] = all_lines[-1] + "\n"
|
||||
all_lines = [a.split(",")[0] for a in all_lines]
|
||||
# TODO support split(",")
|
||||
if config.filter:
|
||||
all_lines = [
|
||||
a
|
||||
for a in all_lines
|
||||
if config.filter in a and a not in IGNORE_REPO_LIST
|
||||
]
|
||||
else:
|
||||
all_lines = [a for a in all_lines if a not in IGNORE_REPO_LIST]
|
||||
return all_lines
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -9,13 +9,19 @@ import sys
|
|||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from typing import Tuple
|
||||
|
||||
import aioshutil
|
||||
import git
|
||||
from colorama import Fore
|
||||
|
||||
new_path = os.path.normpath(
|
||||
os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
)
|
||||
sys.path.append(new_path)
|
||||
|
||||
from script import lib_asyncio
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -172,27 +178,6 @@ def print_log(lst_task, tpl_result):
|
|||
f.write("\n")
|
||||
|
||||
|
||||
async def run_command_get_output(*args, cwd=None):
|
||||
if cwd is not None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=cwd,
|
||||
)
|
||||
else:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
# stdout must a pipe to be accessible as process.stdout
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
# Wait for the subprocess to finish
|
||||
stdout, stderr = await process.communicate()
|
||||
return stdout.decode()
|
||||
|
||||
|
||||
async def run_command(*args, test_name=None):
|
||||
# Create subprocess
|
||||
start_time = time.time()
|
||||
|
|
@ -378,8 +363,10 @@ async def test_exec(
|
|||
lst_module_to_test = tested_module.split(",")
|
||||
for module_name in lst_module_to_test:
|
||||
# Update path to change new emplacement
|
||||
s_lst_path_tested_module = await run_command_get_output(
|
||||
"find", ".", "-name", module_name
|
||||
s_lst_path_tested_module = (
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"find", "./addons/", "-name", module_name
|
||||
)
|
||||
)
|
||||
if not s_lst_path_tested_module:
|
||||
return (
|
||||
|
|
@ -420,12 +407,14 @@ async def test_exec(
|
|||
os.path.join(temp_dir_name, s_first_path)
|
||||
)
|
||||
|
||||
s_lst_path_generated_module = await run_command_get_output(
|
||||
"find",
|
||||
".",
|
||||
"-name",
|
||||
generated_module,
|
||||
cwd=temp_dir_name,
|
||||
s_lst_path_generated_module = (
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"find",
|
||||
"./addons/",
|
||||
"-name",
|
||||
generated_module,
|
||||
cwd=temp_dir_name,
|
||||
)
|
||||
)
|
||||
if s_lst_path_generated_module:
|
||||
lst_path_generated_module = (
|
||||
|
|
@ -507,15 +496,19 @@ async def test_exec(
|
|||
hook.write(new_hook_line)
|
||||
|
||||
# Format editing code before commit
|
||||
await run_command_get_output(
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"./script/maintenance/black.sh", temp_dir_name
|
||||
)
|
||||
|
||||
# init repo with git
|
||||
for dir_to_git in lst_path_to_add_config:
|
||||
await run_command_get_output("git", "init", ".", cwd=dir_to_git)
|
||||
await run_command_get_output("git", "add", ".", cwd=dir_to_git)
|
||||
await run_command_get_output(
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "init", ".", cwd=dir_to_git
|
||||
)
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "add", ".", cwd=dir_to_git
|
||||
)
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "commit", "-am", "'first commit'", cwd=dir_to_git
|
||||
)
|
||||
|
||||
|
|
@ -714,11 +707,6 @@ def check_git_change():
|
|||
return not status
|
||||
|
||||
|
||||
def print_summary_task(task_list):
|
||||
for task in task_list:
|
||||
print(task.cr_code.co_name)
|
||||
|
||||
|
||||
# START TEST
|
||||
|
||||
|
||||
|
|
@ -1115,6 +1103,29 @@ async def run_code_generator_template_demo_portal_test(
|
|||
return test_result, test_status
|
||||
|
||||
|
||||
async def run_code_generator_template_demo_internal_test(
|
||||
config,
|
||||
) -> Tuple[str, int]:
|
||||
test_result = ""
|
||||
test_status = 0
|
||||
# Template
|
||||
res, status = await test_exec(
|
||||
config,
|
||||
"./addons/TechnoLibre_odoo-code-generator-template",
|
||||
generated_module="code_generator_demo_internal",
|
||||
tested_module="code_generator_template_demo_internal",
|
||||
search_class_module="demo_internal",
|
||||
lst_init_module_name=[
|
||||
"demo_internal",
|
||||
],
|
||||
test_name="code_generator_template_demo_internal",
|
||||
run_in_sandbox=True,
|
||||
)
|
||||
test_result += res
|
||||
test_status += status
|
||||
|
||||
return test_result, test_status
|
||||
|
||||
async def run_code_generator_template_demo_internal_inherit_test(
|
||||
config,
|
||||
) -> Tuple[str, int]:
|
||||
|
|
@ -1173,75 +1184,6 @@ async def run_helloworld_test(config) -> Tuple[str, int]:
|
|||
return res, status
|
||||
|
||||
|
||||
async def run_in_serial(task_list):
|
||||
q = asyncio.Queue()
|
||||
for task in task_list:
|
||||
await q.put(task)
|
||||
lst_result = []
|
||||
for i in range(len(task_list)):
|
||||
co = await q.get()
|
||||
result = await co
|
||||
lst_result.append(result)
|
||||
return lst_result
|
||||
|
||||
|
||||
class AsyncioPool:
|
||||
# TODO check to replace this pool by https://github.com/dano/aioprocessing
|
||||
def __init__(self, concurrency, loop=None):
|
||||
"""
|
||||
@param loop: asyncio loop
|
||||
@param concurrency: Maximum number of concurrently running tasks
|
||||
"""
|
||||
self._loop = loop or asyncio.get_event_loop()
|
||||
self._concurrency = concurrency
|
||||
self._coros = deque([]) # All coroutines queued for execution
|
||||
self._futures = [] # All currently running coroutines
|
||||
self._lst_result = []
|
||||
|
||||
def close(self):
|
||||
self._loop.close()
|
||||
|
||||
def add_coro(self, coro):
|
||||
"""
|
||||
@param coro: coroutine to add
|
||||
"""
|
||||
self._coros.append(coro)
|
||||
self.print_status()
|
||||
|
||||
def run_until_complete(self):
|
||||
self._loop.run_until_complete(self._wait_for_futures())
|
||||
return self._lst_result
|
||||
|
||||
def print_status(self):
|
||||
print(
|
||||
" Status: coros:%s - futures:%s"
|
||||
% (len(self._coros), len(self._futures))
|
||||
)
|
||||
|
||||
def _start_futures(self):
|
||||
num_to_start = self._concurrency - len(self._futures)
|
||||
num_to_start = min(num_to_start, len(self._coros))
|
||||
for _ in range(num_to_start):
|
||||
coro = self._coros.popleft()
|
||||
future = asyncio.ensure_future(coro, loop=self._loop)
|
||||
self._futures.append(future)
|
||||
self.print_status()
|
||||
|
||||
async def _wait_for_futures(self):
|
||||
while len(self._coros) > 0 or len(self._futures) > 0:
|
||||
self._start_futures()
|
||||
futures_completed, futures_pending = await asyncio.wait(
|
||||
self._futures,
|
||||
loop=self._loop,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
for future in futures_completed:
|
||||
self._lst_result.append(future.result())
|
||||
self._futures.remove(future)
|
||||
self._start_futures()
|
||||
|
||||
|
||||
def run_all_test(config) -> None:
|
||||
# low in time, at the end for more speed
|
||||
task_list = [
|
||||
|
|
@ -1250,6 +1192,7 @@ def run_all_test(config) -> None:
|
|||
run_code_generator_demo_mariadb_sql_example_1_test(config),
|
||||
run_code_generator_auto_backup_test(config),
|
||||
run_code_generator_template_demo_portal_test(config),
|
||||
run_code_generator_template_demo_internal_test(config),
|
||||
run_code_generator_template_demo_internal_inherit_test(config),
|
||||
run_code_generator_template_demo_sysadmin_cron_test(config),
|
||||
run_code_generator_demo_test(config),
|
||||
|
|
@ -1268,31 +1211,10 @@ def run_all_test(config) -> None:
|
|||
run_demo_test(config),
|
||||
]
|
||||
|
||||
print_summary_task(task_list)
|
||||
lib_asyncio.print_summary_task(task_list)
|
||||
|
||||
if asyncio.get_event_loop().is_closed():
|
||||
asyncio.set_event_loop(asyncio.new_event_loop())
|
||||
tpl_result = lib_asyncio.execute(config, task_list)
|
||||
|
||||
if config.no_parallel:
|
||||
tpl_result = asyncio.run(run_in_serial(task_list))
|
||||
elif config.max_process:
|
||||
pool = AsyncioPool(config.max_process)
|
||||
for task in task_list:
|
||||
pool.add_coro(task)
|
||||
try:
|
||||
tpl_result = pool.run_until_complete()
|
||||
finally:
|
||||
pool.close()
|
||||
else:
|
||||
# Use maximal resource
|
||||
loop = asyncio.get_event_loop()
|
||||
if config.debug:
|
||||
loop.set_debug(True)
|
||||
try:
|
||||
commands = asyncio.gather(*task_list)
|
||||
tpl_result = loop.run_until_complete(commands)
|
||||
finally:
|
||||
loop.close()
|
||||
print_log(task_list, tpl_result)
|
||||
status = check_result(task_list, tpl_result)
|
||||
if status:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Green='\033[0;32m' # Green
|
|||
|
||||
tmp_dir=$(mktemp -d -t "erplibre_code_generator_helloworld-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX")
|
||||
|
||||
./script/code_generator/new_project.py -m test -d "${tmp_dir}"
|
||||
./script/code_generator/new_project.py -m test -d "${tmp_dir}" --coverage
|
||||
|
||||
MODULE_PATH="${tmp_dir}/test"
|
||||
if [[ -d "${MODULE_PATH}" ]]; then
|
||||
|
|
|
|||
Loading…
Reference in a new issue