From 586c9bc53f564e9d865d487c56340c4066190700 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Wed, 15 Feb 2023 01:19:24 -0500 Subject: [PATCH 01/17] [ADD] script stat: show evolution between module for OCA addons --- script/statistic/show_evolution_module.py | 372 ++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100755 script/statistic/show_evolution_module.py diff --git a/script/statistic/show_evolution_module.py b/script/statistic/show_evolution_module.py new file mode 100755 index 0000000..1b7bb2d --- /dev/null +++ b/script/statistic/show_evolution_module.py @@ -0,0 +1,372 @@ +#!./.venv/bin/python +import argparse +import datetime +import logging +import os +import shutil +import sys +from collections import defaultdict + +from dateutil.relativedelta import relativedelta +from git import Repo + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) + +_logger = logging.getLogger(__name__) + +PROJECT_NAME = os.path.basename(os.getcwd()) +VENV_PATH = "./.venv" +CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_OCA") +CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv" +IGNORE_REPO_LIST = ( + "https://github.com/OCA/odoo-module-migrator.git", + "https://github.com/OCA/maintainer-tools.git", + "https://github.com/OCA/connector.git", + "https://github.com/itpp-labs/odoo-development.git", + "https://github.com/itpp-labs/odoo-port-docs.git", + "https://github.com/itpp-labs/odoo-test-docs.git", + "https://github.com/odoo/documentation.git", + "https://github.com/OCA/odoo-module-migrator.git", + "https://github.com/OCA/maintainer-tools.git", +) +DCT_CHANGE_ADDONS_PATH = {"odoo": "/addons/"} +DCT_VERSION_RELEASE = { + "1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(), + "2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(), + "3.0": datetime.datetime.strptime("2005-09-01", "%Y-%m-%d").date(), + "4.0": datetime.datetime.strptime("2006-12-01", "%Y-%m-%d").date(), + "5.0": datetime.datetime.strptime("2009-04-01", "%Y-%m-%d").date(), + "6.0": datetime.datetime.strptime("2009-10-01", "%Y-%m-%d").date(), + "6.1": datetime.datetime.strptime("2012-02-27", "%Y-%m-%d").date(), + "7.0": datetime.datetime.strptime("2012-12-01", "%Y-%m-%d").date(), + "8.0": datetime.datetime.strptime("2014-09-01", "%Y-%m-%d").date(), + "9.0": datetime.datetime.strptime("2015-11-01", "%Y-%m-%d").date(), + "10.0": datetime.datetime.strptime("2016-10-01", "%Y-%m-%d").date(), + "11.0": datetime.datetime.strptime("2017-10-01", "%Y-%m-%d").date(), + "12.0": datetime.datetime.strptime("2018-10-01", "%Y-%m-%d").date(), + "13.0": datetime.datetime.strptime("2019-10-01", "%Y-%m-%d").date(), + "14.0": datetime.datetime.strptime("2020-10-01", "%Y-%m-%d").date(), + "15.0": datetime.datetime.strptime("2021-10-01", "%Y-%m-%d").date(), + "16.0": datetime.datetime.strptime("2022-10-01", "%Y-%m-%d").date(), +} +# TODO use rich progress bar when clone +# TODO support conflict repo name + + +def get_config(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + Clone all OCA repo from file 'source_repo_addons.csv', checkout branch from param --branches +""", + epilog="""\ +""", + ) + parser.add_argument( + "--clean_all", + action="store_true", + help=f"Wipe all data from path '{CACHE_CLONE_PATH}'", + ) + parser.add_argument( + "--verbose", + action="store_true", + help=f"More information", + ) + parser.add_argument( + "--ignore_release_date", + action="store_true", + help=( + f"Give result without release date logic, this means it gives" + f" result in past of release version." + ), + ) + parser.add_argument( + "--history_length", + default=0, + type=int, + help=( + f"Set to 0 to clone all history, default 0. Can be another number." + f" This broke other version." + ), + ) + parser.add_argument( + "--more_year", + default=0, + type=int, + help=( + f"At 0, search only present before_date, > 0, add year to" + f" before_date for multiple stat. Depend on param 'before_date'" + ), + ) + parser.add_argument( + "--before_date", + type=lambda d: datetime.datetime.strptime(d, "%Y-%m-%d").date(), + default=None, + help=f"Set a check date before, example '2020-01-01'", + ) + parser.add_argument( + "--filter", + default="/OCA/", + help=f"keyword to ignore separate by ','", + ) + parser.add_argument( + "-b", + "--branches", + default="6.0,6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", + help="Branch to analyse, separate by ','", + ) + args = parser.parse_args() + + die( + args.history_length < 0, + f"Argument 'history_length' need to be positive", + ) + die( + args.more_year < 0, + f"Argument 'more_year' need to be positive", + ) + die( + args.before_date is None and args.more_year > 0, + f"Cannot specify more_year when before_date is not set.", + ) + # clean parameter + args.branches = ",".join([a for a in args.branches.split(",") if a]) + die( + any( + [ + a + for a in args.branches.split(",") + if a not in DCT_VERSION_RELEASE.keys() + ] + ), + "The params branches contain not supported version, check" + f" DCT_VERSION_RELEASE : {list(DCT_VERSION_RELEASE.keys())}", + ) + return args + + +def main(): + config = get_config() + lst_branch = config.branches.split(",") + before_date = config.before_date + + die( + not os.path.isdir(VENV_PATH), + f"Missing {VENV_PATH} venv path, did you install ERPLibre?", + ) + + if config.clean_all and os.path.isdir(CACHE_CLONE_PATH): + _logger.info(f"Delete directory {CACHE_CLONE_PATH}") + shutil.rmtree(CACHE_CLONE_PATH) + + if not os.path.isdir(CACHE_CLONE_PATH): + _logger.info(f"Create directory {CACHE_CLONE_PATH}") + os.mkdir(CACHE_CLONE_PATH) + + lst_repo_url = get_OCA_repo_list(config) + dct_repo = clone_all_repo(config, lst_repo_url) + lst_stat = [] + for nb_year in range(config.more_year + 1): + # key is odoo version, value is set of module name + from_date = before_date + relativedelta(years=nb_year) + if config.ignore_release_date: + lst_branch_adapt = lst_branch + else: + lst_branch_adapt = get_branches_supported_date( + lst_branch, from_date + ) + ( + result, + result_module, + lst_all_unique_module, + dct_reverse_lst_module, + dct_missing_branch, + ) = extract_module_from_all_version( + lst_branch_adapt, from_date, dct_repo + ) + lst_stat.append( + ( + result, + result_module, + lst_all_unique_module, + dct_reverse_lst_module, + dct_missing_branch, + ) + ) + for nb_year, ( + result, + result_module, + lst_all_unique_module, + dct_reverse_lst_module, + dct_missing_branch, + ) in enumerate(lst_stat): + str_extra = "" + + from_date = before_date + relativedelta(years=nb_year) + if config.ignore_release_date: + lst_branch_adapt = lst_branch + else: + lst_branch_adapt = get_branches_supported_date( + lst_branch, from_date + ) + + if before_date: + str_extra = f" before {from_date}" + # Count remove repo + count_remove_repo_stat = 0 + for repo_name, list_branch in dct_missing_branch.items(): + if len(list_branch) == len(lst_branch_adapt): + count_remove_repo_stat += 1 + + print(f"Stat nb module by branch name{str_extra}") + print(f"With {len(lst_repo_url) - count_remove_repo_stat}") + for branch_name in lst_branch_adapt: + if branch_name: + print(f"{branch_name}\t{len(result_module[branch_name])}") + print("end of stat") + + +def die(cond, message, code=1): + if cond: + print(message, file=sys.stderr) + sys.exit(code) + + +def get_branches_supported_date(lst_branch, from_date): + return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date] + + +def extract_module_from_all_version(lst_branch, before_date, dct_repo): + result = defaultdict(dict) + result_module = {} + lst_all_unique_module = set() + dct_reverse_lst_module = defaultdict(list) + dct_missing_branch = defaultdict(list) + + for branch_name in lst_branch: + if not branch_name: + continue + # TODO bug conflict repo name + _logger.info(f"Check branch name '{branch_name}'") + lst_branch_unique_module = set() + result_module[branch_name] = lst_branch_unique_module + remote_branch_name = f"origin/{branch_name}" + for repo_name, repo_git in dct_repo.items(): + try: + repo_git.git.rev_parse("--verify", remote_branch_name) + except: + _logger.warning( + f"Missing branch '{branch_name}' for repo '{repo_name}'" + ) + result[repo_name][branch_name] = None + dct_missing_branch[repo_name].append(branch_name) + continue + if before_date: + checkout_commit = repo_git.git.rev_list( + "-n", "1", "--before", before_date, remote_branch_name + ) + if not checkout_commit: + _logger.warning( + f"Branch '{branch_name}' repo '{repo_name}' contain no" + f" commit before date {before_date}" + ) + continue + else: + checkout_commit = branch_name + # clean before checkout + repo_git.git.stash("save") + repo_git.git.clean("-xdf") + try: + repo_git.git.checkout(checkout_commit) + except Exception as e: + _logger.error( + f"Cannot checkout for branch '{branch_name}' repo" + f" '{repo_name}'" + ) + raise e + lst_unique_module = set() + result[repo_name][branch_name] = lst_unique_module + + suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo_name, "") + repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + suffix_path + for dir_name in os.listdir(repo_path): + path_module_dir = os.path.join(repo_path, dir_name) + path_module_dir_manifest = os.path.join( + path_module_dir, "__manifest__.py" + ) + path_module_dir_openerp = os.path.join( + path_module_dir, "__openerp__.py" + ) + if os.path.isdir(path_module_dir) and ( + os.path.isfile(path_module_dir_manifest) + or os.path.isfile(path_module_dir_openerp) + ): + lst_unique_module.add(dir_name) + lst_all_unique_module.add(dir_name) + lst_branch_unique_module.add(dir_name) + dct_reverse_lst_module[dir_name].append( + f"{branch_name} - {repo_name}" + ) + return ( + result, + result_module, + lst_all_unique_module, + dct_reverse_lst_module, + dct_missing_branch, + ) + + +def clone_all_repo(config, lst_repo): + dct_repo = {} + for i, repo_url in enumerate(lst_repo): + repo_name = repo_url.split("/")[-1][:-4] + repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + _logger.info(f"{i} - Repo '{repo_name}' url '{repo_url}'") + if os.path.isdir(repo_path): + cloned_repo = Repo(repo_path) + assert cloned_repo.__class__ is Repo + else: + if config.history_length: + cloned_repo = Repo.clone_from( + repo_url, repo_path, depth=config.history_length + ) + else: + cloned_repo = Repo.clone_from(repo_url, repo_path) + assert cloned_repo.__class__ is Repo + dct_repo[repo_name] = cloned_repo + return dct_repo + + +def get_OCA_repo_list(config): + with open(CST_FILE_SOURCE_REPO_ADDONS) as file: + all_lines = file.readlines() + if all_lines: + # Validate first line is supported column + expected_header = "url,path,revision,clone-depth\n" + if all_lines[0] != expected_header: + raise Exception( + "Not supported csv, please validate" + f" {CST_FILE_SOURCE_REPO_ADDONS} with first line" + f" {expected_header}" + ) + # Ignore first line + all_lines = all_lines[1:] + + # Be sure empty endline at the end of file + if all_lines[-1][-1] != "\n": + all_lines[-1] = all_lines[-1] + "\n" + all_lines = [a.split(",")[0] for a in all_lines] + # TODO support split(",") + if config.filter: + all_lines = [ + a + for a in all_lines + if config.filter in a and a not in IGNORE_REPO_LIST + ] + else: + all_lines = [a for a in all_lines if a not in IGNORE_REPO_LIST] + return all_lines + + +if __name__ == "__main__": + main() From b7113526723530bdb496b913e230caf028921f5b Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sat, 18 Feb 2023 23:03:22 -0500 Subject: [PATCH 02/17] [FIX] script test parallel: force check in addons when copy in sandbox --- script/test/run_parallel_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test/run_parallel_test.py b/script/test/run_parallel_test.py index dd999a7..de1b041 100755 --- a/script/test/run_parallel_test.py +++ b/script/test/run_parallel_test.py @@ -379,7 +379,7 @@ async def test_exec( for module_name in lst_module_to_test: # Update path to change new emplacement s_lst_path_tested_module = await run_command_get_output( - "find", ".", "-name", module_name + "find", "./addons/", "-name", module_name ) if not s_lst_path_tested_module: return ( @@ -422,7 +422,7 @@ async def test_exec( s_lst_path_generated_module = await run_command_get_output( "find", - ".", + "./addons/", "-name", generated_module, cwd=temp_dir_name, From fa844327fbbe01e6ea93fbae401c80cce6455508 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sat, 18 Feb 2023 23:20:42 -0500 Subject: [PATCH 03/17] [UPD] script test parallel: move async to lib --- script/lib_asyncio.py | 152 +++++++++++++++++++++++++++++ script/test/run_parallel_test.py | 160 ++++++------------------------- 2 files changed, 181 insertions(+), 131 deletions(-) create mode 100644 script/lib_asyncio.py diff --git a/script/lib_asyncio.py b/script/lib_asyncio.py new file mode 100644 index 0000000..7b64e90 --- /dev/null +++ b/script/lib_asyncio.py @@ -0,0 +1,152 @@ +#!./.venv/bin/python +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) +import asyncio +from collections import deque + +import uvloop + + +async def run_in_serial(task_list): + q = asyncio.Queue() + for task in task_list: + await q.put(task) + lst_result = [] + for i in range(len(task_list)): + co = await q.get() + result = await co + lst_result.append(result) + return lst_result + + +async def run_command_get_output(*args, cwd=None): + if cwd is not None: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + else: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + # Wait for the subprocess to finish + stdout, stderr = await process.communicate() + return stdout.decode() + + +async def run_command_get_output_and_status(*args, cwd=None): + if cwd is not None: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + else: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + # Wait for the subprocess to finish + stdout, stderr = await process.communicate() + return stdout.decode(), stderr.decode(), process.returncode + + +def print_summary_task(task_list): + for task in task_list: + print(task.cr_code.co_name) + + +def execute(config, lst_task, use_uvloop=False): + + if not config.no_parallel and asyncio.get_event_loop().is_closed(): + asyncio.set_event_loop(asyncio.new_event_loop()) + + if config.no_parallel: + return_value = asyncio.run(run_in_serial(lst_task)) + elif config.max_process: + pool = AsyncioPool(config.max_process) + for task in lst_task: + pool.add_coro(task) + try: + return_value = pool.run_until_complete() + finally: + pool.close() + else: + # Use maximal resource + if use_uvloop: + uvloop.install() + loop = asyncio.get_event_loop() + if config.debug: + loop.set_debug(True) + try: + commands = asyncio.gather(*lst_task) + return_value = loop.run_until_complete(commands) + finally: + loop.close() + return return_value + + +class AsyncioPool: + # TODO check to replace this pool by https://github.com/dano/aioprocessing + def __init__(self, concurrency, loop=None): + """ + @param loop: asyncio loop + @param concurrency: Maximum number of concurrently running tasks + """ + self._loop = loop or asyncio.get_event_loop() + self._concurrency = concurrency + self._coros = deque([]) # All coroutines queued for execution + self._futures = [] # All currently running coroutines + self._lst_result = [] + + def close(self): + self._loop.close() + + def add_coro(self, coro): + """ + @param coro: coroutine to add + """ + self._coros.append(coro) + self.print_status() + + def run_until_complete(self): + self._loop.run_until_complete(self._wait_for_futures()) + return self._lst_result + + def print_status(self): + print( + " Status: coros:%s - futures:%s" + % (len(self._coros), len(self._futures)) + ) + + def _start_futures(self): + num_to_start = self._concurrency - len(self._futures) + num_to_start = min(num_to_start, len(self._coros)) + for _ in range(num_to_start): + coro = self._coros.popleft() + future = asyncio.ensure_future(coro, loop=self._loop) + self._futures.append(future) + self.print_status() + + async def _wait_for_futures(self): + while len(self._coros) > 0 or len(self._futures) > 0: + self._start_futures() + futures_completed, futures_pending = await asyncio.wait( + self._futures, + loop=self._loop, + return_when=asyncio.FIRST_COMPLETED, + ) + + for future in futures_completed: + self._lst_result.append(future.result()) + self._futures.remove(future) + self._start_futures() diff --git a/script/test/run_parallel_test.py b/script/test/run_parallel_test.py index de1b041..f36bcb0 100755 --- a/script/test/run_parallel_test.py +++ b/script/test/run_parallel_test.py @@ -9,13 +9,19 @@ import sys import tempfile import time import uuid -from collections import deque from typing import Tuple import aioshutil import git from colorama import Fore +new_path = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +sys.path.append(new_path) + +from script import lib_asyncio + logging.basicConfig(level=logging.DEBUG) _logger = logging.getLogger(__name__) @@ -172,27 +178,6 @@ def print_log(lst_task, tpl_result): f.write("\n") -async def run_command_get_output(*args, cwd=None): - if cwd is not None: - process = await asyncio.create_subprocess_exec( - *args, - # stdout must a pipe to be accessible as process.stdout - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=cwd, - ) - else: - process = await asyncio.create_subprocess_exec( - *args, - # stdout must a pipe to be accessible as process.stdout - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - # Wait for the subprocess to finish - stdout, stderr = await process.communicate() - return stdout.decode() - - async def run_command(*args, test_name=None): # Create subprocess start_time = time.time() @@ -378,8 +363,10 @@ async def test_exec( lst_module_to_test = tested_module.split(",") for module_name in lst_module_to_test: # Update path to change new emplacement - s_lst_path_tested_module = await run_command_get_output( - "find", "./addons/", "-name", module_name + s_lst_path_tested_module = ( + await lib_asyncio.run_command_get_output( + "find", "./addons/", "-name", module_name + ) ) if not s_lst_path_tested_module: return ( @@ -420,12 +407,14 @@ async def test_exec( os.path.join(temp_dir_name, s_first_path) ) - s_lst_path_generated_module = await run_command_get_output( - "find", - "./addons/", - "-name", - generated_module, - cwd=temp_dir_name, + s_lst_path_generated_module = ( + await lib_asyncio.run_command_get_output( + "find", + "./addons/", + "-name", + generated_module, + cwd=temp_dir_name, + ) ) if s_lst_path_generated_module: lst_path_generated_module = ( @@ -507,15 +496,19 @@ async def test_exec( hook.write(new_hook_line) # Format editing code before commit - await run_command_get_output( + await lib_asyncio.run_command_get_output( "./script/maintenance/black.sh", temp_dir_name ) # init repo with git for dir_to_git in lst_path_to_add_config: - await run_command_get_output("git", "init", ".", cwd=dir_to_git) - await run_command_get_output("git", "add", ".", cwd=dir_to_git) - await run_command_get_output( + await lib_asyncio.run_command_get_output( + "git", "init", ".", cwd=dir_to_git + ) + await lib_asyncio.run_command_get_output( + "git", "add", ".", cwd=dir_to_git + ) + await lib_asyncio.run_command_get_output( "git", "commit", "-am", "'first commit'", cwd=dir_to_git ) @@ -714,11 +707,6 @@ def check_git_change(): return not status -def print_summary_task(task_list): - for task in task_list: - print(task.cr_code.co_name) - - # START TEST @@ -1173,75 +1161,6 @@ async def run_helloworld_test(config) -> Tuple[str, int]: return res, status -async def run_in_serial(task_list): - q = asyncio.Queue() - for task in task_list: - await q.put(task) - lst_result = [] - for i in range(len(task_list)): - co = await q.get() - result = await co - lst_result.append(result) - return lst_result - - -class AsyncioPool: - # TODO check to replace this pool by https://github.com/dano/aioprocessing - def __init__(self, concurrency, loop=None): - """ - @param loop: asyncio loop - @param concurrency: Maximum number of concurrently running tasks - """ - self._loop = loop or asyncio.get_event_loop() - self._concurrency = concurrency - self._coros = deque([]) # All coroutines queued for execution - self._futures = [] # All currently running coroutines - self._lst_result = [] - - def close(self): - self._loop.close() - - def add_coro(self, coro): - """ - @param coro: coroutine to add - """ - self._coros.append(coro) - self.print_status() - - def run_until_complete(self): - self._loop.run_until_complete(self._wait_for_futures()) - return self._lst_result - - def print_status(self): - print( - " Status: coros:%s - futures:%s" - % (len(self._coros), len(self._futures)) - ) - - def _start_futures(self): - num_to_start = self._concurrency - len(self._futures) - num_to_start = min(num_to_start, len(self._coros)) - for _ in range(num_to_start): - coro = self._coros.popleft() - future = asyncio.ensure_future(coro, loop=self._loop) - self._futures.append(future) - self.print_status() - - async def _wait_for_futures(self): - while len(self._coros) > 0 or len(self._futures) > 0: - self._start_futures() - futures_completed, futures_pending = await asyncio.wait( - self._futures, - loop=self._loop, - return_when=asyncio.FIRST_COMPLETED, - ) - - for future in futures_completed: - self._lst_result.append(future.result()) - self._futures.remove(future) - self._start_futures() - - def run_all_test(config) -> None: # low in time, at the end for more speed task_list = [ @@ -1268,31 +1187,10 @@ def run_all_test(config) -> None: run_demo_test(config), ] - print_summary_task(task_list) + lib_asyncio.print_summary_task(task_list) - if asyncio.get_event_loop().is_closed(): - asyncio.set_event_loop(asyncio.new_event_loop()) + tpl_result = lib_asyncio.execute(config, task_list) - if config.no_parallel: - tpl_result = asyncio.run(run_in_serial(task_list)) - elif config.max_process: - pool = AsyncioPool(config.max_process) - for task in task_list: - pool.add_coro(task) - try: - tpl_result = pool.run_until_complete() - finally: - pool.close() - else: - # Use maximal resource - loop = asyncio.get_event_loop() - if config.debug: - loop.set_debug(True) - try: - commands = asyncio.gather(*task_list) - tpl_result = loop.run_until_complete(commands) - finally: - loop.close() print_log(task_list, tpl_result) status = check_result(task_list, tpl_result) if status: From d6f652f58c6b4edf6218f80625f29748b0ff907a Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sun, 19 Feb 2023 04:23:17 -0500 Subject: [PATCH 04/17] [UPD] script show_evolution_module: parallelism --- Makefile | 11 + poetry.lock | 68 +++- pyproject.toml | 2 + requirements.txt | 5 +- script/statistic/show_evolution_module.py | 376 +++++++++++++--------- 5 files changed, 303 insertions(+), 159 deletions(-) diff --git a/Makefile b/Makefile index ec480b5..420f708 100644 --- a/Makefile +++ b/Makefile @@ -1037,6 +1037,17 @@ i18n_generate_demo_portal: clean: find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete +############### +# Statistic # +############### +.PHONY: stat_module_evolution_per_year +stat_module_evolution_per_year: + ./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7 + +.PHONY: stat_module_evolution_per_year_OCA +stat_module_evolution_per_year_OCA: + ./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7 + ################### # Documentation # ################### diff --git a/poetry.lock b/poetry.lock index b908285..f4964eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5210,6 +5210,27 @@ files = [ {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, ] +[[package]] +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +files = [ + {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"}, + {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typed-ast" version = "1.5.4" @@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "uvloop" +version = "0.17.0" +description = "Fast implementation of asyncio event loop on top of libuv" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"}, + {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"}, + {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"}, + {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"}, + {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"}, + {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"}, + {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"}, + {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"}, + {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"}, + {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"}, + {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"}, + {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"}, + {file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"}, + {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"}, + {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"}, + {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"}, + {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"}, + {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"}, + {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"}, + {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"}, + {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"}, + {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"}, + {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"}, + {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"}, + {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"}, + {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"}, + {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"}, + {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"}, + {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"}, + {file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"}, +] + +[package.extras] +dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"] + [[package]] name = "validators" version = "0.20.0" @@ -5879,4 +5945,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.7.16" -content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e" +content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368" diff --git a/pyproject.toml b/pyproject.toml index 6935dd9..6a696c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,8 @@ xmltodict = "^0.13.0" zeep = "^4.2.1" zipp = ">=3.6.0" zxcvbn = "^4.4.28" +tqdm = "^4.64.1" +uvloop = "^0.17.0" [tool.poetry.dev-dependencies] websocket_client = "^1.1.0" diff --git a/requirements.txt b/requirements.txt index 441cfc7..04cc620 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ openupgradelib unidiff colorama wheel -aioshutil<1.2 # For OSX cython @@ -80,6 +79,10 @@ numpy==1.21.1 feedparser==6.0.10 geojson<3.0.0 +# asyncio +aioshutil<1.2 +tqdm +uvloop # Solve pcodedmp, oletools, rtfde and extract-msg win_unicode_console diff --git a/script/statistic/show_evolution_module.py b/script/statistic/show_evolution_module.py index 1b7bb2d..7741fe6 100755 --- a/script/statistic/show_evolution_module.py +++ b/script/statistic/show_evolution_module.py @@ -8,28 +8,35 @@ import sys from collections import defaultdict from dateutil.relativedelta import relativedelta -from git import Repo -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +new_path = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +sys.path.append(new_path) + +from script import lib_asyncio + +FORMAT = "%(asctime)s [%(levelname)s] %(message)s" +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT) _logger = logging.getLogger(__name__) PROJECT_NAME = os.path.basename(os.getcwd()) VENV_PATH = "./.venv" -CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_OCA") +CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo") CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv" +MAX_COROUTINE = 300 IGNORE_REPO_LIST = ( "https://github.com/OCA/odoo-module-migrator.git", "https://github.com/OCA/maintainer-tools.git", - "https://github.com/OCA/connector.git", "https://github.com/itpp-labs/odoo-development.git", "https://github.com/itpp-labs/odoo-port-docs.git", "https://github.com/itpp-labs/odoo-test-docs.git", "https://github.com/odoo/documentation.git", - "https://github.com/OCA/odoo-module-migrator.git", - "https://github.com/OCA/maintainer-tools.git", + "https://github.com/ERPLibre/ERPLibre_image_db.git", + "https://github.com/muk-it/muk_docs.git", ) -DCT_CHANGE_ADDONS_PATH = {"odoo": "/addons/"} +DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": "/addons/"} DCT_VERSION_RELEASE = { "1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(), "2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(), @@ -72,6 +79,11 @@ def get_config(): action="store_true", help=f"More information", ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable asyncio debugging", + ) parser.add_argument( "--ignore_release_date", action="store_true", @@ -80,6 +92,21 @@ def get_config(): f" result in past of release version." ), ) + parser.add_argument( + "--no_parallel", + action="store_true", + help=( + f"By default run in parallel with all CPU, this disable" + f" parallelism with asyncio" + ), + ) + parser.add_argument( + "-p", + "--max_process", + type=int, + default=0, + help="Max processor to use. If 0, use max.", + ) parser.add_argument( "--history_length", default=0, @@ -106,13 +133,13 @@ def get_config(): ) parser.add_argument( "--filter", - default="/OCA/", - help=f"keyword to ignore separate by ','", + default="", + help=f"keyword to ignore separate by ','. Suggest /OCA/", ) parser.add_argument( "-b", "--branches", - default="6.0,6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", + default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", help="Branch to analyse, separate by ','", ) args = parser.parse_args() @@ -149,12 +176,16 @@ def main(): config = get_config() lst_branch = config.branches.split(",") before_date = config.before_date + lst_stat = [] + # Check die( not os.path.isdir(VENV_PATH), f"Missing {VENV_PATH} venv path, did you install ERPLibre?", ) + # Setup environment + _logger.info(f"Setup {CACHE_CLONE_PATH}") if config.clean_all and os.path.isdir(CACHE_CLONE_PATH): _logger.info(f"Delete directory {CACHE_CLONE_PATH}") shutil.rmtree(CACHE_CLONE_PATH) @@ -163,66 +194,79 @@ def main(): _logger.info(f"Create directory {CACHE_CLONE_PATH}") os.mkdir(CACHE_CLONE_PATH) + # Get list of Repo lst_repo_url = get_OCA_repo_list(config) - dct_repo = clone_all_repo(config, lst_repo_url) - lst_stat = [] + + # Clone all repo + _logger.info(f"Clone repo") + lst_task_clone = [clone_repo(i, a) for i, a in enumerate(lst_repo_url)] + lst_repo_path = lib_asyncio.execute( + config, lst_task_clone, use_uvloop=True + ) + + # Extract module list + _logger.info("Extract module") + lst_task = [] for nb_year in range(config.more_year + 1): # key is odoo version, value is set of module name - from_date = before_date + relativedelta(years=nb_year) - if config.ignore_release_date: - lst_branch_adapt = lst_branch - else: - lst_branch_adapt = get_branches_supported_date( - lst_branch, from_date - ) - ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) = extract_module_from_all_version( - lst_branch_adapt, from_date, dct_repo - ) - lst_stat.append( - ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) - ) - for nb_year, ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) in enumerate(lst_stat): - str_extra = "" - - from_date = before_date + relativedelta(years=nb_year) - if config.ignore_release_date: - lst_branch_adapt = lst_branch - else: - lst_branch_adapt = get_branches_supported_date( - lst_branch, from_date - ) - + lst_branch_adapt = lst_branch + from_date = before_date if before_date: - str_extra = f" before {from_date}" - # Count remove repo - count_remove_repo_stat = 0 - for repo_name, list_branch in dct_missing_branch.items(): - if len(list_branch) == len(lst_branch_adapt): - count_remove_repo_stat += 1 + from_date += relativedelta(years=nb_year) + if not config.ignore_release_date: + lst_branch_adapt = get_branches_supported_date( + lst_branch, from_date + ) + # for branch in lst_branch_adapt: + for repo_name, repo_path in lst_repo_path: + lst_task.append( + extract_module( + lst_branch_adapt, repo_name, repo_path, from_date + ) + ) + if config.debug: + _logger.info( + "Setup extract module:" + f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}" + ) + + _logger.info(f"Execute extract module with {len(lst_task)} coroutine.") + # Need to set a max, or crash + if len(lst_task) / MAX_COROUTINE > 1: + lst_result = [] + for i in range(int(len(lst_task) / MAX_COROUTINE) + 1): + min_i = i * MAX_COROUTINE + max_i = min((i + 1) * MAX_COROUTINE, len(lst_task)) + _logger.info(f"Partial execution {min_i} to {max_i}") + tpl_result = lib_asyncio.execute( + config, lst_task[min_i:max_i], use_uvloop=True + ) + lst_result += tpl_result + tpl_result = lst_result + else: + tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True) + _logger.info("Analyse information") + dct_result = defaultdict(lambda: defaultdict(int)) + for lst_result_stack in tpl_result: + for dct_result_module in lst_result_stack: + # lst_module, branch, path, repo, before_date + before_date = dct_result_module.get("before_date") + lst_module = dct_result_module.get("lst_module") + branch = dct_result_module.get("branch") + # path = dct_result_module.get("path") + # repo = dct_result_module.get("repo") + dct_result[before_date][branch] += len(lst_module) + + # Show result + _logger.info("Show result") + for before_date, dct_branch_result in dct_result.items(): + str_extra = "" + if before_date: + str_extra = f" before {before_date}" print(f"Stat nb module by branch name{str_extra}") - print(f"With {len(lst_repo_url) - count_remove_repo_stat}") - for branch_name in lst_branch_adapt: - if branch_name: - print(f"{branch_name}\t{len(result_module[branch_name])}") + for branch_name, count_module in dct_branch_result.items(): + print(f"{branch_name}\t{count_module}") print("end of stat") @@ -236,105 +280,123 @@ def get_branches_supported_date(lst_branch, from_date): return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date] -def extract_module_from_all_version(lst_branch, before_date, dct_repo): - result = defaultdict(dict) - result_module = {} - lst_all_unique_module = set() - dct_reverse_lst_module = defaultdict(list) - dct_missing_branch = defaultdict(list) +async def extract_module(lst_branch, repo, path, before_date): + # print(f"{branch} - {path} - {before_date}") + lst_result = [] + for branch in lst_branch: + return_value = { + "lst_module": [], + "branch": branch, + "path": path, + "repo": repo, + "before_date": before_date, + } + remote_branch_name = f"origin/{branch}" - for branch_name in lst_branch: - if not branch_name: + # Detect commit to check + # git rev-list -n 1 --before 2022-01-01 origin/12.0 + lst_args = ["git", "rev-list", "-n", "1"] + if before_date: + lst_args += ["--before", str(before_date)] + lst_args += [remote_branch_name] + ( + commit, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=path + ) + + if status: + # The branch doesn't exist + # _logger.error(f"Receive status 'git rev-list' {status}") continue - # TODO bug conflict repo name - _logger.info(f"Check branch name '{branch_name}'") - lst_branch_unique_module = set() - result_module[branch_name] = lst_branch_unique_module - remote_branch_name = f"origin/{branch_name}" - for repo_name, repo_git in dct_repo.items(): - try: - repo_git.git.rev_parse("--verify", remote_branch_name) - except: - _logger.warning( - f"Missing branch '{branch_name}' for repo '{repo_name}'" + + # TODO send msg when no result + if not commit: + continue + + commit = commit.strip() + + # List directory + suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "") + adapt_path = path + suffix_path + lst_args = ["git", "ls-tree", "-d", "--name-only", commit] + ( + str_folders, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=adapt_path + ) + if status: + _logger.error(f"Receive status 'git ls-tree' {status}") + continue + + # TODO send msg when no result + if not str_folders: + continue + lst_module = str_folders.split() + if not lst_module: + continue + + if suffix_path: + suffix_path = suffix_path[1:] + + lst_module_installable = [] + for module in lst_module: + # List directory + lst_args = [ + "git", + "cat-file", + "-p", + f"{commit}:{suffix_path}{module}/__manifest__.py", + ] + ( + manifest, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=path + ) + if status: + # Maybe __openerp__.py + lst_args = [ + "git", + "cat-file", + "-p", + f"{commit}:{module}/__openerp__.py", + ] + ( + manifest, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=adapt_path ) - result[repo_name][branch_name] = None - dct_missing_branch[repo_name].append(branch_name) + if status: + # Ignore, it's not a module + # _logger.error(f"Receive status 'git cat-file' {status}") + # return return_value continue - if before_date: - checkout_commit = repo_git.git.rev_list( - "-n", "1", "--before", before_date, remote_branch_name - ) - if not checkout_commit: - _logger.warning( - f"Branch '{branch_name}' repo '{repo_name}' contain no" - f" commit before date {before_date}" - ) - continue - else: - checkout_commit = branch_name - # clean before checkout - repo_git.git.stash("save") - repo_git.git.clean("-xdf") - try: - repo_git.git.checkout(checkout_commit) - except Exception as e: - _logger.error( - f"Cannot checkout for branch '{branch_name}' repo" - f" '{repo_name}'" - ) - raise e - lst_unique_module = set() - result[repo_name][branch_name] = lst_unique_module - - suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo_name, "") - repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + suffix_path - for dir_name in os.listdir(repo_path): - path_module_dir = os.path.join(repo_path, dir_name) - path_module_dir_manifest = os.path.join( - path_module_dir, "__manifest__.py" - ) - path_module_dir_openerp = os.path.join( - path_module_dir, "__openerp__.py" - ) - if os.path.isdir(path_module_dir) and ( - os.path.isfile(path_module_dir_manifest) - or os.path.isfile(path_module_dir_openerp) - ): - lst_unique_module.add(dir_name) - lst_all_unique_module.add(dir_name) - lst_branch_unique_module.add(dir_name) - dct_reverse_lst_module[dir_name].append( - f"{branch_name} - {repo_name}" - ) - return ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) + dct_manifest = eval(manifest + "\n") + if dct_manifest.get("installable", True): + lst_module_installable.append(module) + return_value["lst_module"] = lst_module_installable + lst_result.append(return_value) + return lst_result -def clone_all_repo(config, lst_repo): - dct_repo = {} - for i, repo_url in enumerate(lst_repo): - repo_name = repo_url.split("/")[-1][:-4] - repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) - _logger.info(f"{i} - Repo '{repo_name}' url '{repo_url}'") - if os.path.isdir(repo_path): - cloned_repo = Repo(repo_path) - assert cloned_repo.__class__ is Repo - else: - if config.history_length: - cloned_repo = Repo.clone_from( - repo_url, repo_path, depth=config.history_length - ) - else: - cloned_repo = Repo.clone_from(repo_url, repo_path) - assert cloned_repo.__class__ is Repo - dct_repo[repo_name] = cloned_repo - return dct_repo +async def clone_repo(i, repo_url): + split_repo_url = repo_url.split("/") + repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}" + repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + if not os.path.isdir(repo_path): + await lib_asyncio.run_command_get_output( + "git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH + ) + _logger.info(f"[{i}]New clone {repo_path} with {repo_url}") + return repo_name, repo_path def get_OCA_repo_list(config): From 8dd49e44479f01ae1dc960ce8fb8edf6f1ac4b4d Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sun, 19 Feb 2023 23:03:43 -0500 Subject: [PATCH 05/17] [UPD] script show_evolution_module: compare CSV information - fix odoo_odoo addons to include test - add option to force fetch. - add check uninstall module - add stat unique module --- script/statistic/show_evolution_module.py | 185 ++++++++++++++++------ 1 file changed, 135 insertions(+), 50 deletions(-) diff --git a/script/statistic/show_evolution_module.py b/script/statistic/show_evolution_module.py index 7741fe6..5b06779 100755 --- a/script/statistic/show_evolution_module.py +++ b/script/statistic/show_evolution_module.py @@ -5,6 +5,7 @@ import logging import os import shutil import sys +import csv from collections import defaultdict from dateutil.relativedelta import relativedelta @@ -36,7 +37,7 @@ IGNORE_REPO_LIST = ( "https://github.com/ERPLibre/ERPLibre_image_db.git", "https://github.com/muk-it/muk_docs.git", ) -DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": "/addons/"} +DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": ["/addons/", "/odoo/addons/"]} DCT_VERSION_RELEASE = { "1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(), "2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(), @@ -56,6 +57,7 @@ DCT_VERSION_RELEASE = { "15.0": datetime.datetime.strptime("2021-10-01", "%Y-%m-%d").date(), "16.0": datetime.datetime.strptime("2022-10-01", "%Y-%m-%d").date(), } +CSV_HEADER_MODULE_NAME = "Nom technique" # TODO use rich progress bar when clone # TODO support conflict repo name @@ -100,6 +102,11 @@ def get_config(): f" parallelism with asyncio" ), ) + parser.add_argument( + "--force_git_fetch", + action="store_true", + help=f"Force git fetch to update it all remote.", + ) parser.add_argument( "-p", "--max_process", @@ -142,6 +149,14 @@ def get_config(): default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", help="Branch to analyse, separate by ','", ) + parser.add_argument( + "--compare_csv", + help=( + "Path of CSV, search column 'Nom technique' and give list of" + " difference module, missing from CSV. Need only 1 branches and no" + " more_year." + ), + ) args = parser.parse_args() die( @@ -169,6 +184,16 @@ def get_config(): "The params branches contain not supported version, check" f" DCT_VERSION_RELEASE : {list(DCT_VERSION_RELEASE.keys())}", ) + die( + args.compare_csv + and (args.more_year > 0 or len(args.branches.split(",")) > 1), + f"When use param compare_csv, cannot use more_year and only 1 branch.", + ) + if args.compare_csv: + die( + not os.path.isfile(args.compare_csv), + f"Path of {args.compare_csv} need to be a file path.", + ) return args @@ -176,7 +201,8 @@ def main(): config = get_config() lst_branch = config.branches.split(",") before_date = config.before_date - lst_stat = [] + lst_unique_module = set() + lst_unique_uninstallable_module = set() # Check die( @@ -199,7 +225,10 @@ def main(): # Clone all repo _logger.info(f"Clone repo") - lst_task_clone = [clone_repo(i, a) for i, a in enumerate(lst_repo_url)] + lst_task_clone = [ + clone_repo(i, a, config.force_git_fetch) + for i, a in enumerate(lst_repo_url) + ] lst_repo_path = lib_asyncio.execute( config, lst_task_clone, use_uvloop=True ) @@ -248,15 +277,22 @@ def main(): tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True) _logger.info("Analyse information") dct_result = defaultdict(lambda: defaultdict(int)) + dct_result_unique = defaultdict(set) for lst_result_stack in tpl_result: for dct_result_module in lst_result_stack: # lst_module, branch, path, repo, before_date before_date = dct_result_module.get("before_date") lst_module = dct_result_module.get("lst_module") + lst_uninstallable_module = dct_result_module.get( + "lst_uninstallable_module" + ) branch = dct_result_module.get("branch") # path = dct_result_module.get("path") # repo = dct_result_module.get("repo") dct_result[before_date][branch] += len(lst_module) + dct_result_unique[before_date].update(lst_module) + lst_unique_module.update(lst_module) + lst_unique_uninstallable_module.update(lst_uninstallable_module) # Show result _logger.info("Show result") @@ -267,8 +303,43 @@ def main(): print(f"Stat nb module by branch name{str_extra}") for branch_name, count_module in dct_branch_result.items(): print(f"{branch_name}\t{count_module}") + for before_date, set_result in dct_result_unique.items(): + str_extra = "" + if before_date: + str_extra = f" before {before_date}" + print(f"Diff nb unique module{str_extra}, length {len(set_result)}") print("end of stat") + if config.compare_csv: + index_header = -1 + lst_unique_module_csv = set() + with open(config.compare_csv, "r") as csvfile: + for csv_module in csv.reader(csvfile): + if index_header == -1: + try: + index_header = csv_module.index(CSV_HEADER_MODULE_NAME) + except Exception as e: + _logger.error( + f"Missing header {CSV_HEADER_MODULE_NAME} in CSV." + ) + raise e + else: + module_name = csv_module[index_header] + if module_name not in lst_unique_uninstallable_module: + lst_unique_module_csv.add(module_name) + + lst_missing_module_repo = lst_unique_module_csv.difference( + lst_unique_module + ) + lst_missing_module_csv = lst_unique_module.difference( + lst_unique_module_csv + ) + print("CSV compare") + print(f"Module missing from CSV '{len(lst_missing_module_repo)}':") + print(lst_missing_module_repo) + print(f"Module missing into CSV '{len(lst_missing_module_csv)}':") + print(lst_missing_module_csv) + def die(cond, message, code=1): if cond: @@ -318,76 +389,84 @@ async def extract_module(lst_branch, repo, path, before_date): commit = commit.strip() - # List directory - suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "") - adapt_path = path + suffix_path - lst_args = ["git", "ls-tree", "-d", "--name-only", commit] - ( - str_folders, - _, - status, - ) = await lib_asyncio.run_command_get_output_and_status( - *lst_args, cwd=adapt_path - ) - if status: - _logger.error(f"Receive status 'git ls-tree' {status}") - continue - - # TODO send msg when no result - if not str_folders: - continue - lst_module = str_folders.split() - if not lst_module: - continue - - if suffix_path: - suffix_path = suffix_path[1:] - lst_module_installable = [] - for module in lst_module: - # List directory - lst_args = [ - "git", - "cat-file", - "-p", - f"{commit}:{suffix_path}{module}/__manifest__.py", - ] + lst_module_uninstallable = [] + # List directory + lst_suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "") + if type(lst_suffix_path) is str: + lst_suffix_path = [lst_suffix_path] + for suffix_path in lst_suffix_path: + adapt_path = path + suffix_path + lst_args = ["git", "ls-tree", "-d", "--name-only", commit] ( - manifest, + str_folders, _, status, ) = await lib_asyncio.run_command_get_output_and_status( - *lst_args, cwd=path + *lst_args, cwd=adapt_path ) if status: - # Maybe __openerp__.py + _logger.error(f"Receive status 'git ls-tree' {status}") + continue + + # TODO send msg when no result + if not str_folders: + continue + lst_module = str_folders.split() + if not lst_module: + continue + + if suffix_path: + suffix_path = suffix_path[1:] + + for module in lst_module: + # List directory lst_args = [ "git", "cat-file", "-p", - f"{commit}:{module}/__openerp__.py", + f"{commit}:{suffix_path}{module}/__manifest__.py", ] ( manifest, _, status, ) = await lib_asyncio.run_command_get_output_and_status( - *lst_args, cwd=adapt_path + *lst_args, cwd=path ) - if status: - # Ignore, it's not a module - # _logger.error(f"Receive status 'git cat-file' {status}") - # return return_value - continue - dct_manifest = eval(manifest + "\n") - if dct_manifest.get("installable", True): - lst_module_installable.append(module) + if status: + # Maybe __openerp__.py + lst_args = [ + "git", + "cat-file", + "-p", + f"{commit}:{module}/__openerp__.py", + ] + ( + manifest, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=adapt_path + ) + if status: + # Ignore, it's not a module + # _logger.error(f"Receive status 'git cat-file' {status}") + # return return_value + continue + dct_manifest = eval(manifest + "\n") + if dct_manifest.get("installable", True): + lst_module_installable.append(module) + else: + lst_module_uninstallable.append(module) + return_value["lst_module"] = lst_module_installable + return_value["lst_uninstallable_module"] = lst_module_uninstallable lst_result.append(return_value) return lst_result -async def clone_repo(i, repo_url): +async def clone_repo(i, repo_url, force_fetch): split_repo_url = repo_url.split("/") repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}" repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) @@ -396,6 +475,12 @@ async def clone_repo(i, repo_url): "git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH ) _logger.info(f"[{i}]New clone {repo_path} with {repo_url}") + elif force_fetch: + await lib_asyncio.run_command_get_output( + "git", "fetch", "--all", cwd=CACHE_CLONE_PATH + ) + _logger.info(f"[{i}]Fetch {repo_path} with {repo_url}") + return repo_name, repo_path From 451358ff058b0bcb681ab3d960728bf4095ff527 Mon Sep 17 00:00:00 2001 From: Alexandre Ferreira Benevides Date: Wed, 22 Feb 2023 14:58:03 -0500 Subject: [PATCH 06/17] [UPD] install_OSX_dependency.sh: remove poetry, pyenv add parallel --- script/install/install_OSX_dependency.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/script/install/install_OSX_dependency.sh b/script/install/install_OSX_dependency.sh index 3480572..8319814 100755 --- a/script/install/install_OSX_dependency.sh +++ b/script/install/install_OSX_dependency.sh @@ -25,23 +25,21 @@ sudo su - postgres -c "CREATE EXTENSION postgis;\nCREATE EXTENSION postgis_topol # Install Dependencies #-------------------------------------------------- echo "\n--- Installing Python 3 + pip3 --" -brew install git python3 wget pyenv +brew install git python3 wget brew link git brew link wget + +echo "\n--- Installing extra --" +brew install parallel echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----" brew install nodejs npm openssl sudo npm install -g rtlcss sudo npm install -g less sudo npm install -g prettier sudo npm install -g prettier @prettier/plugin-xml -yes n|pyenv install 3.7.16 -pyenv local 3.7.16 echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.zshrc -echo -e "\n---- Installing poetry for reliable python package ----" -curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - #-------------------------------------------------- # Install Wkhtmltopdf if needed #-------------------------------------------------- From 42b0ccee97a802478c8bacdb8a2f57b6bf43fb03 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Thu, 2 Mar 2023 00:39:27 -0500 Subject: [PATCH 07/17] [UPD] github funding --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 1940626..f66958e 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: mathben963 # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: mathben # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username From e87a307d5636842bd0fdc5dea9b960285cf7c5b3 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Fri, 3 Mar 2023 16:21:54 -0500 Subject: [PATCH 08/17] [ADD] script code_generator: transform_xml_data_website_page_to_controller - need to add auth="user" to website.page, so need to support it in ctrl --- ...orm_xml_data_website_page_to_controller.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 script/code_generator/technical/transform_xml_data_website_page_to_controller.py diff --git a/script/code_generator/technical/transform_xml_data_website_page_to_controller.py b/script/code_generator/technical/transform_xml_data_website_page_to_controller.py new file mode 100644 index 0000000..9a30636 --- /dev/null +++ b/script/code_generator/technical/transform_xml_data_website_page_to_controller.py @@ -0,0 +1,149 @@ +#!./.venv/bin/python +import argparse +import logging +import os +import sys + +import black +import xmltodict + +logging.basicConfig(level=logging.DEBUG) +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + Transform an XML into Python +""", + epilog="""\ +""", + ) + parser.add_argument( + "--source", + dest="source", + required=True, + help="File to transform (XML)", + ) + # parser.add_argument( + # "--destination", + # dest="destination", + # help=( + # "Result, output (Python). If not set, output will be console." + # ), + # ) + # parser.add_argument( + # "-q", + # "--quiet", + # action="store_true", + # help="Don't show output of found model.", + # ) + args = parser.parse_args() + if not os.path.isfile(args.source): + raise Exception("Argument --source need to be a valid path to a file.") + return args + + +def get_module_name(filepath): + manifest_file = "__manifest__.py" + current_path = os.path.abspath(os.path.dirname(filepath)) + while current_path != "/": + manifest_path = os.path.join(current_path, manifest_file) + if os.path.isfile(manifest_path): + return os.path.basename(current_path) + current_path = os.path.dirname(current_path) + raise FileNotFoundError(f"No {manifest_file} found for {filepath}") + + +def get_value_from_field(lst_field, by_attr, name, record_id, get_attr): + field_info = [a for a in lst_field if a.get(by_attr) == name] + if not field_info: + raise ValueError( + f"Missing from record {record_id}" + ) + field_info = field_info[0] + return field_info.get(get_attr) + + +def main(): + config = get_config() + with open(config.source) as xml: + xml_as_string = xml.read() + xml_dict = xmltodict.parse(xml_as_string) + # TODO replace this by get_value(xml_dict, "odoo.data.record") + xml_dict_odoo = xml_dict.get("odoo") + if not xml_dict_odoo: + raise ValueError(f"Missing into {config.source}") + xml_dict_data = xml_dict_odoo.get("data") + if not xml_dict_data: + raise ValueError(f"Missing into {config.source}") + xml_dict_record = xml_dict_data.get("record") + if not xml_dict_record: + raise ValueError(f"Missing into {config.source}") + if type(xml_dict_record) is not list: + lst_xml_record = [xml_dict_record] + else: + lst_xml_record = xml_dict_record + lst_output = [] + for record in lst_xml_record: + lst_field = record.get("field") + if type(lst_field) is not list: + lst_field = [lst_field] + + # Condition unique + try: + get_value_from_field( + lst_field, "@name", "groups_id", record.get("@id"), "@eval" + ) + except: + continue + + url = get_value_from_field( + lst_field, "@name", "url", record.get("@id"), "#text" + ) + view_id = get_value_from_field( + lst_field, "@name", "view_id", record.get("@id"), "@ref" + ) + module_name = get_module_name(config.source) + method_name = ( + f'get_{"_".join(url.replace("-", "_").strip("/").split("/"))}' + ) + +# template = f""" +# @http.route( +# ["{url}"], +# type="http", +# auth="user", +# website=True, +# ) +# def {method_name}(self, **kw): +# return request.env["ir.ui.view"].render_template( +# "{module_name}.{view_id}", +# ) + template = f""" +@http.route( + ["{url}"], + type="http", + auth="user", + website=True, +) +def {method_name}(self, **kw): + return request.env.ref('{module_name}.{view_id}').render() +""" + lst_output.append(template) + str_output = "\n".join(lst_output) + formated_str_output = black.format_str(str_output, mode=black.Mode()) + print(formated_str_output) + return 0 + + +if __name__ == "__main__": + status = main() + sys.exit(status) From aa7689edec115b4a1514d8fd91559d26073f8523 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sat, 4 Mar 2023 23:31:41 -0500 Subject: [PATCH 09/17] [FIX] make test with coverage: missing coverage parameter --- Makefile | 19 ++++++++++++++++++- test/code_generator/coverage_hello_world.sh | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 420f708..1327705 100644 --- a/Makefile +++ b/Makefile @@ -616,12 +616,25 @@ test_full_fast_coverage: ./.venv/bin/coverage erase ./script/test/run_parallel_test.py --coverage # TODO This test is broken in parallel - ./script/make.sh test_code_generator_hello_world + ./script/make.sh test_coverage_code_generator_hello_world ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" # run: make open_test_coverage + +.PHONY: test_cg_demo +test_cg_demo: + ./script/make.sh clean + # Need to create a BD to create cache _cache_erplibre_base + ./script/database/db_restore.py --database test + ./.venv/bin/coverage erase + ./script/addons/coverage_install_addons_dev.sh test code_generator_demo + ./.venv/bin/coverage combine -a + ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + + .PHONY: test_base test_base: ./script/make.sh test_format @@ -645,6 +658,10 @@ test_format: test_code_generator_hello_world: ./test/code_generator/hello_world.sh +.PHONY: test_coverage_code_generator_hello_world +test_coverage_code_generator_hello_world: + ./test/code_generator/coverage_hello_world.sh + .PHONY: test_installation_demo test_installation_demo: ./script/code_generator/check_git_change_code_generator.sh ./addons/TechnoLibre_odoo-code-generator-template diff --git a/test/code_generator/coverage_hello_world.sh b/test/code_generator/coverage_hello_world.sh index 4b4bda2..883facf 100755 --- a/test/code_generator/coverage_hello_world.sh +++ b/test/code_generator/coverage_hello_world.sh @@ -6,7 +6,7 @@ Green='\033[0;32m' # Green tmp_dir=$(mktemp -d -t "erplibre_code_generator_helloworld-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX") -./script/code_generator/new_project.py -m test -d "${tmp_dir}" +./script/code_generator/new_project.py -m test -d "${tmp_dir}" --coverage MODULE_PATH="${tmp_dir}/test" if [[ -d "${MODULE_PATH}" ]]; then From 90229ad155e9b997dcd9a715fcc5947f240e0142 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sun, 5 Mar 2023 01:53:30 -0500 Subject: [PATCH 10/17] [ADD] coverage json --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 1327705..e2db312 100644 --- a/Makefile +++ b/Makefile @@ -620,6 +620,7 @@ test_full_fast_coverage: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*" # run: make open_test_coverage @@ -633,6 +634,7 @@ test_cg_demo: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*" .PHONY: test_base @@ -769,6 +771,7 @@ test_addons_sale: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m ./.venv/bin/coverage html + ./.venv/bin/coverage json .PHONY: test_addons_helpdesk test_addons_helpdesk: @@ -778,6 +781,7 @@ test_addons_helpdesk: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m ./.venv/bin/coverage html + ./.venv/bin/coverage json .PHONY: test_addons_code_generator test_addons_code_generator: @@ -788,6 +792,7 @@ test_addons_code_generator: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*" # run: make open_test_coverage .PHONY: test_addons_code_generator_code_generator @@ -799,6 +804,7 @@ test_addons_code_generator_code_generator: ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*" # run: make open_test_coverage .PHONY: open_test_coverage From de4c7196d2a5e51122fb1d6876d07614441fb7d5 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Tue, 7 Mar 2023 15:41:24 -0500 Subject: [PATCH 11/17] [UPD] doc FAQ: add OSX error current limit exceeds maximum limit --- doc/FAQ.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/FAQ.md b/doc/FAQ.md index ea13d20..f94565a 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -124,3 +124,19 @@ docker-machine create --driver virtualbox default docker-machine env default eval "$(docker-machine env default)" ``` + +### Run error `current limit exceeds maximum limit` + +When you got + +``` +File "/Users/test/Desktop/ERPLibre/odoo/odoo/service/server.py", line 83, in set_limit_memory_hard +resource.setrlimit(rlimit, (config['limit_memory_hard'], hard)) +ValueError: current limit exceeds maximum limit +``` + +Add line at the end of config.conf + +``` +limit_memory_hard = 0 +``` From b3e8ce608b97a0c221e6d21d7691d24884150cc8 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Tue, 7 Mar 2023 15:42:40 -0500 Subject: [PATCH 12/17] [UPD] makefile test coverage: add test template_code_generator and fix dev --- Makefile | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e2db312..73af3c3 100644 --- a/Makefile +++ b/Makefile @@ -788,7 +788,7 @@ test_addons_code_generator: ./.venv/bin/coverage erase ./.venv/bin/python3 ./odoo/odoo-bin db --drop --database test_addons_code_generator # TODO missing test in code_generator - ./test.sh --dev all -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator + ./test.sh --dev cg -d test_addons_code_generator --db-filter test_addons_code_generator -i code_generator ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" @@ -800,7 +800,20 @@ test_addons_code_generator_code_generator: # TODO this test only generation, not test ./.venv/bin/coverage erase ./script/database/db_restore.py --database test_addons_code_generator_code_generator - ./test.sh --dev all -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator + ./test.sh --dev cg -d test_addons_code_generator_code_generator --db-filter test_addons_code_generator_code_generator -i code_generator_code_generator + ./.venv/bin/coverage combine -a + ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" + ./.venv/bin/coverage json --include="addons/TechnoLibre_odoo-code-generator/*" + # run: make open_test_coverage + +.PHONY: test_addons_code_generator_template_code_generator +test_addons_code_generator_template_code_generator: + # TODO this test only generation, not test + ./.venv/bin/coverage erase + ./script/database/db_restore.py --database test_addons_code_generator_template_code_generator + ./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator + ./test.sh --dev cg -d test_addons_code_generator_template_code_generator --db-filter test_addons_code_generator_template_code_generator -i code_generator_template_code_generator ./.venv/bin/coverage combine -a ./.venv/bin/coverage report -m --include="addons/TechnoLibre_odoo-code-generator/*" ./.venv/bin/coverage html --include="addons/TechnoLibre_odoo-code-generator/*" From ffa4cbb1607bd876e9211bbc98df558888146041 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Tue, 7 Mar 2023 15:43:27 -0500 Subject: [PATCH 13/17] [UPD] .gitignore: remove coverage.json --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 03104fb..c484315 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ config.conf *.DS_Store *.pyc .coverage* +coverage* .repo !addons/ From 8275e149ee73aef551632f2ded2cd37c67af6461 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Wed, 8 Mar 2023 12:02:53 -0500 Subject: [PATCH 14/17] [ADD] run_parallel_test: missing test demo_internal --- script/test/run_parallel_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/script/test/run_parallel_test.py b/script/test/run_parallel_test.py index f36bcb0..7891dc1 100755 --- a/script/test/run_parallel_test.py +++ b/script/test/run_parallel_test.py @@ -1103,6 +1103,29 @@ async def run_code_generator_template_demo_portal_test( return test_result, test_status +async def run_code_generator_template_demo_internal_test( + config, +) -> Tuple[str, int]: + test_result = "" + test_status = 0 + # Template + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module="code_generator_demo_internal", + tested_module="code_generator_template_demo_internal", + search_class_module="demo_internal", + lst_init_module_name=[ + "demo_internal", + ], + test_name="code_generator_template_demo_internal", + run_in_sandbox=True, + ) + test_result += res + test_status += status + + return test_result, test_status + async def run_code_generator_template_demo_internal_inherit_test( config, ) -> Tuple[str, int]: @@ -1169,6 +1192,7 @@ def run_all_test(config) -> None: run_code_generator_demo_mariadb_sql_example_1_test(config), run_code_generator_auto_backup_test(config), run_code_generator_template_demo_portal_test(config), + run_code_generator_template_demo_internal_test(config), run_code_generator_template_demo_internal_inherit_test(config), run_code_generator_template_demo_sysadmin_cron_test(config), run_code_generator_demo_test(config), From cb3b2fe862eab8414e9ef818e715a9709598839d Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sun, 12 Mar 2023 17:46:00 -0400 Subject: [PATCH 15/17] [FIX] script update_prod_to_dev.sh: need install_addons_dev.sh --- script/addons/update_prod_to_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/addons/update_prod_to_dev.sh b/script/addons/update_prod_to_dev.sh index 071c701..6b948eb 100755 --- a/script/addons/update_prod_to_dev.sh +++ b/script/addons/update_prod_to_dev.sh @@ -2,7 +2,7 @@ # This script will remove mail configuration, remove backup configuration, and force admin user to test/test echo "Update prod to dev on BD '$1'" -./script/addons/install_addons.sh "$1" user_test,disable_mail_server,disable_auto_backup +./script/addons/install_addons_dev.sh "$1" user_test,disable_mail_server,disable_auto_backup retVal=$? if [[ $retVal -ne 0 ]]; then From 7f973524785eda8d0309904145907638e1640725 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Mon, 13 Mar 2023 19:27:26 -0400 Subject: [PATCH 16/17] =?UTF-8?q?[UPD]=C2=A0script=20code=5Fcount.sh:=20ad?= =?UTF-8?q?d=20csv=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script/statistic/code_count.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/statistic/code_count.sh b/script/statistic/code_count.sh index 23ef2d8..4eeb285 100755 --- a/script/statistic/code_count.sh +++ b/script/statistic/code_count.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # all argument is directory/file separate by space -./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,css,scss --format=summary "$@" +./.venv/bin/pygount --duplicates --folders-to-skip="[...],*/libs/*,*/lib/*,lib" --names-to-skip="[...],*.min.*" --suffix=py,xml,html,js,csv,css,scss --format=summary "$@" From c067425033d5e43b17050641b1f5114a2ead2099 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Wed, 29 Mar 2023 15:23:12 -0400 Subject: [PATCH 17/17] [ADD] manifest: muk_web to code_generator --- manifest/default.dev.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest/default.dev.xml b/manifest/default.dev.xml index 19471ec..50c6782 100644 --- a/manifest/default.dev.xml +++ b/manifest/default.dev.xml @@ -93,7 +93,7 @@ - +