diff --git a/Makefile b/Makefile index 752ce25..33504d8 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,8 @@ test_full: test_full_fast: ./script/make.sh clean ./script/test/run_parallel_test.py + # TODO This test is broken in parallel + ./script/make.sh test_code_generator_hello_world .PHONY: test_base test_base: diff --git a/poetry.lock b/poetry.lock index 198ef4a..93c2341 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,6 +6,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "aioshutil" +version = "1.1" +description = "Asynchronous shutil module." +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "alabaster" version = "0.7.12" @@ -1820,7 +1828,7 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [[package]] name = "pylint-odoo" -version = "5.0.5" +version = "5.0.6.dev1" description = "Pylint plugin for Odoo" category = "main" optional = false @@ -1844,7 +1852,7 @@ whichcraft = "*" type = "git" url = "https://github.com/oca/pylint-odoo.git" reference = "master" -resolved_reference = "c03dd158acd79dea665588af38a00a49f54e210a" +resolved_reference = "151d31d452f8963c560672e2abe1919649769a45" [[package]] name = "pylint-plugin-utils" @@ -3062,13 +3070,17 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.7.12" -content-hash = "83010756f7980ae45b19114177a7018314bede3c0fd49ab139d1a73782b8c851" +content-hash = "a0ff20f5203ceac245624c04e5d9b96956a3453545738056f9a24be8f22f9775" [metadata.files] agithub = [ {file = "agithub-2.2.2-py2.py3-none-any.whl", hash = "sha256:5f50071536db0cc9611859e761d7c3dc6f951a8535029bf6af7fb22744b7e118"}, {file = "agithub-2.2.2.tar.gz", hash = "sha256:321aa5fc688fde2970562b35c991881b69a13f57372d554a0fab184df85fa8a7"}, ] +aioshutil = [ + {file = "aioshutil-1.1-py3-none-any.whl", hash = "sha256:4c17e1da55cf928b4a85bd6ff5e4f1560cf21db7a16b5da5844f8f3edf3e2895"}, + {file = "aioshutil-1.1.tar.gz", hash = "sha256:d2e8d6baddab13137410b27ce24f39ce9889684cb47503d5af182ea8d038b0f1"}, +] alabaster = [ {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, diff --git a/pyproject.toml b/pyproject.toml index 657fbc7..9a87311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,6 +169,7 @@ xmlformatter = "^0.2.4" xmltodict = "^0.12.0" zeep = "^4.1.0" zxcvbn = "^4.4.28" +aioshutil = "^1.1" [tool.poetry.dev-dependencies] websocket_client = "^1.1.0" diff --git a/requirements.txt b/requirements.txt index f219101..dfc3a01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ openupgradelib unidiff colorama wheel +aioshutil # For OSX cython diff --git a/script/addons/check_addons_exist.py b/script/addons/check_addons_exist.py index 09b8f81..f3d7d48 100755 --- a/script/addons/check_addons_exist.py +++ b/script/addons/check_addons_exist.py @@ -6,8 +6,6 @@ import os import sys from collections import defaultdict -CONFIG_PATH = "./config.conf" - logging.basicConfig( format=( "%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d]" @@ -34,6 +32,12 @@ def get_config(): required=True, help="Module name to search, a list can be use separate by ,", ) + parser.add_argument( + "-c", + "--config", + default="./config.conf", + help="The config path.", + ) parser.add_argument( "--debug", action="store_true", @@ -49,18 +53,18 @@ def main(): logging.getLogger().setLevel(logging.DEBUG) config_parser = configparser.ConfigParser() - config_parser.read(CONFIG_PATH) + config_parser.read(config.config) if "options" in config_parser: if "addons_path" in config_parser["options"]: addons_path = config_parser["options"]["addons_path"] else: _logger.error( "Missing item 'addons_path' in section 'options' in" - f" '{CONFIG_PATH}'" + f" '{config.config}'" ) return -1 else: - _logger.error(f"Missing section 'options' in '{CONFIG_PATH}'") + _logger.error(f"Missing section 'options' in '{config.config}'") return -1 lst_addons_path = addons_path.split(",") diff --git a/script/addons/install_addons.sh b/script/addons/install_addons.sh index 0ce140f..8b0a6de 100755 --- a/script/addons/install_addons.sh +++ b/script/addons/install_addons.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash -./script/addons/check_addons_exist.py -m "$2" +# Argument 3 is config +if [[ $# -eq 3 ]]; then + ./script/addons/check_addons_exist.py -m "$2" -c "$3" +else + ./script/addons/check_addons_exist.py -m "$2" +fi retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons.sh" - exit 1 + echo "Error check_addons_exist.py into install_addons.sh" + exit 1 fi -./run.sh --no-http --stop-after-init -d "$1" -i "$2" -u "$2" +if [[ $# -eq 3 ]]; then + ./run.sh --no-http --stop-after-init -d "$1" -i "$2" -u "$2" -c "$3" +else + ./run.sh --no-http --stop-after-init -d "$1" -i "$2" -u "$2" +fi + retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons.sh" - exit 1 + echo "Error run.sh into install_addons.sh" + exit 1 fi diff --git a/script/addons/install_addons_dev.sh b/script/addons/install_addons_dev.sh index b0fd487..9d91b24 100755 --- a/script/addons/install_addons_dev.sh +++ b/script/addons/install_addons_dev.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash -./script/addons/check_addons_exist.py -m "$2" +# Argument 3 is config +if [[ $# -eq 3 ]]; then + ./script/addons/check_addons_exist.py -m "$2" -c "$3" +else + ./script/addons/check_addons_exist.py -m "$2" +fi retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons.sh" - exit 1 + echo "Error check_addons_exist.py into install_addons_dev.sh" + exit 1 fi -./run.sh --no-http --stop-after-init --dev qweb -d "$1" -i "$2" -u "$2" +if [[ $# -eq 3 ]]; then + ./run.sh --no-http --stop-after-init --dev qweb -d "$1" -i "$2" -u "$2" -c "$3" +else + ./run.sh --no-http --stop-after-init --dev qweb -d "$1" -i "$2" -u "$2" +fi + retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons_dev.sh" - exit 1 + echo "Error run.sh into install_addons_dev.sh" + exit 1 fi diff --git a/script/addons/reinstall_addons.sh b/script/addons/reinstall_addons.sh index 5e1b292..fa66b8c 100755 --- a/script/addons/reinstall_addons.sh +++ b/script/addons/reinstall_addons.sh @@ -1,14 +1,28 @@ #!/usr/bin/env bash -./script/addons/check_addons_exist.py -m "$2" +# Argument 3 is config +if [[ $# -eq 3 ]]; then + ./script/addons/check_addons_exist.py -m "$2" -c "$3" +else + ./script/addons/check_addons_exist.py -m "$2" +fi retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons.sh" - exit 1 + echo "Error check_addons_exist.py into reinstall_addons_dev.sh" + exit 1 +fi +if [[ $# -eq 3 ]]; then + ./script/addons/uninstall_addons.sh "$1" "$2" "$3" +else + ./script/addons/uninstall_addons.sh "$1" "$2" +fi +# Ignore if not uninstall +if [[ $# -eq 3 ]]; then + ./script/addons/install_addons_dev.sh "$1" "$2" "$3" +else + ./script/addons/install_addons_dev.sh "$1" "$2" fi -./script/addons/uninstall_addons.sh "$1" "$2" -./script/addons/install_addons_dev.sh "$1" "$2" retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error reinstall_addons.sh" - exit 1 + echo "Error install_addons_dev.sh into reinstall_addons.sh" + exit 1 fi diff --git a/script/addons/uninstall_addons.sh b/script/addons/uninstall_addons.sh index 0685095..67e9308 100755 --- a/script/addons/uninstall_addons.sh +++ b/script/addons/uninstall_addons.sh @@ -1,8 +1,17 @@ #!/usr/bin/env bash -./script/addons/check_addons_exist.py -m "$2" +# Argument 3 is config +if [[ $# -eq 3 ]]; then + ./script/addons/check_addons_exist.py -m "$2" -c "$3" +else + ./script/addons/check_addons_exist.py -m "$2" +fi retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error install_addons.sh" - exit 1 + echo "Error check_addons_exist.py into uninstall_addons.sh" + exit 1 +fi +if [[ $# -eq 3 ]]; then + ./run.sh --no-http --stop-after-init -d "$1" --uninstall "$2" "$3" +else + ./run.sh --no-http --stop-after-init -d "$1" --uninstall "$2" fi -./run.sh --no-http --stop-after-init -d "$1" --uninstall "$2" diff --git a/script/code_generator/install_and_test_code_generator.sh b/script/code_generator/install_and_test_code_generator.sh index d28454d..faaf2c5 100755 --- a/script/code_generator/install_and_test_code_generator.sh +++ b/script/code_generator/install_and_test_code_generator.sh @@ -3,31 +3,41 @@ # $2 is module name separate by , # $3 is directory path to check # $4 is generated module name separate by , +# $5 optional, the config path -if (($# < 4)); then +if [[ $# -lt 4 ]]; then echo "ERROR, need 4 arguments: 1-database name, 2-list of module to install, 3-directory to check difference, 4-list of generated module" exit 1 fi INIT_DATETIME=$(date +%s) -./script/addons/install_addons_dev.sh "$1" "$2" -retVal=$? -if [[ $retVal -ne 0 ]]; then +if [[ $# -eq 5 ]]; then + ./script/addons/install_addons_dev.sh "$1" "$2" "$5" + retVal=$? + if [[ $retVal -ne 0 ]]; then + echo "Error ./script/addons/install_addons_dev.sh ${1} ${2} ${5}" + exit 1 + fi +else + ./script/addons/install_addons_dev.sh "$1" "$2" + retVal=$? + if [[ $retVal -ne 0 ]]; then echo "Error ./script/addons/install_addons_dev.sh ${1} ${2}" exit 1 + fi fi # Check if the code was updated ./script/code_generator/test_code_generator_update_module.py -m "$4" -d "$3" --datetime "${INIT_DATETIME}" retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error ./script/code_generator/test_code_generator_update_module.py ${4} ${3}" - exit 1 + echo "Error ./script/code_generator/test_code_generator_update_module.py ${4} ${3}" + exit 1 fi # TODO check output when got warning -./script/repo_revert_git_diff_date_from_code_generator.py +./script/repo_revert_git_diff_date_from_code_generator.py --repo "$3" # Remove pot and po diff cd "$3" || exit 1 # git 2.22 and more, else use next command @@ -44,6 +54,6 @@ echo "TEST ${2}" ./script/code_generator/check_git_change_code_generator.sh "$3" retVal=$? if [[ $retVal -ne 0 ]]; then - echo "Error ./script/code_generator/check_git_change_code_generator.sh" - exit 1 + echo "Error ./script/code_generator/check_git_change_code_generator.sh" + exit 1 fi diff --git a/script/code_generator/new_project.py b/script/code_generator/new_project.py index 9c2b89d..bf76d55 100755 --- a/script/code_generator/new_project.py +++ b/script/code_generator/new_project.py @@ -3,6 +3,7 @@ import argparse import logging import os import sys +import tempfile import uuid from git import Repo @@ -73,14 +74,6 @@ def get_config(): action="store_true", help="Force override directory and module.", ) - parser.add_argument( - "--do_not_update_config", - action="store_true", - help=( - "Ignore updating config file. This is a patch for a bug for" - " duplicate path, but 1 relative and the other is absolute." - ), - ) parser.add_argument( "--keep_bd_alive", action="store_true", @@ -105,11 +98,9 @@ class ProjectManagement: template_name="", template_directory="", force=False, - ignore_config=False, keep_bd_alive=False, ): self.force = force - self.ignore_config = ignore_config self.keep_bd_alive = keep_bd_alive self.msg_error = "" self.origin_config_txt = "" @@ -233,6 +224,7 @@ class ProjectManagement: git_repo.git.restore(relative_path) def generate_module(self): + # TODO copy directory in temp workspace file before update it module_path = os.path.join(self.module_directory, self.module_name) if not self.force and not self.validate_path_ready_to_be_override( self.module_name, self.module_directory, path=module_path @@ -316,7 +308,7 @@ class ProjectManagement: ) ): return False - self.update_config() + config_path = self.update_config() bd_name_demo = f"new_project_code_generator_demo_{uuid.uuid4()}"[:63] cmd = f"./script/db_restore.py --database {bd_name_demo}" @@ -325,7 +317,7 @@ class ProjectManagement: _logger.info("========= GENERATE code_generator_demo =========") cmd = ( f"./script/addons/install_addons_dev.sh {bd_name_demo}" - " code_generator_demo" + f" code_generator_demo {config_path}" ) os.system(cmd) @@ -345,7 +337,6 @@ class ProjectManagement: # Validate if not os.path.exists(template_path): _logger.error(f"Module template not exists '{template_path}'") - self.revert_config() return False else: _logger.info(f"Module template exists '{template_path}'") @@ -363,7 +354,7 @@ class ProjectManagement: # Execute all bd_name_template = ( f"new_project_code_generator_template_{uuid.uuid4()}"[:63] - )[:63] + ) cmd = f"./script/db_restore.py --database {bd_name_template}" os.system(cmd) _logger.info(cmd) @@ -379,14 +370,14 @@ class ProjectManagement: os.system(cmd) cmd = ( f"./script/addons/install_addons_dev.sh {bd_name_template}" - f" {self.module_name}" + f" {self.module_name} {config_path}" ) _logger.info(cmd) os.system(cmd) cmd = ( f"./script/addons/install_addons_dev.sh {bd_name_template}" - f" {self.template_name}" + f" {self.template_name} {config_path}" ) _logger.info(cmd) os.system(cmd) @@ -402,7 +393,6 @@ class ProjectManagement: # Validate if not os.path.exists(cg_path): _logger.error(f"Module cg not exists '{cg_path}'") - self.revert_config() return False else: _logger.info(f"Module cg exists '{cg_path}'") @@ -415,7 +405,7 @@ class ProjectManagement: cmd = ( f"./script/addons/install_addons_dev.sh {bd_name_generator}" - f" {self.cg_name}" + f" {self.cg_name} {config_path}" ) _logger.info(cmd) os.system(cmd) @@ -424,24 +414,20 @@ class ProjectManagement: cmd = ( "./.venv/bin/python3 ./odoo/odoo-bin db --drop --database" f" {bd_name_generator}" - )[:63] + ) _logger.info(cmd) os.system(cmd) # Validate if not os.path.exists(template_path): _logger.error(f"Module not exists '{module_path}'") - self.revert_config() return False else: _logger.info(f"Module exists '{module_path}'") - self.revert_config() return True def update_config(self): - if self.ignore_config: - return # Backup config and restore it after, check if path exist or add it temporary with open("./config.conf") as config: config_txt = config.read() @@ -462,14 +448,11 @@ class ProjectManagement: if lst_directory_to_add: new_str = "addons_path = " + ",".join(lst_directory_to_add) + "," config_txt = config_txt.replace("addons_path = ", new_str) - with open("./config.conf", "w") as config: - config.write(config_txt) - - def revert_config(self): - if self.ignore_config: - return - with open("./config.conf", "w") as config: - config.write(self.origin_config_txt) + temp_file = tempfile.mktemp() + with open(temp_file, "w") as config: + config.write(config_txt) + print(f"mathben \n {temp_file} \n") + return temp_file def main(): @@ -482,7 +465,6 @@ def main(): cg_name=config.code_generator_name, template_name=config.template_name, force=config.force, - ignore_config=config.do_not_update_config, keep_bd_alive=config.keep_bd_alive, ) if project.msg_error: diff --git a/script/repo_revert_git_diff_date_from_code_generator.py b/script/repo_revert_git_diff_date_from_code_generator.py index 4c2980d..0c33a7c 100755 --- a/script/repo_revert_git_diff_date_from_code_generator.py +++ b/script/repo_revert_git_diff_date_from_code_generator.py @@ -32,6 +32,10 @@ def get_config(): epilog="""\ """, ) + parser.add_argument( + "--repo", + default="./addons/TechnoLibre_odoo-code-generator-template;./addons/OCA_server-tools", + ) args = parser.parse_args() return args @@ -43,11 +47,7 @@ def main(): r"[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])" r" (2[0-3]|[01][0-9]):[0-5][0-9]" ) - # TODO support argument instead of hardcoded values - lst_path = [ - "./addons/TechnoLibre_odoo-code-generator-template", - "./addons/OCA_server-tools", - ] + lst_path = config.repo.split(";") for path in lst_path: repo = git.Repo(path) supported_ext = [".xml", ".pot", ".po"] diff --git a/script/test/run_parallel_test.py b/script/test/run_parallel_test.py index 7c13058..8171fda 100755 --- a/script/test/run_parallel_test.py +++ b/script/test/run_parallel_test.py @@ -1,14 +1,18 @@ #!./.venv/bin/python import argparse import asyncio +import configparser import datetime import logging import os import sys +import tempfile import time import uuid +from collections import deque from typing import Tuple +import aioshutil from colorama import Fore logging.basicConfig(level=logging.DEBUG) @@ -21,7 +25,7 @@ def get_config(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""\ - Run code generator test in parallel. + Run code generator test in parallel (asyncio). """, epilog="""\ """, @@ -31,6 +35,30 @@ def get_config(): action="store_true", help="Will not stop or init check if contain git change.", ) + parser.add_argument( + "--no_parallel", + action="store_true", + help="Will run in serial.", + ) + parser.add_argument( + "--keep_cache", + action="store_true", + help=( + "Will not delete the temporary directory, check in print and log." + ), + ) + parser.add_argument( + "-p", + "--max_process", + type=int, + default=0, + help="Max processor to use. If 0, use max.", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable asyncio debugging", + ) args = parser.parse_args() return args @@ -134,6 +162,27 @@ def print_log(lst_task, tpl_result): print(f"Log file {LOG_FILE}") +async def run_command_get_output(*args, cwd=None): + if cwd is not None: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=cwd, + ) + else: + process = await asyncio.create_subprocess_exec( + *args, + # stdout must a pipe to be accessible as process.stdout + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + # Wait for the subprocess to finish + stdout, stderr = await process.communicate() + return stdout.decode() + + async def run_command(*args, test_name=None): # Create subprocess start_time = time.time() @@ -176,21 +225,306 @@ async def run_command(*args, test_name=None): return str_output_init + all_output, process.returncode +def update_config( + origin_config, + new_path_config, + lst_path_to_add_config=None, + lst_path_to_remove_config=None, + module_name=None, +): + config_parser = configparser.ConfigParser() + config_parser.read(origin_config) + str_path = config_parser["options"]["addons_path"].strip(",") + lst_path = str_path.split(",") + + # Clean path from test + if lst_path_to_remove_config: + for remove_key in lst_path_to_remove_config: + if remove_key.startswith("./"): + remove_key = remove_key[2:] + if module_name and remove_key.endswith(module_name): + remove_key = remove_key[: -(len(module_name) + 1)] + for s_path in lst_path: + if s_path.endswith(remove_key): + lst_path.remove(s_path) + break + if lst_path_to_add_config: + for add_path in lst_path_to_add_config: + if module_name and add_path.endswith(module_name): + add_path = add_path[: -(len(module_name) + 1)] + lst_path.insert(0, add_path) + s_new_path = ",".join(lst_path) + config_parser["options"]["addons_path"] = s_new_path + with open(new_path_config, "w") as configfile: + config_parser.write(configfile) + + async def test_exec( + config, path_module_check: str, generated_module=None, + generate_path=None, tested_module=None, search_class_module=None, script_after_init_check=None, lst_init_module_name=None, test_name=None, install_path=None, + run_in_sandbox=False, ) -> Tuple[str, int]: - test_result = "" test_status = 0 + new_destination_path = None + if search_class_module: + if install_path is not None: + path_template_to_generate = os.path.join( + install_path, tested_module + ) + elif generate_path: + path_template_to_generate = os.path.join( + generate_path, tested_module + ) + else: + path_template_to_generate = os.path.join( + path_module_check, tested_module + ) + if generate_path is not None: + path_module_to_generate = os.path.join( + generate_path, search_class_module + ) + else: + path_module_to_generate = os.path.join( + path_module_check, search_class_module + ) + else: + path_template_to_generate = None + path_module_to_generate = None + + use_test_path_generic = False + destination_path = None + temp_dir_name = None + if run_in_sandbox: + if config.keep_cache: + temp_dir = tempfile.mkdtemp() + temp_dir_name = temp_dir + else: + temp_dir = tempfile.TemporaryDirectory() + temp_dir_name = temp_dir.name + print(temp_dir_name) + temp_dir_name = os.path.join(temp_dir_name, "workspace") + os.mkdir(temp_dir_name) + + lst_path_to_add_config = [] + lst_path_to_remove_config = [] + if not os.path.exists(path_module_check): + return ( + f"Error var path_module_check '{path_module_check}' not" + " exist.", + -1, + ) + + copy_path_module_check = path_module_check + path_module_check = os.path.normpath( + os.path.join(temp_dir_name, path_module_check) + ) + + if generated_module and path_module_check.endswith( + "/" + generated_module + ): + use_test_path_generic = True + destination_path = path_module_check[ + : -(len(generated_module) + 1) + ] + copy_path = copy_path_module_check[: -(len(generated_module) + 1)] + elif search_class_module and path_module_check.endswith( + "/" + search_class_module + ): + destination_path = path_module_check[ + : -(len(search_class_module) + 1) + ] + copy_path = copy_path_module_check[ + : -(len(search_class_module) + 1) + ] + else: + destination_path = path_module_check + copy_path = copy_path_module_check + + lst_path_to_add_config.append(destination_path) + lst_path_to_remove_config.append(copy_path) + + ignore_tree = await aioshutil.ignore_patterns(".git", "setup") + await aioshutil.copytree( + copy_path, destination_path, ignore=ignore_tree + ) + # destination_path_with_git = os.path.join(destination_path, ".git") + # if os.path.exists(destination_path_with_git): + # await aioshutil.rmtree(destination_path_with_git) + # destination_path_with_setup = os.path.join(destination_path, "setup") + # if os.path.exists(destination_path_with_setup): + # await aioshutil.rmtree(destination_path_with_setup) + + if tested_module: + lst_module_to_test = tested_module.split(",") + for module_name in lst_module_to_test: + # Update path to change new emplacement + s_lst_path_tested_module = await run_command_get_output( + "find", ".", "-name", module_name + ) + if not s_lst_path_tested_module: + return ( + f"Error cannot find module '{path_module_check}' not" + " exist.", + -1, + ) + else: + lst_path_tested_module = ( + s_lst_path_tested_module.strip().split("\n") + ) + s_first_path = lst_path_tested_module[0] + parent_dir = os.path.dirname(s_first_path) + # Copy it + if copy_path != parent_dir: + os.path.basename(parent_dir) + new_destination_path = os.path.join( + os.path.dirname(destination_path), + os.path.basename(parent_dir), + ) + ignore_tree = await aioshutil.ignore_patterns( + ".git", "setup" + ) + await aioshutil.copytree( + parent_dir, + new_destination_path, + ignore=ignore_tree, + ) + # destination_path_with_git = os.path.join( + # new_destination_path, ".git" + # ) + # if os.path.exists(destination_path_with_git): + # await aioshutil.rmtree(destination_path_with_git) + lst_path_to_add_config.append(new_destination_path) + lst_path_to_remove_config.append(parent_dir) + + new_s_first_path = os.path.normpath( + os.path.join(temp_dir_name, s_first_path) + ) + + s_lst_path_generated_module = await run_command_get_output( + "find", + ".", + "-name", + generated_module, + cwd=temp_dir_name, + ) + if s_lst_path_generated_module: + lst_path_generated_module = ( + s_lst_path_generated_module.strip().split("\n") + ) + s_first_path = os.path.normpath( + os.path.join( + temp_dir_name, + os.path.dirname(lst_path_generated_module[0]), + ) + ) + else: + # TODO This is wrong... bug in template if reach this case + s_first_path = destination_path + + hook_file = os.path.join(new_s_first_path, "hooks.py") + with open(hook_file) as hook: + hook_line = hook.read() + has_template = ( + "template_dir = os.path.normpath" in hook_line + ) + + # Goal, update path_module_generate, maybe commented + # Goal, update template_dir, maybe not exist + # TODO need to refactor this and use AST and not string research + + # try find nb space indentation + lst_f_key = [ + "# path_module_generate = ", + "#path_module_generate = ", + "path_module_generate = ", + ] + nb_space_indentation = None + first_index = None + for f_key in lst_f_key: + if f_key in hook_line: + first_index = hook_line.find(f_key) + f_begin_index = ( + hook_line.rfind("\n", 0, first_index) + 1 + ) + nb_space_indentation = ( + first_index - f_begin_index + ) + break + if nb_space_indentation is None: + return ( + f"Cannot find keys '{lst_f_key}' in" + f" {hook_file}", + -1, + ) + end_string = "\n\n" + index_end_string = hook_line.find( + end_string, first_index + ) + + if has_template: + new_hook_line = ( + hook_line[: first_index + len(f_key)] + + f'"{s_first_path}"\n' + + f'{nb_space_indentation * " "}template_dir =' + f' "{s_first_path}/" + MODULE_NAME\n\n' + + hook_line[index_end_string:] + ) + else: + new_hook_line = ( + hook_line[: first_index + len(f_key)] + + f'"{s_first_path}"\n' + + hook_line[index_end_string:] + ) + new_hook_line = new_hook_line.replace( + "# path_module_generate = ", + "path_module_generate = ", + ) + new_hook_line = new_hook_line.replace( + '# "path_sync_code": path_module_generate,', + '"path_sync_code": path_module_generate,', + ) + with open(hook_file, "w") as hook: + hook.write(new_hook_line) + + # Format editing code before commit + await run_command_get_output( + "./script/maintenance/black.sh", temp_dir_name + ) + + # init repo with git + for dir_to_git in lst_path_to_add_config: + await run_command_get_output("git", "init", ".", cwd=dir_to_git) + await run_command_get_output("git", "add", ".", cwd=dir_to_git) + await run_command_get_output( + "git", "commit", "-am", "'first commit'", cwd=dir_to_git + ) + + new_config_path = os.path.join(temp_dir_name, "config.conf") + update_config( + "./config.conf", + new_config_path, + lst_path_to_add_config=lst_path_to_add_config, + lst_path_to_remove_config=lst_path_to_remove_config, + module_name=generated_module, + ) + else: + new_config_path = None + if install_path is None: install_path = path_module_check + elif run_in_sandbox: + install_path = os.path.normpath( + os.path.join(temp_dir_name, install_path) + ) # Check code, init module to install if lst_init_module_name: @@ -228,6 +562,7 @@ async def test_exec( is_db_create = False unique_database_name = f"test_demo_{uuid.uuid4()}"[:63] if not test_status: + # Create database res, status = await run_command( "./script/db_restore.py", "--database", @@ -239,34 +574,33 @@ async def test_exec( is_db_create = not status if not test_status and lst_init_module_name: - # Parallel execution here - - # No parallel execution here + # Install required module str_test = ",".join(lst_init_module_name) script_name = ( "./script/addons/install_addons_dev.sh" if tested_module else "./script/addons/install_addons.sh" ) - res, status = await run_command( - script_name, - unique_database_name, - str_test, - test_name=test_name, - ) + if new_config_path: + res, status = await run_command( + script_name, + unique_database_name, + str_test, + new_config_path, + test_name=test_name, + ) + else: + res, status = await run_command( + script_name, + unique_database_name, + str_test, + test_name=test_name, + ) test_result += res test_status += status if not test_status and search_class_module and generated_module: - path_template_to_generate = os.path.join( - path_module_check, tested_module - ) - path_module_to_generate = os.path.join( - path_module_check, search_class_module - ) - # Parallel execution here - - # No parallel execution here + # Update template with class/model/inherit res, status = await run_command( "./script/code_generator/search_class_model.py", "--quiet", @@ -279,18 +613,36 @@ async def test_exec( test_result += res test_status += status - if not test_status and tested_module and generated_module: - # Parallel execution here + test_generated_path = ( + install_path if destination_path is None else new_destination_path + ) + test_generated_path = ( + destination_path if use_test_path_generic else install_path + ) + # if destination_path is None: + # destination_path = install_path - # No parallel execution here - res, status = await run_command( - "./script/code_generator/install_and_test_code_generator.sh", - unique_database_name, - tested_module, - install_path, - generated_module, - test_name=test_name, - ) + if not test_status and tested_module and generated_module: + # Finally, the test + if new_config_path: + res, status = await run_command( + "./script/code_generator/install_and_test_code_generator.sh", + unique_database_name, + tested_module, + test_generated_path, + generated_module, + new_config_path, + test_name=test_name, + ) + else: + res, status = await run_command( + "./script/code_generator/install_and_test_code_generator.sh", + unique_database_name, + tested_module, + install_path, + generated_module, + test_name=test_name, + ) test_result += res test_status += status @@ -337,7 +689,7 @@ def print_summary_task(task_list): # START TEST -async def run_demo_test() -> Tuple[str, int]: +async def run_demo_test(config) -> Tuple[str, int]: lst_test_name = [ "demo_helpdesk_data", "demo_internal", @@ -349,6 +701,7 @@ async def run_demo_test() -> Tuple[str, int]: "demo_website_snippet", ] res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", lst_init_module_name=lst_test_name, test_name="demo_test", @@ -357,11 +710,12 @@ async def run_demo_test() -> Tuple[str, int]: return res, status -async def run_mariadb_test() -> Tuple[str, int]: - test_result = "" - test_status = 0 +async def run_code_generator_migrator_demo_mariadb_sql_example_1_test( + config, +) -> Tuple[str, int]: # Migrator res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module="demo_mariadb_sql_example_1", tested_module="code_generator_migrator_demo_mariadb_sql_example_1", @@ -372,12 +726,18 @@ async def run_mariadb_test() -> Tuple[str, int]: "code_generator_portal", ], test_name="mariadb_test-migrator", + run_in_sandbox=True, ) - test_result += res - test_status += status + return res, status + + +async def run_code_generator_template_demo_mariadb_sql_example_1_test( + config, +) -> Tuple[str, int]: # Template res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module="code_generator_demo_mariadb_sql_example_1", tested_module="code_generator_template_demo_mariadb_sql_example_1", @@ -387,12 +747,18 @@ async def run_mariadb_test() -> Tuple[str, int]: "demo_mariadb_sql_example_1", ], test_name="mariadb_test-template", + run_in_sandbox=True, ) - test_result += res - test_status += status + return res, status + + +async def run_code_generator_demo_mariadb_sql_example_1_test( + config, +) -> Tuple[str, int]: # Code generator res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module="demo_mariadb_sql_example_1", lst_init_module_name=[ @@ -400,6 +766,31 @@ async def run_mariadb_test() -> Tuple[str, int]: ], tested_module="code_generator_demo_mariadb_sql_example_1", test_name="mariadb_test-code-generator", + run_in_sandbox=True, + ) + + return res, status + + +async def run_code_generator_data_test(config) -> Tuple[str, int]: + test_result = "" + test_status = 0 + lst_generated_module = [ + "demo_helpdesk_data", + "demo_website_data", + ] + lst_tested_module = [ + "code_generator_demo_export_helpdesk", + "code_generator_demo_export_website", + ] + # Multiple + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module=",".join(lst_generated_module), + tested_module=",".join(lst_tested_module), + test_name="code_generator_data_test", + run_in_sandbox=True, ) test_result += res test_status += status @@ -407,35 +798,85 @@ async def run_mariadb_test() -> Tuple[str, int]: return test_result, test_status -async def run_code_generator_multiple_test() -> Tuple[str, int]: +async def run_code_generator_theme_test(config) -> Tuple[str, int]: + test_result = "" + test_status = 0 + lst_generated_module = [ + "theme_website_demo_code_generator", + ] + lst_tested_module = [ + "code_generator_demo_theme_website", + ] + # Multiple + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module=",".join(lst_generated_module), + tested_module=",".join(lst_tested_module), + test_name="code_generator_theme_test", + run_in_sandbox=True, + ) + test_result += res + test_status += status + + return test_result, test_status + + +async def run_code_generator_generic_all_test(config) -> Tuple[str, int]: test_result = "" test_status = 0 lst_generated_module = [ - "code_generator_demo", - "demo_helpdesk_data", - "demo_website_data", "demo_internal", "demo_portal", + "demo_helpdesk_data", + "demo_website_data", + "demo_website_leaflet", + "demo_website_snippet", "theme_website_demo_code_generator", + ] + lst_tested_module = [ + "code_generator_demo_internal", + "code_generator_demo_portal", + "code_generator_demo_export_helpdesk", + "code_generator_demo_export_website", + "code_generator_demo_website_leaflet", + "code_generator_demo_website_snippet", + "code_generator_demo_theme_website", + ] + # Multiple + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module=",".join(lst_generated_module), + tested_module=",".join(lst_tested_module), + test_name="code_generator_generic_all_test", + run_in_sandbox=True, + ) + test_result += res + test_status += status + + return test_result, test_status + + +async def run_code_generator_website_snippet_test(config) -> Tuple[str, int]: + test_result = "" + test_status = 0 + lst_generated_module = [ "demo_website_leaflet", "demo_website_snippet", ] lst_tested_module = [ - "code_generator_demo", - "code_generator_demo_export_helpdesk", - "code_generator_demo_export_website", - "code_generator_demo_internal", - "code_generator_demo_portal", - "code_generator_demo_theme_website", "code_generator_demo_website_leaflet", "code_generator_demo_website_snippet", ] # Multiple res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module=",".join(lst_generated_module), tested_module=",".join(lst_tested_module), - test_name="code_generator_multiple_test", + test_name="code_generator_website_snippet_test", + run_in_sandbox=True, ) test_result += res test_status += status @@ -443,7 +884,56 @@ async def run_code_generator_multiple_test() -> Tuple[str, int]: return test_result, test_status -async def run_code_generator_inherit_test() -> Tuple[str, int]: +async def run_code_generator_demo_generic_test(config) -> Tuple[str, int]: + test_result = "" + test_status = 0 + lst_generated_module = [ + "demo_internal", + "demo_portal", + ] + lst_tested_module = [ + "code_generator_demo_internal", + "code_generator_demo_portal", + ] + # Multiple + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module=",".join(lst_generated_module), + tested_module=",".join(lst_tested_module), + test_name="code_generator_demo_generic_test", + run_in_sandbox=True, + ) + test_result += res + test_status += status + + return test_result, test_status + + +async def run_code_generator_demo_test(config) -> Tuple[str, int]: + test_result = "" + test_status = 0 + lst_generated_module = [ + "code_generator_demo", + ] + lst_tested_module = [ + "code_generator_demo", + ] + # Multiple + res, status = await test_exec( + config, + "./addons/TechnoLibre_odoo-code-generator-template", + generated_module=",".join(lst_generated_module), + tested_module=",".join(lst_tested_module), + test_name="code_generator_demo_test", + ) + test_result += res + test_status += status + + return test_result, test_status + + +async def run_code_generator_inherit_test(config) -> Tuple[str, int]: # TODO can be merge into code_generator_multiple test_result = "" test_status = 0 @@ -455,10 +945,12 @@ async def run_code_generator_inherit_test() -> Tuple[str, int]: ] # Inherit res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module=",".join(lst_generated_module), tested_module=",".join(lst_tested_module), test_name="code_generator_inherit_test", + run_in_sandbox=True, ) test_result += res test_status += status @@ -466,7 +958,7 @@ async def run_code_generator_inherit_test() -> Tuple[str, int]: return test_result, test_status -async def run_code_generator_auto_backup_test() -> Tuple[str, int]: +async def run_code_generator_auto_backup_test(config) -> Tuple[str, int]: test_result = "" test_status = 0 lst_generated_module = [ @@ -477,10 +969,12 @@ async def run_code_generator_auto_backup_test() -> Tuple[str, int]: ] # Auto-backup res, status = await test_exec( + config, "./addons/OCA_server-tools/auto_backup", generated_module=",".join(lst_generated_module), tested_module=",".join(lst_tested_module), test_name="code_generator_auto_backup_test", + run_in_sandbox=True, ) test_result += res test_status += status @@ -488,19 +982,23 @@ async def run_code_generator_auto_backup_test() -> Tuple[str, int]: return test_result, test_status -async def run_code_generator_template_demo_portal_test() -> Tuple[str, int]: +async def run_code_generator_template_demo_portal_test( + config, +) -> Tuple[str, int]: test_result = "" test_status = 0 # Template res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module="code_generator_demo_portal", tested_module="code_generator_template_demo_portal", - # search_class_module="demo_mariadb_sql_example_1", + search_class_module="demo_portal", lst_init_module_name=[ "demo_portal", ], test_name="code_generator_template_demo_portal", + run_in_sandbox=True, ) test_result += res test_status += status @@ -508,21 +1006,23 @@ async def run_code_generator_template_demo_portal_test() -> Tuple[str, int]: return test_result, test_status -async def run_code_generator_template_demo_internal_inherit_test() -> Tuple[ - str, int -]: +async def run_code_generator_template_demo_internal_inherit_test( + config, +) -> Tuple[str, int]: test_result = "" test_status = 0 # Template res, status = await test_exec( + config, "./addons/TechnoLibre_odoo-code-generator-template", generated_module="code_generator_demo_internal_inherit", tested_module="code_generator_template_demo_internal_inherit", - # search_class_module="code_generator_demo_internal_inherit", + search_class_module="demo_internal_inherit", lst_init_module_name=[ "demo_internal_inherit", ], test_name="code_generator_template_demo_internal_inherit", + run_in_sandbox=True, ) test_result += res test_status += status @@ -530,22 +1030,25 @@ async def run_code_generator_template_demo_internal_inherit_test() -> Tuple[ return test_result, test_status -async def run_code_generator_template_demo_sysadmin_cron_test() -> Tuple[ - str, int -]: +async def run_code_generator_template_demo_sysadmin_cron_test( + config, +) -> Tuple[str, int]: test_result = "" test_status = 0 # Template res, status = await test_exec( + config, "./addons/OCA_server-tools/auto_backup", generated_module="code_generator_auto_backup", + generate_path="./addons/OCA_server-tools/", tested_module="code_generator_template_demo_sysadmin_cron", - # search_class_module="code_generator_demo_internal_inherit", + search_class_module="auto_backup", lst_init_module_name=[ "auto_backup", ], test_name="code_generator_template_demo_sysadmin_cron", install_path="./addons/TechnoLibre_odoo-code-generator-template", + run_in_sandbox=True, ) test_result += res test_status += status @@ -553,7 +1056,7 @@ async def run_code_generator_template_demo_sysadmin_cron_test() -> Tuple[ return test_result, test_status -async def run_helloworld_test() -> Tuple[str, int]: +async def run_helloworld_test(config) -> Tuple[str, int]: res, status = await run_command( "./test/code_generator/hello_world.sh", test_name="helloworld_test" ) @@ -561,32 +1064,134 @@ async def run_helloworld_test() -> Tuple[str, int]: return res, status -def run_all_test() -> None: - task_list = [] +async def run_in_serial(task_list): + q = asyncio.Queue() + for task in task_list: + await q.put(task) + lst_result = [] + for i in range(len(task_list)): + co = await q.get() + result = await co + lst_result.append(result) + return lst_result - task_list.append(run_demo_test()) - task_list.append(run_helloworld_test()) - task_list.append(run_mariadb_test()) - task_list.append(run_code_generator_multiple_test()) - task_list.append(run_code_generator_inherit_test()) - task_list.append(run_code_generator_auto_backup_test()) - task_list.append(run_code_generator_template_demo_portal_test()) - task_list.append(run_code_generator_template_demo_internal_inherit_test()) - task_list.append(run_code_generator_template_demo_sysadmin_cron_test()) + +class AsyncioPool: + # TODO check to replace this pool by https://github.com/dano/aioprocessing + def __init__(self, concurrency, loop=None): + """ + @param loop: asyncio loop + @param concurrency: Maximum number of concurrently running tasks + """ + self._loop = loop or asyncio.get_event_loop() + self._concurrency = concurrency + self._coros = deque([]) # All coroutines queued for execution + self._futures = [] # All currently running coroutines + self._lst_result = [] + + def close(self): + self._loop.close() + + def add_coro(self, coro): + """ + @param coro: coroutine to add + """ + self._coros.append(coro) + self.print_status() + + def run_until_complete(self): + self._loop.run_until_complete(self._wait_for_futures()) + return self._lst_result + + def print_status(self): + print( + " Status: coros:%s - futures:%s" + % (len(self._coros), len(self._futures)) + ) + + def _start_futures(self): + num_to_start = self._concurrency - len(self._futures) + num_to_start = min(num_to_start, len(self._coros)) + for _ in range(num_to_start): + coro = self._coros.popleft() + future = asyncio.ensure_future(coro, loop=self._loop) + self._futures.append(future) + self.print_status() + + async def _wait_for_futures(self): + while len(self._coros) > 0 or len(self._futures) > 0: + self._start_futures() + futures_completed, futures_pending = await asyncio.wait( + self._futures, + loop=self._loop, + return_when=asyncio.FIRST_COMPLETED, + ) + + for future in futures_completed: + self._lst_result.append(future.result()) + self._futures.remove(future) + self._start_futures() + + +def run_all_test(config) -> None: + # low in time, at the end for more speed + task_list = [ + run_code_generator_migrator_demo_mariadb_sql_example_1_test(config), + run_code_generator_template_demo_mariadb_sql_example_1_test(config), + run_code_generator_demo_mariadb_sql_example_1_test(config), + run_code_generator_auto_backup_test(config), + run_code_generator_template_demo_portal_test(config), + run_code_generator_template_demo_internal_inherit_test(config), + run_code_generator_template_demo_sysadmin_cron_test(config), + run_code_generator_demo_test(config), + run_code_generator_generic_all_test(config), + # save 30 seconds with generic_all and next commented + # run_code_generator_data_test(config), + # run_code_generator_theme_test(config), + # run_code_generator_website_snippet_test(config), + # run_code_generator_demo_generic_test(config), + # TODO Will cause conflict with the other because write in code_generator_demo/hooks.py + # run_helloworld_test(config), + run_code_generator_inherit_test(config), + run_demo_test(config), + ] print_summary_task(task_list) if asyncio.get_event_loop().is_closed(): asyncio.set_event_loop(asyncio.new_event_loop()) - loop = asyncio.get_event_loop() - commands = asyncio.gather(*task_list) - tpl_result = loop.run_until_complete(commands) - loop.close() + + if config.no_parallel: + tpl_result = asyncio.run(run_in_serial(task_list)) + elif config.max_process: + pool = AsyncioPool(config.max_process) + for task in task_list: + pool.add_coro(task) + try: + tpl_result = pool.run_until_complete() + finally: + pool.close() + else: + # Use maximal resource + loop = asyncio.get_event_loop() + if config.debug: + loop.set_debug(True) + try: + commands = asyncio.gather(*task_list) + tpl_result = loop.run_until_complete(commands) + finally: + loop.close() print_log(task_list, tpl_result) check_result(task_list, tpl_result) def main(): + # TODO configure logger with thread + # logging.basicConfig( + # level=logging.INFO, + # format='%(threadName)10s %(name)18s: %(message)s', + # stream=sys.stderr, + # ) config = get_config() start_time = time.time() if not config.ignore_init_check_git: @@ -594,7 +1199,7 @@ def main(): else: success = True if success: - run_all_test() + run_all_test(config) end_time = time.time() diff_sec = end_time - start_time # print(f"Time execution {diff_sec:.3f}s")