diff --git a/Makefile b/Makefile index ec480b5..420f708 100644 --- a/Makefile +++ b/Makefile @@ -1037,6 +1037,17 @@ i18n_generate_demo_portal: clean: find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete +############### +# Statistic # +############### +.PHONY: stat_module_evolution_per_year +stat_module_evolution_per_year: + ./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7 + +.PHONY: stat_module_evolution_per_year_OCA +stat_module_evolution_per_year_OCA: + ./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7 + ################### # Documentation # ################### diff --git a/poetry.lock b/poetry.lock index b908285..f4964eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5210,6 +5210,27 @@ files = [ {file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"}, ] +[[package]] +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +files = [ + {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"}, + {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typed-ast" version = "1.5.4" @@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "uvloop" +version = "0.17.0" +description = "Fast implementation of asyncio event loop on top of libuv" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"}, + {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"}, + {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"}, + {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"}, + {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"}, + {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"}, + {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"}, + {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"}, + {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"}, + {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"}, + {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"}, + {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"}, + {file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"}, + {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"}, + {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"}, + {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"}, + {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"}, + {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"}, + {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"}, + {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"}, + {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"}, + {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"}, + {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"}, + {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"}, + {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"}, + {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"}, + {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"}, + {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"}, + {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"}, + {file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"}, +] + +[package.extras] +dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"] + [[package]] name = "validators" version = "0.20.0" @@ -5879,4 +5945,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.7.16" -content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e" +content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368" diff --git a/pyproject.toml b/pyproject.toml index 6935dd9..6a696c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,8 @@ xmltodict = "^0.13.0" zeep = "^4.2.1" zipp = ">=3.6.0" zxcvbn = "^4.4.28" +tqdm = "^4.64.1" +uvloop = "^0.17.0" [tool.poetry.dev-dependencies] websocket_client = "^1.1.0" diff --git a/requirements.txt b/requirements.txt index 441cfc7..04cc620 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ openupgradelib unidiff colorama wheel -aioshutil<1.2 # For OSX cython @@ -80,6 +79,10 @@ numpy==1.21.1 feedparser==6.0.10 geojson<3.0.0 +# asyncio +aioshutil<1.2 +tqdm +uvloop # Solve pcodedmp, oletools, rtfde and extract-msg win_unicode_console diff --git a/script/statistic/show_evolution_module.py b/script/statistic/show_evolution_module.py index 1b7bb2d..7741fe6 100755 --- a/script/statistic/show_evolution_module.py +++ b/script/statistic/show_evolution_module.py @@ -8,28 +8,35 @@ import sys from collections import defaultdict from dateutil.relativedelta import relativedelta -from git import Repo -logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +new_path = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +sys.path.append(new_path) + +from script import lib_asyncio + +FORMAT = "%(asctime)s [%(levelname)s] %(message)s" +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT) _logger = logging.getLogger(__name__) PROJECT_NAME = os.path.basename(os.getcwd()) VENV_PATH = "./.venv" -CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_OCA") +CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo") CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv" +MAX_COROUTINE = 300 IGNORE_REPO_LIST = ( "https://github.com/OCA/odoo-module-migrator.git", "https://github.com/OCA/maintainer-tools.git", - "https://github.com/OCA/connector.git", "https://github.com/itpp-labs/odoo-development.git", "https://github.com/itpp-labs/odoo-port-docs.git", "https://github.com/itpp-labs/odoo-test-docs.git", "https://github.com/odoo/documentation.git", - "https://github.com/OCA/odoo-module-migrator.git", - "https://github.com/OCA/maintainer-tools.git", + "https://github.com/ERPLibre/ERPLibre_image_db.git", + "https://github.com/muk-it/muk_docs.git", ) -DCT_CHANGE_ADDONS_PATH = {"odoo": "/addons/"} +DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": "/addons/"} DCT_VERSION_RELEASE = { "1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(), "2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(), @@ -72,6 +79,11 @@ def get_config(): action="store_true", help=f"More information", ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable asyncio debugging", + ) parser.add_argument( "--ignore_release_date", action="store_true", @@ -80,6 +92,21 @@ def get_config(): f" result in past of release version." ), ) + parser.add_argument( + "--no_parallel", + action="store_true", + help=( + f"By default run in parallel with all CPU, this disable" + f" parallelism with asyncio" + ), + ) + parser.add_argument( + "-p", + "--max_process", + type=int, + default=0, + help="Max processor to use. If 0, use max.", + ) parser.add_argument( "--history_length", default=0, @@ -106,13 +133,13 @@ def get_config(): ) parser.add_argument( "--filter", - default="/OCA/", - help=f"keyword to ignore separate by ','", + default="", + help=f"keyword to ignore separate by ','. Suggest /OCA/", ) parser.add_argument( "-b", "--branches", - default="6.0,6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", + default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0", help="Branch to analyse, separate by ','", ) args = parser.parse_args() @@ -149,12 +176,16 @@ def main(): config = get_config() lst_branch = config.branches.split(",") before_date = config.before_date + lst_stat = [] + # Check die( not os.path.isdir(VENV_PATH), f"Missing {VENV_PATH} venv path, did you install ERPLibre?", ) + # Setup environment + _logger.info(f"Setup {CACHE_CLONE_PATH}") if config.clean_all and os.path.isdir(CACHE_CLONE_PATH): _logger.info(f"Delete directory {CACHE_CLONE_PATH}") shutil.rmtree(CACHE_CLONE_PATH) @@ -163,66 +194,79 @@ def main(): _logger.info(f"Create directory {CACHE_CLONE_PATH}") os.mkdir(CACHE_CLONE_PATH) + # Get list of Repo lst_repo_url = get_OCA_repo_list(config) - dct_repo = clone_all_repo(config, lst_repo_url) - lst_stat = [] + + # Clone all repo + _logger.info(f"Clone repo") + lst_task_clone = [clone_repo(i, a) for i, a in enumerate(lst_repo_url)] + lst_repo_path = lib_asyncio.execute( + config, lst_task_clone, use_uvloop=True + ) + + # Extract module list + _logger.info("Extract module") + lst_task = [] for nb_year in range(config.more_year + 1): # key is odoo version, value is set of module name - from_date = before_date + relativedelta(years=nb_year) - if config.ignore_release_date: - lst_branch_adapt = lst_branch - else: - lst_branch_adapt = get_branches_supported_date( - lst_branch, from_date - ) - ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) = extract_module_from_all_version( - lst_branch_adapt, from_date, dct_repo - ) - lst_stat.append( - ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) - ) - for nb_year, ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) in enumerate(lst_stat): - str_extra = "" - - from_date = before_date + relativedelta(years=nb_year) - if config.ignore_release_date: - lst_branch_adapt = lst_branch - else: - lst_branch_adapt = get_branches_supported_date( - lst_branch, from_date - ) - + lst_branch_adapt = lst_branch + from_date = before_date if before_date: - str_extra = f" before {from_date}" - # Count remove repo - count_remove_repo_stat = 0 - for repo_name, list_branch in dct_missing_branch.items(): - if len(list_branch) == len(lst_branch_adapt): - count_remove_repo_stat += 1 + from_date += relativedelta(years=nb_year) + if not config.ignore_release_date: + lst_branch_adapt = get_branches_supported_date( + lst_branch, from_date + ) + # for branch in lst_branch_adapt: + for repo_name, repo_path in lst_repo_path: + lst_task.append( + extract_module( + lst_branch_adapt, repo_name, repo_path, from_date + ) + ) + if config.debug: + _logger.info( + "Setup extract module:" + f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}" + ) + + _logger.info(f"Execute extract module with {len(lst_task)} coroutine.") + # Need to set a max, or crash + if len(lst_task) / MAX_COROUTINE > 1: + lst_result = [] + for i in range(int(len(lst_task) / MAX_COROUTINE) + 1): + min_i = i * MAX_COROUTINE + max_i = min((i + 1) * MAX_COROUTINE, len(lst_task)) + _logger.info(f"Partial execution {min_i} to {max_i}") + tpl_result = lib_asyncio.execute( + config, lst_task[min_i:max_i], use_uvloop=True + ) + lst_result += tpl_result + tpl_result = lst_result + else: + tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True) + _logger.info("Analyse information") + dct_result = defaultdict(lambda: defaultdict(int)) + for lst_result_stack in tpl_result: + for dct_result_module in lst_result_stack: + # lst_module, branch, path, repo, before_date + before_date = dct_result_module.get("before_date") + lst_module = dct_result_module.get("lst_module") + branch = dct_result_module.get("branch") + # path = dct_result_module.get("path") + # repo = dct_result_module.get("repo") + dct_result[before_date][branch] += len(lst_module) + + # Show result + _logger.info("Show result") + for before_date, dct_branch_result in dct_result.items(): + str_extra = "" + if before_date: + str_extra = f" before {before_date}" print(f"Stat nb module by branch name{str_extra}") - print(f"With {len(lst_repo_url) - count_remove_repo_stat}") - for branch_name in lst_branch_adapt: - if branch_name: - print(f"{branch_name}\t{len(result_module[branch_name])}") + for branch_name, count_module in dct_branch_result.items(): + print(f"{branch_name}\t{count_module}") print("end of stat") @@ -236,105 +280,123 @@ def get_branches_supported_date(lst_branch, from_date): return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date] -def extract_module_from_all_version(lst_branch, before_date, dct_repo): - result = defaultdict(dict) - result_module = {} - lst_all_unique_module = set() - dct_reverse_lst_module = defaultdict(list) - dct_missing_branch = defaultdict(list) +async def extract_module(lst_branch, repo, path, before_date): + # print(f"{branch} - {path} - {before_date}") + lst_result = [] + for branch in lst_branch: + return_value = { + "lst_module": [], + "branch": branch, + "path": path, + "repo": repo, + "before_date": before_date, + } + remote_branch_name = f"origin/{branch}" - for branch_name in lst_branch: - if not branch_name: + # Detect commit to check + # git rev-list -n 1 --before 2022-01-01 origin/12.0 + lst_args = ["git", "rev-list", "-n", "1"] + if before_date: + lst_args += ["--before", str(before_date)] + lst_args += [remote_branch_name] + ( + commit, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=path + ) + + if status: + # The branch doesn't exist + # _logger.error(f"Receive status 'git rev-list' {status}") continue - # TODO bug conflict repo name - _logger.info(f"Check branch name '{branch_name}'") - lst_branch_unique_module = set() - result_module[branch_name] = lst_branch_unique_module - remote_branch_name = f"origin/{branch_name}" - for repo_name, repo_git in dct_repo.items(): - try: - repo_git.git.rev_parse("--verify", remote_branch_name) - except: - _logger.warning( - f"Missing branch '{branch_name}' for repo '{repo_name}'" + + # TODO send msg when no result + if not commit: + continue + + commit = commit.strip() + + # List directory + suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "") + adapt_path = path + suffix_path + lst_args = ["git", "ls-tree", "-d", "--name-only", commit] + ( + str_folders, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=adapt_path + ) + if status: + _logger.error(f"Receive status 'git ls-tree' {status}") + continue + + # TODO send msg when no result + if not str_folders: + continue + lst_module = str_folders.split() + if not lst_module: + continue + + if suffix_path: + suffix_path = suffix_path[1:] + + lst_module_installable = [] + for module in lst_module: + # List directory + lst_args = [ + "git", + "cat-file", + "-p", + f"{commit}:{suffix_path}{module}/__manifest__.py", + ] + ( + manifest, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=path + ) + if status: + # Maybe __openerp__.py + lst_args = [ + "git", + "cat-file", + "-p", + f"{commit}:{module}/__openerp__.py", + ] + ( + manifest, + _, + status, + ) = await lib_asyncio.run_command_get_output_and_status( + *lst_args, cwd=adapt_path ) - result[repo_name][branch_name] = None - dct_missing_branch[repo_name].append(branch_name) + if status: + # Ignore, it's not a module + # _logger.error(f"Receive status 'git cat-file' {status}") + # return return_value continue - if before_date: - checkout_commit = repo_git.git.rev_list( - "-n", "1", "--before", before_date, remote_branch_name - ) - if not checkout_commit: - _logger.warning( - f"Branch '{branch_name}' repo '{repo_name}' contain no" - f" commit before date {before_date}" - ) - continue - else: - checkout_commit = branch_name - # clean before checkout - repo_git.git.stash("save") - repo_git.git.clean("-xdf") - try: - repo_git.git.checkout(checkout_commit) - except Exception as e: - _logger.error( - f"Cannot checkout for branch '{branch_name}' repo" - f" '{repo_name}'" - ) - raise e - lst_unique_module = set() - result[repo_name][branch_name] = lst_unique_module - - suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo_name, "") - repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + suffix_path - for dir_name in os.listdir(repo_path): - path_module_dir = os.path.join(repo_path, dir_name) - path_module_dir_manifest = os.path.join( - path_module_dir, "__manifest__.py" - ) - path_module_dir_openerp = os.path.join( - path_module_dir, "__openerp__.py" - ) - if os.path.isdir(path_module_dir) and ( - os.path.isfile(path_module_dir_manifest) - or os.path.isfile(path_module_dir_openerp) - ): - lst_unique_module.add(dir_name) - lst_all_unique_module.add(dir_name) - lst_branch_unique_module.add(dir_name) - dct_reverse_lst_module[dir_name].append( - f"{branch_name} - {repo_name}" - ) - return ( - result, - result_module, - lst_all_unique_module, - dct_reverse_lst_module, - dct_missing_branch, - ) + dct_manifest = eval(manifest + "\n") + if dct_manifest.get("installable", True): + lst_module_installable.append(module) + return_value["lst_module"] = lst_module_installable + lst_result.append(return_value) + return lst_result -def clone_all_repo(config, lst_repo): - dct_repo = {} - for i, repo_url in enumerate(lst_repo): - repo_name = repo_url.split("/")[-1][:-4] - repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) - _logger.info(f"{i} - Repo '{repo_name}' url '{repo_url}'") - if os.path.isdir(repo_path): - cloned_repo = Repo(repo_path) - assert cloned_repo.__class__ is Repo - else: - if config.history_length: - cloned_repo = Repo.clone_from( - repo_url, repo_path, depth=config.history_length - ) - else: - cloned_repo = Repo.clone_from(repo_url, repo_path) - assert cloned_repo.__class__ is Repo - dct_repo[repo_name] = cloned_repo - return dct_repo +async def clone_repo(i, repo_url): + split_repo_url = repo_url.split("/") + repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}" + repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + if not os.path.isdir(repo_path): + await lib_asyncio.run_command_get_output( + "git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH + ) + _logger.info(f"[{i}]New clone {repo_path} with {repo_url}") + return repo_name, repo_path def get_OCA_repo_list(config):