[UPD] script show_evolution_module: parallelism
This commit is contained in:
parent
fa844327fb
commit
d6f652f58c
5 changed files with 303 additions and 159 deletions
11
Makefile
11
Makefile
|
|
@ -1037,6 +1037,17 @@ i18n_generate_demo_portal:
|
|||
clean:
|
||||
find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
|
||||
|
||||
###############
|
||||
# Statistic #
|
||||
###############
|
||||
.PHONY: stat_module_evolution_per_year
|
||||
stat_module_evolution_per_year:
|
||||
./script/statistic/show_evolution_module.py --before_date "2016-01-01" --more_year 7
|
||||
|
||||
.PHONY: stat_module_evolution_per_year_OCA
|
||||
stat_module_evolution_per_year_OCA:
|
||||
./script/statistic/show_evolution_module.py --filter "/OCA/" --before_date "2016-01-01" --more_year 7
|
||||
|
||||
###################
|
||||
# Documentation #
|
||||
###################
|
||||
|
|
|
|||
68
poetry.lock
generated
68
poetry.lock
generated
|
|
@ -5210,6 +5210,27 @@ files = [
|
|||
{file = "tornado-6.1.tar.gz", hash = "sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.64.1"
|
||||
description = "Fast, Extensible Progress Meter"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
|
||||
files = [
|
||||
{file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"},
|
||||
{file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[package.extras]
|
||||
dev = ["py-make (>=0.1.0)", "twine", "wheel"]
|
||||
notebook = ["ipywidgets (>=6)"]
|
||||
slack = ["slack-sdk"]
|
||||
telegram = ["requests"]
|
||||
|
||||
[[package]]
|
||||
name = "typed-ast"
|
||||
version = "1.5.4"
|
||||
|
|
@ -5354,6 +5375,51 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
|
|||
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "uvloop"
|
||||
version = "0.17.0"
|
||||
description = "Fast implementation of asyncio event loop on top of libuv"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"},
|
||||
{file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"},
|
||||
{file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"},
|
||||
{file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"},
|
||||
{file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"},
|
||||
{file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"},
|
||||
{file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "validators"
|
||||
version = "0.20.0"
|
||||
|
|
@ -5879,4 +5945,4 @@ files = [
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.7.16"
|
||||
content-hash = "7b1ce61933412872fbb3bb10c4b03949b84e029161184c2bd45a132bf8ad9b1e"
|
||||
content-hash = "e1dc161d6db7cd748de03405d541ac8c9f2ddabf8bc3beeddd45707e4e10d368"
|
||||
|
|
|
|||
|
|
@ -185,6 +185,8 @@ xmltodict = "^0.13.0"
|
|||
zeep = "^4.2.1"
|
||||
zipp = ">=3.6.0"
|
||||
zxcvbn = "^4.4.28"
|
||||
tqdm = "^4.64.1"
|
||||
uvloop = "^0.17.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
websocket_client = "^1.1.0"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ openupgradelib
|
|||
unidiff
|
||||
colorama
|
||||
wheel
|
||||
aioshutil<1.2
|
||||
|
||||
# For OSX
|
||||
cython
|
||||
|
|
@ -80,6 +79,10 @@ numpy==1.21.1
|
|||
feedparser==6.0.10
|
||||
geojson<3.0.0
|
||||
|
||||
# asyncio
|
||||
aioshutil<1.2
|
||||
tqdm
|
||||
uvloop
|
||||
|
||||
# Solve pcodedmp, oletools, rtfde and extract-msg
|
||||
win_unicode_console
|
||||
|
|
|
|||
|
|
@ -8,28 +8,35 @@ import sys
|
|||
from collections import defaultdict
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from git import Repo
|
||||
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
new_path = os.path.normpath(
|
||||
os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
)
|
||||
sys.path.append(new_path)
|
||||
|
||||
from script import lib_asyncio
|
||||
|
||||
FORMAT = "%(asctime)s [%(levelname)s] %(message)s"
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format=FORMAT)
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_NAME = os.path.basename(os.getcwd())
|
||||
VENV_PATH = "./.venv"
|
||||
CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_OCA")
|
||||
CACHE_CLONE_PATH = os.path.join(VENV_PATH, "stat_repo")
|
||||
CST_FILE_SOURCE_REPO_ADDONS = "source_repo_addons.csv"
|
||||
MAX_COROUTINE = 300
|
||||
IGNORE_REPO_LIST = (
|
||||
"https://github.com/OCA/odoo-module-migrator.git",
|
||||
"https://github.com/OCA/maintainer-tools.git",
|
||||
"https://github.com/OCA/connector.git",
|
||||
"https://github.com/itpp-labs/odoo-development.git",
|
||||
"https://github.com/itpp-labs/odoo-port-docs.git",
|
||||
"https://github.com/itpp-labs/odoo-test-docs.git",
|
||||
"https://github.com/odoo/documentation.git",
|
||||
"https://github.com/OCA/odoo-module-migrator.git",
|
||||
"https://github.com/OCA/maintainer-tools.git",
|
||||
"https://github.com/ERPLibre/ERPLibre_image_db.git",
|
||||
"https://github.com/muk-it/muk_docs.git",
|
||||
)
|
||||
DCT_CHANGE_ADDONS_PATH = {"odoo": "/addons/"}
|
||||
DCT_CHANGE_ADDONS_PATH = {"odoo_odoo": "/addons/"}
|
||||
DCT_VERSION_RELEASE = {
|
||||
"1.0": datetime.datetime.strptime("2005-02-01", "%Y-%m-%d").date(),
|
||||
"2.0": datetime.datetime.strptime("2005-03-01", "%Y-%m-%d").date(),
|
||||
|
|
@ -72,6 +79,11 @@ def get_config():
|
|||
action="store_true",
|
||||
help=f"More information",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
action="store_true",
|
||||
help="Enable asyncio debugging",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ignore_release_date",
|
||||
action="store_true",
|
||||
|
|
@ -80,6 +92,21 @@ def get_config():
|
|||
f" result in past of release version."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no_parallel",
|
||||
action="store_true",
|
||||
help=(
|
||||
f"By default run in parallel with all CPU, this disable"
|
||||
f" parallelism with asyncio"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p",
|
||||
"--max_process",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Max processor to use. If 0, use max.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--history_length",
|
||||
default=0,
|
||||
|
|
@ -106,13 +133,13 @@ def get_config():
|
|||
)
|
||||
parser.add_argument(
|
||||
"--filter",
|
||||
default="/OCA/",
|
||||
help=f"keyword to ignore separate by ','",
|
||||
default="",
|
||||
help=f"keyword to ignore separate by ','. Suggest /OCA/",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--branches",
|
||||
default="6.0,6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0",
|
||||
default="6.1,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0",
|
||||
help="Branch to analyse, separate by ','",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
|
@ -149,12 +176,16 @@ def main():
|
|||
config = get_config()
|
||||
lst_branch = config.branches.split(",")
|
||||
before_date = config.before_date
|
||||
lst_stat = []
|
||||
|
||||
# Check
|
||||
die(
|
||||
not os.path.isdir(VENV_PATH),
|
||||
f"Missing {VENV_PATH} venv path, did you install ERPLibre?",
|
||||
)
|
||||
|
||||
# Setup environment
|
||||
_logger.info(f"Setup {CACHE_CLONE_PATH}")
|
||||
if config.clean_all and os.path.isdir(CACHE_CLONE_PATH):
|
||||
_logger.info(f"Delete directory {CACHE_CLONE_PATH}")
|
||||
shutil.rmtree(CACHE_CLONE_PATH)
|
||||
|
|
@ -163,66 +194,79 @@ def main():
|
|||
_logger.info(f"Create directory {CACHE_CLONE_PATH}")
|
||||
os.mkdir(CACHE_CLONE_PATH)
|
||||
|
||||
# Get list of Repo
|
||||
lst_repo_url = get_OCA_repo_list(config)
|
||||
dct_repo = clone_all_repo(config, lst_repo_url)
|
||||
lst_stat = []
|
||||
|
||||
# Clone all repo
|
||||
_logger.info(f"Clone repo")
|
||||
lst_task_clone = [clone_repo(i, a) for i, a in enumerate(lst_repo_url)]
|
||||
lst_repo_path = lib_asyncio.execute(
|
||||
config, lst_task_clone, use_uvloop=True
|
||||
)
|
||||
|
||||
# Extract module list
|
||||
_logger.info("Extract module")
|
||||
lst_task = []
|
||||
for nb_year in range(config.more_year + 1):
|
||||
# key is odoo version, value is set of module name
|
||||
from_date = before_date + relativedelta(years=nb_year)
|
||||
if config.ignore_release_date:
|
||||
lst_branch_adapt = lst_branch
|
||||
else:
|
||||
lst_branch_adapt = get_branches_supported_date(
|
||||
lst_branch, from_date
|
||||
)
|
||||
(
|
||||
result,
|
||||
result_module,
|
||||
lst_all_unique_module,
|
||||
dct_reverse_lst_module,
|
||||
dct_missing_branch,
|
||||
) = extract_module_from_all_version(
|
||||
lst_branch_adapt, from_date, dct_repo
|
||||
)
|
||||
lst_stat.append(
|
||||
(
|
||||
result,
|
||||
result_module,
|
||||
lst_all_unique_module,
|
||||
dct_reverse_lst_module,
|
||||
dct_missing_branch,
|
||||
)
|
||||
)
|
||||
for nb_year, (
|
||||
result,
|
||||
result_module,
|
||||
lst_all_unique_module,
|
||||
dct_reverse_lst_module,
|
||||
dct_missing_branch,
|
||||
) in enumerate(lst_stat):
|
||||
str_extra = ""
|
||||
|
||||
from_date = before_date + relativedelta(years=nb_year)
|
||||
if config.ignore_release_date:
|
||||
lst_branch_adapt = lst_branch
|
||||
else:
|
||||
lst_branch_adapt = get_branches_supported_date(
|
||||
lst_branch, from_date
|
||||
)
|
||||
|
||||
lst_branch_adapt = lst_branch
|
||||
from_date = before_date
|
||||
if before_date:
|
||||
str_extra = f" before {from_date}"
|
||||
# Count remove repo
|
||||
count_remove_repo_stat = 0
|
||||
for repo_name, list_branch in dct_missing_branch.items():
|
||||
if len(list_branch) == len(lst_branch_adapt):
|
||||
count_remove_repo_stat += 1
|
||||
from_date += relativedelta(years=nb_year)
|
||||
if not config.ignore_release_date:
|
||||
lst_branch_adapt = get_branches_supported_date(
|
||||
lst_branch, from_date
|
||||
)
|
||||
|
||||
# for branch in lst_branch_adapt:
|
||||
for repo_name, repo_path in lst_repo_path:
|
||||
lst_task.append(
|
||||
extract_module(
|
||||
lst_branch_adapt, repo_name, repo_path, from_date
|
||||
)
|
||||
)
|
||||
if config.debug:
|
||||
_logger.info(
|
||||
"Setup extract module:"
|
||||
f" {repo_name} {repo_path} {lst_branch_adapt} {from_date}"
|
||||
)
|
||||
|
||||
_logger.info(f"Execute extract module with {len(lst_task)} coroutine.")
|
||||
# Need to set a max, or crash
|
||||
if len(lst_task) / MAX_COROUTINE > 1:
|
||||
lst_result = []
|
||||
for i in range(int(len(lst_task) / MAX_COROUTINE) + 1):
|
||||
min_i = i * MAX_COROUTINE
|
||||
max_i = min((i + 1) * MAX_COROUTINE, len(lst_task))
|
||||
_logger.info(f"Partial execution {min_i} to {max_i}")
|
||||
tpl_result = lib_asyncio.execute(
|
||||
config, lst_task[min_i:max_i], use_uvloop=True
|
||||
)
|
||||
lst_result += tpl_result
|
||||
tpl_result = lst_result
|
||||
else:
|
||||
tpl_result = lib_asyncio.execute(config, lst_task, use_uvloop=True)
|
||||
_logger.info("Analyse information")
|
||||
dct_result = defaultdict(lambda: defaultdict(int))
|
||||
for lst_result_stack in tpl_result:
|
||||
for dct_result_module in lst_result_stack:
|
||||
# lst_module, branch, path, repo, before_date
|
||||
before_date = dct_result_module.get("before_date")
|
||||
lst_module = dct_result_module.get("lst_module")
|
||||
branch = dct_result_module.get("branch")
|
||||
# path = dct_result_module.get("path")
|
||||
# repo = dct_result_module.get("repo")
|
||||
dct_result[before_date][branch] += len(lst_module)
|
||||
|
||||
# Show result
|
||||
_logger.info("Show result")
|
||||
for before_date, dct_branch_result in dct_result.items():
|
||||
str_extra = ""
|
||||
if before_date:
|
||||
str_extra = f" before {before_date}"
|
||||
print(f"Stat nb module by branch name{str_extra}")
|
||||
print(f"With {len(lst_repo_url) - count_remove_repo_stat}")
|
||||
for branch_name in lst_branch_adapt:
|
||||
if branch_name:
|
||||
print(f"{branch_name}\t{len(result_module[branch_name])}")
|
||||
for branch_name, count_module in dct_branch_result.items():
|
||||
print(f"{branch_name}\t{count_module}")
|
||||
print("end of stat")
|
||||
|
||||
|
||||
|
|
@ -236,105 +280,123 @@ def get_branches_supported_date(lst_branch, from_date):
|
|||
return [a for a in lst_branch if DCT_VERSION_RELEASE[a] < from_date]
|
||||
|
||||
|
||||
def extract_module_from_all_version(lst_branch, before_date, dct_repo):
|
||||
result = defaultdict(dict)
|
||||
result_module = {}
|
||||
lst_all_unique_module = set()
|
||||
dct_reverse_lst_module = defaultdict(list)
|
||||
dct_missing_branch = defaultdict(list)
|
||||
async def extract_module(lst_branch, repo, path, before_date):
|
||||
# print(f"{branch} - {path} - {before_date}")
|
||||
lst_result = []
|
||||
for branch in lst_branch:
|
||||
return_value = {
|
||||
"lst_module": [],
|
||||
"branch": branch,
|
||||
"path": path,
|
||||
"repo": repo,
|
||||
"before_date": before_date,
|
||||
}
|
||||
remote_branch_name = f"origin/{branch}"
|
||||
|
||||
for branch_name in lst_branch:
|
||||
if not branch_name:
|
||||
# Detect commit to check
|
||||
# git rev-list -n 1 --before 2022-01-01 origin/12.0
|
||||
lst_args = ["git", "rev-list", "-n", "1"]
|
||||
if before_date:
|
||||
lst_args += ["--before", str(before_date)]
|
||||
lst_args += [remote_branch_name]
|
||||
(
|
||||
commit,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=path
|
||||
)
|
||||
|
||||
if status:
|
||||
# The branch doesn't exist
|
||||
# _logger.error(f"Receive status 'git rev-list' {status}")
|
||||
continue
|
||||
# TODO bug conflict repo name
|
||||
_logger.info(f"Check branch name '{branch_name}'")
|
||||
lst_branch_unique_module = set()
|
||||
result_module[branch_name] = lst_branch_unique_module
|
||||
remote_branch_name = f"origin/{branch_name}"
|
||||
for repo_name, repo_git in dct_repo.items():
|
||||
try:
|
||||
repo_git.git.rev_parse("--verify", remote_branch_name)
|
||||
except:
|
||||
_logger.warning(
|
||||
f"Missing branch '{branch_name}' for repo '{repo_name}'"
|
||||
|
||||
# TODO send msg when no result
|
||||
if not commit:
|
||||
continue
|
||||
|
||||
commit = commit.strip()
|
||||
|
||||
# List directory
|
||||
suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo, "")
|
||||
adapt_path = path + suffix_path
|
||||
lst_args = ["git", "ls-tree", "-d", "--name-only", commit]
|
||||
(
|
||||
str_folders,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=adapt_path
|
||||
)
|
||||
if status:
|
||||
_logger.error(f"Receive status 'git ls-tree' {status}")
|
||||
continue
|
||||
|
||||
# TODO send msg when no result
|
||||
if not str_folders:
|
||||
continue
|
||||
lst_module = str_folders.split()
|
||||
if not lst_module:
|
||||
continue
|
||||
|
||||
if suffix_path:
|
||||
suffix_path = suffix_path[1:]
|
||||
|
||||
lst_module_installable = []
|
||||
for module in lst_module:
|
||||
# List directory
|
||||
lst_args = [
|
||||
"git",
|
||||
"cat-file",
|
||||
"-p",
|
||||
f"{commit}:{suffix_path}{module}/__manifest__.py",
|
||||
]
|
||||
(
|
||||
manifest,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=path
|
||||
)
|
||||
if status:
|
||||
# Maybe __openerp__.py
|
||||
lst_args = [
|
||||
"git",
|
||||
"cat-file",
|
||||
"-p",
|
||||
f"{commit}:{module}/__openerp__.py",
|
||||
]
|
||||
(
|
||||
manifest,
|
||||
_,
|
||||
status,
|
||||
) = await lib_asyncio.run_command_get_output_and_status(
|
||||
*lst_args, cwd=adapt_path
|
||||
)
|
||||
result[repo_name][branch_name] = None
|
||||
dct_missing_branch[repo_name].append(branch_name)
|
||||
if status:
|
||||
# Ignore, it's not a module
|
||||
# _logger.error(f"Receive status 'git cat-file' {status}")
|
||||
# return return_value
|
||||
continue
|
||||
if before_date:
|
||||
checkout_commit = repo_git.git.rev_list(
|
||||
"-n", "1", "--before", before_date, remote_branch_name
|
||||
)
|
||||
if not checkout_commit:
|
||||
_logger.warning(
|
||||
f"Branch '{branch_name}' repo '{repo_name}' contain no"
|
||||
f" commit before date {before_date}"
|
||||
)
|
||||
continue
|
||||
else:
|
||||
checkout_commit = branch_name
|
||||
# clean before checkout
|
||||
repo_git.git.stash("save")
|
||||
repo_git.git.clean("-xdf")
|
||||
try:
|
||||
repo_git.git.checkout(checkout_commit)
|
||||
except Exception as e:
|
||||
_logger.error(
|
||||
f"Cannot checkout for branch '{branch_name}' repo"
|
||||
f" '{repo_name}'"
|
||||
)
|
||||
raise e
|
||||
lst_unique_module = set()
|
||||
result[repo_name][branch_name] = lst_unique_module
|
||||
|
||||
suffix_path = DCT_CHANGE_ADDONS_PATH.get(repo_name, "")
|
||||
repo_path = os.path.join(CACHE_CLONE_PATH, repo_name) + suffix_path
|
||||
for dir_name in os.listdir(repo_path):
|
||||
path_module_dir = os.path.join(repo_path, dir_name)
|
||||
path_module_dir_manifest = os.path.join(
|
||||
path_module_dir, "__manifest__.py"
|
||||
)
|
||||
path_module_dir_openerp = os.path.join(
|
||||
path_module_dir, "__openerp__.py"
|
||||
)
|
||||
if os.path.isdir(path_module_dir) and (
|
||||
os.path.isfile(path_module_dir_manifest)
|
||||
or os.path.isfile(path_module_dir_openerp)
|
||||
):
|
||||
lst_unique_module.add(dir_name)
|
||||
lst_all_unique_module.add(dir_name)
|
||||
lst_branch_unique_module.add(dir_name)
|
||||
dct_reverse_lst_module[dir_name].append(
|
||||
f"{branch_name} - {repo_name}"
|
||||
)
|
||||
return (
|
||||
result,
|
||||
result_module,
|
||||
lst_all_unique_module,
|
||||
dct_reverse_lst_module,
|
||||
dct_missing_branch,
|
||||
)
|
||||
dct_manifest = eval(manifest + "\n")
|
||||
if dct_manifest.get("installable", True):
|
||||
lst_module_installable.append(module)
|
||||
return_value["lst_module"] = lst_module_installable
|
||||
lst_result.append(return_value)
|
||||
return lst_result
|
||||
|
||||
|
||||
def clone_all_repo(config, lst_repo):
|
||||
dct_repo = {}
|
||||
for i, repo_url in enumerate(lst_repo):
|
||||
repo_name = repo_url.split("/")[-1][:-4]
|
||||
repo_path = os.path.join(CACHE_CLONE_PATH, repo_name)
|
||||
_logger.info(f"{i} - Repo '{repo_name}' url '{repo_url}'")
|
||||
if os.path.isdir(repo_path):
|
||||
cloned_repo = Repo(repo_path)
|
||||
assert cloned_repo.__class__ is Repo
|
||||
else:
|
||||
if config.history_length:
|
||||
cloned_repo = Repo.clone_from(
|
||||
repo_url, repo_path, depth=config.history_length
|
||||
)
|
||||
else:
|
||||
cloned_repo = Repo.clone_from(repo_url, repo_path)
|
||||
assert cloned_repo.__class__ is Repo
|
||||
dct_repo[repo_name] = cloned_repo
|
||||
return dct_repo
|
||||
async def clone_repo(i, repo_url):
|
||||
split_repo_url = repo_url.split("/")
|
||||
repo_name = f"{split_repo_url[-2]}_{split_repo_url[-1][:-4]}"
|
||||
repo_path = os.path.join(CACHE_CLONE_PATH, repo_name)
|
||||
if not os.path.isdir(repo_path):
|
||||
await lib_asyncio.run_command_get_output(
|
||||
"git", "clone", repo_url, repo_name, cwd=CACHE_CLONE_PATH
|
||||
)
|
||||
_logger.info(f"[{i}]New clone {repo_path} with {repo_url}")
|
||||
return repo_name, repo_path
|
||||
|
||||
|
||||
def get_OCA_repo_list(config):
|
||||
|
|
|
|||
Loading…
Reference in a new issue