From 492fc4c5b0f43e2cf6e527c343e9cc9dcacc721e Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Thu, 3 Sep 2020 04:28:43 -0400 Subject: [PATCH] [ADD] script to remove auto_install in manifest of all repo - Script to show difference between actual code and manifest revision This will help to don't forget divergence between missing push code - Fix default_remote, using dict of all values instead of string --- doc/DEVELOPMENT.md | 11 ++ manifest/default.dev.xml | 16 +-- script/git_show_divergence_repo_manifest.py | 75 +++++++++++++ script/git_tool.py | 6 +- script/poetry_update.py | 36 +++--- script/repo_remove_auto_install.py | 117 ++++++++++++++++++++ 6 files changed, 232 insertions(+), 29 deletions(-) create mode 100755 script/git_show_divergence_repo_manifest.py create mode 100755 script/repo_remove_auto_install.py diff --git a/doc/DEVELOPMENT.md b/doc/DEVELOPMENT.md index e23aca5..a0a23a1 100644 --- a/doc/DEVELOPMENT.md +++ b/doc/DEVELOPMENT.md @@ -90,6 +90,12 @@ To show diff between commits in different manifest ./script/git_diff_repo_manifest.py --input1 ./manifest/MANIFEST1.xml --input2 ./manifest/MANIFEST2.xml ``` +## Diff between actual branch with manifest +To show diff between actual code and expected in manifest +```bash +./script/git_show_divergence_repo_manifest.py --manifest ./manifest/MANIFEST1.xml +``` + ## Add repo Access to a new repo, add your URL to file [source_repo_addons.csv](../source_repo_addons.csv) @@ -103,6 +109,11 @@ To regenerate only manifest.xml. ./script/fork_project_ERPLibre.py --skip_fork ``` +Check if contains "auto_install" in manifest, change to False. +```bash +./script/repo_remove_auto_install.py +``` + # Coding ## Create module scaffold (run in the venv) ```bash diff --git a/manifest/default.dev.xml b/manifest/default.dev.xml index ecf3bb0..6a25d79 100644 --- a/manifest/default.dev.xml +++ b/manifest/default.dev.xml @@ -26,16 +26,16 @@ - - + + - + - + @@ -109,7 +109,7 @@ - + @@ -124,7 +124,7 @@ - + @@ -139,8 +139,8 @@ - - + + diff --git a/script/git_show_divergence_repo_manifest.py b/script/git_show_divergence_repo_manifest.py new file mode 100755 index 0000000..d8e4741 --- /dev/null +++ b/script/git_show_divergence_repo_manifest.py @@ -0,0 +1,75 @@ +#!./.venv/bin/python +import os +import sys +import argparse +import logging +from git import Repo +from git.exc import GitCommandError + +new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(new_path) + +from script.git_tool import GitTool + +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""Compare actual code with a manifest.""", + epilog='''\ +''' + ) + parser.add_argument('-m', '--manifest', required=True, + help="The manifest to compare with actual code.") + args = parser.parse_args() + return args + + +def main(): + config = get_config() + git_tool = GitTool() + + dct_remote, dct_project, default_remote = git_tool.get_manifest_xml_info(filename=config.manifest, add_root=True) + default_branch_name = default_remote.get("@revision", git_tool.default_branch) + i = 0 + total = len(dct_project) + for name, project in dct_project.items(): + i += 1 + path = project.get("@path") + print(f"{i}/{total} - {path}") + branch_name = project.get("@revision", default_branch_name) + organization = project.get("@remote") + if not organization: + print(f"ERROR missing @remote on project {path}.") + continue + + git_repo = Repo(path) + value = git_repo.git.branch("--show-current") + if not value: + # TODO maybe need to check divergence with local branch and not remote branch + commit_head = git_repo.git.rev_parse("HEAD") + try: + commit_branch = git_repo.git.rev_parse(f"{organization}/{branch_name}") + except GitCommandError: + print("ERROR Something wrong with this repo.") + continue + if commit_branch != commit_head: + print("WARNING Not on specified branch, got a divergence.") + else: + print("PASS Not on specified branch, no divergence.") + elif branch_name != value: + print(f"ERROR, manifest revision is {branch_name} and actual revision is {value}.") + else: + print("PASS") + + +if __name__ == '__main__': + main() diff --git a/script/git_tool.py b/script/git_tool.py index 8ce9186..997211e 100644 --- a/script/git_tool.py +++ b/script/git_tool.py @@ -311,7 +311,7 @@ class GitTool: xml_as_string = xml.read() xml_dict = xmltodict.parse(xml_as_string, dict_constructor=dict) dct_manifest = xml_dict.get("manifest") - default_remote = dct_manifest.get("default").get("@remote") + default_remote = dct_manifest.get("default") lst_remote = dct_manifest.get("remote") lst_project = dct_manifest.get("project") dct_remote = {a.get("@name"): a for a in lst_remote} @@ -404,7 +404,7 @@ class GitTool: :param output: filename to write output :param dct_remote: dict of remote information :param dct_project: dict of project information - :param default_remote: name of default remote, optional + :param default_remote: dict of default remote :param keep_original: if True, can manage multiple organization with same name, but with different fetch url :return: @@ -490,7 +490,7 @@ class GitTool: if default_remote and not lst_default: lst_default.append(OrderedDict([ - ('@remote', default_remote), + ('@remote', default_remote.get("@remote")), ('@revision', DEFAULT_BRANCH), ('@sync-j', "4"), ('@sync-c', "true"), diff --git a/script/poetry_update.py b/script/poetry_update.py index 8947f30..ed661f4 100755 --- a/script/poetry_update.py +++ b/script/poetry_update.py @@ -123,14 +123,14 @@ def combine_requirements(config): if dct_requirements_diff_version: # Validate compatibility - for key, lst_requis in dct_requirements_diff_version.items(): + for key, lst_requirement in dct_requirements_diff_version.items(): result = None - lst_version_requis = [] - for requis in lst_requis: - if ".*" in requis: - requis = requis.replace(".*", "") - result_number = iscompatible.parse_requirements(requis) + lst_version_requirement = [] + for requirement in lst_requirement: + if ".*" in requirement: + requirement = requirement.replace(".*", "") + result_number = iscompatible.parse_requirements(requirement) if not result_number: # Ignore empty version continue @@ -144,13 +144,13 @@ def combine_requirements(config): result_number[0] = result_number[0][0], no_version[ :no_version.rfind(".")] result_number = iscompatible.string_to_tuple(result_number[0][1]) - lst_version_requis.append((requis, result_number)) + lst_version_requirement.append((requirement, result_number)) # Check compatibility with all possibility is_compatible = True - if len(lst_version_requis) > 1: - highest_value = sorted(lst_version_requis, key=lambda tup: tup[1])[-1] - for version_requis in lst_version_requis: - is_compatible &= iscompatible.iscompatible(version_requis[0], + if len(lst_version_requirement) > 1: + highest_value = sorted(lst_version_requirement, key=lambda tup: tup[1])[-1] + for version_requirement in lst_version_requirement: + is_compatible &= iscompatible.iscompatible(version_requirement[0], highest_value[1]) if is_compatible: result = highest_value[0] @@ -158,11 +158,11 @@ def combine_requirements(config): # Find the requirements file and print the conflict # Take the version from Odoo by default, else take the more recent odoo_value = None - for version_requis in lst_version_requis: + for version_requirement in lst_version_requirement: filename_1 = dct_requirements_module_filename.get( - version_requis[0]) + version_requirement[0]) if priority_filename_requirement in filename_1: - odoo_value = version_requis[0] + odoo_value = version_requirement[0] break if odoo_value: @@ -173,18 +173,18 @@ def combine_requirements(config): str_result_choose = f"Select highest value {result}" str_versions = " VS ".join( [f"{a[0]} from {dct_requirements_module_filename.get(a[0])}" for - a in lst_version_requis]) + a in lst_version_requirement]) print(f"WARNING - Not compatible {str_versions} - " f"{str_result_choose}.") - elif len(lst_version_requis) == 1: - result = lst_version_requis[0][0] + elif len(lst_version_requirement) == 1: + result = lst_version_requirement[0][0] else: result = key if result: dct_requirements[key] = set((result,)) else: - print(f"Internal error, missing result for {lst_requis}.") + print(f"Internal error, missing result for {lst_requirement}.") # Support ignored requirements lst_ignore = get_list_ignored() diff --git a/script/repo_remove_auto_install.py b/script/repo_remove_auto_install.py new file mode 100755 index 0000000..0c5ef88 --- /dev/null +++ b/script/repo_remove_auto_install.py @@ -0,0 +1,117 @@ +#!./.venv/bin/python +import os +import sys +import argparse +import logging +from pathlib import Path +from git import Repo # pip install gitpython +from git.exc import GitCommandError + +new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(new_path) + +from script.git_tool import GitTool + +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='''\ +''', + epilog='''\ +''' + ) + parser.add_argument('-d', '--dir', dest="dir", default="./", + help="Path of repo to change remote, including submodule.") + args = parser.parse_args() + return args + + +def main(): + config = get_config() + git_tool = GitTool() + + lst_repo = git_tool.get_source_repo_addons(repo_path=config.dir, + add_repo_root=False) + lst_repo_organization = [git_tool.get_transformed_repo_info_from_url( + a.get("url"), repo_path=config.dir, get_obj=True, + is_submodule=a.get("is_submodule"), sub_path=a.get("sub_path"), + revision=a.get("revision"), clone_depth=a.get("clone_depth")) + for a in lst_repo] + + lst_ignore_repo = ["odoo"] + + i = 0 + total = len(lst_repo) + branch_name = "12.0_dev" + for repo in lst_repo_organization: + i += 1 + print(f"\nNb element {i}/{total} - {repo.path}") + is_checkout_branch = False + + if repo.repo_name in lst_ignore_repo: + print(f"Ignore {repo.repo_name}.") + continue + + git_repo = Repo(repo.relative_path) + # Force checkout branch if exist + try: + git_repo.git.checkout(branch_name) + is_checkout_branch = True + except GitCommandError: + try: + git_repo.git.checkout("-t", f"{repo.organization}/{branch_name}") + is_checkout_branch = True + except GitCommandError: + pass + + has_change = get_manifest_external_dependencies(repo) + + if has_change: + if not is_checkout_branch: + git_repo.git.checkout("-b", branch_name) + # change branch, commit and push + git_repo.git.add(".") + git_repo.git.commit("-m", "Set all module auto_install at False") + git_repo.git.push("-u", repo.organization, branch_name) + + +def get_lst_manifest_py(relative_path): + return list(Path(relative_path).rglob("__manifest__.py")) + + +def get_manifest_external_dependencies(repo): + has_change = False + lst_manifest_file = get_lst_manifest_py(repo.relative_path) + for manifest_file in lst_manifest_file: + has_change_manifest = False + with open(manifest_file, 'r') as f: + lst_content = f.readlines() + i = 0 + for content in lst_content: + if "auto_install" in content and "True" in content: + has_change_manifest = True + has_change = True + first_char_index = content.find("auto_install") + index = content.find("True", first_char_index) + lst_content[i] = content[:index] + "False" + content[index + 4:] + i += 1 + + if has_change_manifest: + print(f"Update file {manifest_file}") + with open(manifest_file, 'w') as f: + f.writelines(lst_content) + + return has_change + + +if __name__ == '__main__': + main()