From b35d5db6c566b58045b8eea6d79cc881e9ba975f Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Sat, 17 Jul 2021 01:32:43 -0400 Subject: [PATCH] [ADD] script to compare csv and zip manifest from backup - show module difference --- Makefile | 5 ++ script/csv/compare_database_application.py | 67 +++++++++++++++ script/manifest/compare_backup.py | 99 ++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100755 script/csv/compare_database_application.py create mode 100755 script/manifest/compare_backup.py diff --git a/Makefile b/Makefile index 6526f73..2f6de51 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,11 @@ image_db_create_all: ./script/addons/install_addons.sh test website_sale,erplibre_base_quebec ./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_base +.PHONY: image_diff_base_website +image_diff_base_website: + #./script/manifest/compare_backup.py --backup_file_1 ./image_db/erplibre_base.zip --backup_file_2 ./image_db/erplibre_website.zip + ./script/manifest/compare_backup.py --backup_1 erplibre_base --backup_2 erplibre_website + ######################### # Addons installation # ######################### diff --git a/script/csv/compare_database_application.py b/script/csv/compare_database_application.py new file mode 100755 index 0000000..15421a0 --- /dev/null +++ b/script/csv/compare_database_application.py @@ -0,0 +1,67 @@ +#!./.venv/bin/python +import os +import sys +import argparse +import logging +import csv + +new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.append(new_path) + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) + +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + Take two csv and show difference, intersection and complementarity +""", + epilog="""\ +""", + ) + parser.add_argument("--csv_1", required=True, help="CSV file first") + parser.add_argument("--csv_2", required=True, help="CSV file second") + parser.add_argument("--compare_key", required=True, help="Key to search and compare with it.") + args = parser.parse_args() + return args + + +def main(): + config = get_config() + + with open(config.csv_1, mode='r') as csv_file_1: + with open(config.csv_2, mode='r') as csv_file_2: + csv_reader_1 = csv.DictReader(csv_file_1) + csv_reader_2 = csv.DictReader(csv_file_2) + lst_csv_1 = [a.get(config.compare_key) for a in csv_reader_1] + lst_csv_2 = [a.get(config.compare_key) for a in csv_reader_2] + set_csv_1 = set(lst_csv_1) + set_csv_2 = set(lst_csv_2) + + total = set_csv_1.union(set_csv_2) + same = set_csv_1.intersection(set_csv_2) + difference_1 = set_csv_1.difference(set_csv_2) + difference_2 = set_csv_2.difference(set_csv_1) + print(f"{len(total)} total") + print(f"{len(same)} same") + if same: + print(same) + print(f"{len(difference_1)} difference csv 1 to csv 2") + if difference_1: + print(difference_1) + print(f"{len(difference_2)} difference csv 2 to csv 1") + if difference_2: + print(difference_2) + + +if __name__ == "__main__": + main() diff --git a/script/manifest/compare_backup.py b/script/manifest/compare_backup.py new file mode 100755 index 0000000..e62a853 --- /dev/null +++ b/script/manifest/compare_backup.py @@ -0,0 +1,99 @@ +#!./.venv/bin/python +import os +import sys +import argparse +import logging +import zipfile +import json +from colorama import Fore +from colorama import Style + +new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.append(new_path) + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) + +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + Take two backup and show difference, intersection and complementarity of modules in manifest.json +""", + epilog="""\ +""", + ) + parser.add_argument("--backup_file_1", help="Backup file path first") + parser.add_argument("--backup_file_2", help="Backup file path second") + parser.add_argument("--backup_1", help="Backup name first") + parser.add_argument("--backup_2", help="Backup name second") + args = parser.parse_args() + + die(bool(args.backup_file_1) and bool(args.backup_1), + "Take only --backup_file_1 or --backup_1") + die(not bool(args.backup_file_1) and not bool(args.backup_1), + "Missing --backup_file_1 or --backup_1") + die(bool(args.backup_file_2) and bool(args.backup_2), + "Take only --backup_file_2 or --backup_2") + die(not bool(args.backup_file_2) and not bool(args.backup_2), + "Missing --backup_file_2 or --backup_2") + + return args + + +def main(): + config = get_config() + + if config.backup_file_1: + file_path_1 = config.backup_file_1 + else: + file_path_1 = os.path.join("image_db", f"{config.backup_1}.zip") + + if config.backup_file_2: + file_path_2 = config.backup_file_2 + else: + file_path_2 = os.path.join("image_db", f"{config.backup_2}.zip") + + with zipfile.ZipFile(file_path_1, 'r') as zip_ref: + manifest_file_1 = zip_ref.open("manifest.json") + + with zipfile.ZipFile(file_path_2, 'r') as zip_ref: + manifest_file_2 = zip_ref.open("manifest.json") + + json_manifest_file_1 = json.load(manifest_file_1) + json_manifest_file_2 = json.load(manifest_file_2) + set_1 = set(json_manifest_file_1.get("modules").keys()) + set_2 = set(json_manifest_file_2.get("modules").keys()) + + total = set_1.union(set_2) + same = set_1.intersection(set_2) + difference_1 = set_1.difference(set_2) + difference_2 = set_2.difference(set_1) + print(f"{len(total)} total") + print(f"{len(same)} same") + if same: + print(same) + print(f"{Fore.BLUE}{len(difference_1)}{Style.RESET_ALL} difference manifest 1 to manifest 2") + if difference_1: + print(difference_1) + print(f"{Fore.MAGENTA}{len(difference_2)}{Style.RESET_ALL} difference manifest 2 to manifest 1") + if difference_2: + print(difference_2) + + +def die(cond, message, code=1): + if cond: + print(message, file=sys.stderr) + sys.exit(code) + + +if __name__ == "__main__": + main()